You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudVpcPortService.php

2655 lines
112 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudHost;
use App\Models\CloudHostVpcTransport;
use App\Models\CloudInstance;
use App\Models\CloudInstanceMigration;
use App\Models\CloudJob;
use App\Models\CloudResourcePool;
use App\Models\CloudVpc;
use App\Models\CloudVpcHostState;
use App\Models\CloudVpcMembershipBatch;
use App\Models\CloudVpcMembershipBatchItem;
use App\Models\CloudVpcPort;
/**
* Durable VPC attachment orchestration.
*
* Attach order is host projections -> cold VM port attach. Detach order is
* cold VM port detach -> host projection cleanup -> IP release. Every Agent
* mutation has one operation ID and a row-level fence; an uncertain effect is
* never replayed.
*/
final class CloudVpcPortService
{
private const DELETED_CLEANUP_RETRY_DELAY = 300;
private const JOB_KINDS = [
'vpc_host_apply', 'vpc_host_remove',
'vpc_port_attach', 'vpc_port_detach',
];
/**
* Freeze the optional logical port before the source VM is stopped. This
* is a read-only projection preflight: the target transport must already
* be able to carry the exact port, but routing is not changed yet.
*/
public static function migrationPortIdLocked(
CloudInstance $instance,
int $targetHostId
): ?int {
$reference = CloudVpcPort::query()
->where('instance_id', (int) $instance->id)
->where('active_claim', 1)->first();
if ($reference === null) {
return null;
}
$vpc = CloudVpc::query()->where('id', (int) $reference->vpc_id)
->lockForUpdate()->first();
$port = CloudVpcPort::query()->where('id', (int) $reference->id)
->where('active_claim', 1)->lockForUpdate()->first();
if ($vpc === null || $port === null
|| (int) $port->instance_id !== (int) $instance->id
|| (int) $port->user_id !== (int) $instance->user_id
|| (int) $vpc->user_id !== (int) $instance->user_id
|| (string) $port->state !== 'attached'
|| $port->migration_host_id !== null
|| self::activePortJobLocked($instance, $port) !== null) {
throw new CloudError('实例 VPC 端口尚未收敛,不能维护迁移', 409);
}
self::assertVpcSettledLocked($vpc);
[$rows, $hostIds] = self::projectionRowsLocked(
$vpc, [(int) $port->id => $targetHostId]
);
$projections = CloudVpcProjection::build([
'vpc_key' => (string) $vpc->vpc_key,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
], $rows, self::readyTransports($hostIds),
(int) $vpc->desired_revision + 1);
if (!isset($projections[$targetHostId])) {
throw new CloudError('目标宿主机不能承载该 VPC 端口', 409);
}
return (int) $port->id;
}
/** @return list<CloudJob> */
public static function beginMigrationToTargetLocked(
CloudInstanceMigration $migration,
CloudInstance $instance
): array {
return self::beginMigrationProjectionLocked(
$migration, $instance, 'target'
);
}
/** @return list<CloudJob> */
public static function beginMigrationToSourceLocked(
CloudInstanceMigration $migration,
CloudInstance $instance
): array {
return self::beginMigrationProjectionLocked(
$migration, $instance, 'source'
);
}
/**
* Mark one migration projection complete only after every affected host
* has committed the same generation. Source completion also restores the
* ordinary attached-port state before the source VM may be restarted.
*/
public static function finishMigrationProjectionLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
string $direction
): bool {
if ($migration->vpc_port_id === null) {
return true;
}
$targetHostId = $direction === 'target'
? (int) $migration->target_host_id
: (int) $migration->source_host_id;
if (!in_array($direction, ['target', 'source'], true)
|| (int) $migration->instance_id !== (int) $instance->id) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 路由方向或实例账本无效', 409
);
}
[$port, $vpc] = self::migrationPortAndVpcLocked(
(int) $migration->vpc_port_id
);
if ((int) $port->instance_id !== (int) $instance->id
|| (string) $port->state !== 'migrating'
|| (int) ($port->migration_host_id ?? 0) !== $targetHostId
|| (int) $port->user_id !== (int) $instance->user_id
|| (int) $vpc->user_id !== (int) $instance->user_id) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 端口位置账本不一致', 409
);
}
$states = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
->lockForUpdate()->get();
if ($states->isEmpty() || $states->contains(
static fn ($state): bool => !self::hostReady($state)
)) {
return false;
}
$targetState = $states->firstWhere('host_id', $targetHostId);
if ($targetState === null
|| (int) $targetState->applied_revision
!== (int) $vpc->desired_revision) {
throw new CloudRecoveryInstanceStateConflict(
'迁移目标宿主机缺少已收敛的 VPC 投影', 409
);
}
$port->desired_revision = (int) $vpc->desired_revision;
$port->applied_revision = (int) $vpc->desired_revision;
if ($direction === 'source') {
$port->state = 'attached';
$port->migration_host_id = null;
}
$port->last_error_code = '';
$port->updated_at = time();
$port->save();
$vpc->state = 'active';
$vpc->updated_at = time();
$vpc->save();
return true;
}
/** Exact target-side VPC identity injected into PeerMigration receive. */
public static function migrationTargetMetadataLocked(
CloudInstanceMigration $migration,
CloudInstance $instance
): ?array {
if ($migration->vpc_port_id === null) {
return null;
}
[$port, $vpc] = self::migrationPortAndVpcLocked(
(int) $migration->vpc_port_id
);
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
->where('host_id', (int) $migration->target_host_id)
->lockForUpdate()->first();
if ((int) $port->instance_id !== (int) $instance->id
|| !self::hostReady($state)
|| (string) $port->state !== 'migrating'
|| (int) ($port->migration_host_id ?? 0)
!== (int) $migration->target_host_id
|| (int) $port->applied_revision
!== (int) $vpc->desired_revision
|| (int) $port->user_id !== (int) $instance->user_id) {
throw new CloudRecoveryInstanceStateConflict(
'迁移目标 VPC 端口尚未收敛', 409
);
}
return [
'port_key' => (string) $port->port_key,
'vpc_key' => (string) $vpc->vpc_key,
'private_ip' => (string) $port->private_ip,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
'mac' => (string) $port->mac_address,
'bridge' => CloudVpcProjection::bridgeName(
(string) $vpc->vpc_key
),
'host_revision' => (int) $state->applied_revision,
'host_digest' => (string) $state->applied_digest,
];
}
/** Atomically hand the logical port back to normal placement at cutover. */
public static function finalizeMigrationCutoverLocked(
CloudInstanceMigration $migration,
CloudInstance $instance
): void {
if ($migration->vpc_port_id === null) {
return;
}
self::migrationTargetMetadataLocked($migration, $instance);
$port = CloudVpcPort::query()
->where('id', (int) $migration->vpc_port_id)
->where('active_claim', 1)->lockForUpdate()->firstOrFail();
$port->state = 'attached';
$port->migration_host_id = null;
$port->last_error_code = '';
$port->updated_at = time();
$port->save();
}
public static function listing(CloudInstance $instance,
array $actor): array
{
self::assertOwnerAndInstance($instance, $actor, false);
$decoded = json_decode((string) $instance->spec_json, true);
$publicSpec = CloudInstanceMetadata::publicSpec(
is_array($decoded) && !array_is_list($decoded) ? $decoded : []
);
$networkValid =
($publicSpec['public_network_valid'] ?? false) === true;
$supported = $networkValid
&& ($publicSpec['os_source'] ?? null) === 'cloud_template';
$regionId = self::instanceRegionId($instance);
$port = CloudVpcPort::query()
->where('instance_id', (int) $instance->id)
->where('active_claim', 1)->first();
$vpcs = CloudVpc::query()
->where('user_id', (int) $instance->user_id)
->where('region_id', $regionId)->where('active_claim', 1)
->whereIn('state', ['active', 'degraded'])
->orderBy('name')->orderBy('id')->get();
$blocking = CloudJobService::blockingQueryForInstance(
(int) $instance->id
)->orderBy('id')->first();
$activeVpcJob = $port === null ? null
: self::activePortJob((int) $instance->id, (int) $port->id);
return [
'port' => $port === null ? null : self::portDto($port),
'vpcs' => $vpcs->map(static fn (CloudVpc $vpc): array => [
'vpc_key' => (string) $vpc->vpc_key,
'name' => (string) $vpc->name,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
'state' => (string) $vpc->state,
'revision' => (int) $vpc->desired_revision,
])->all(),
'public_network_valid' => $networkValid,
'can_attach' => $port === null && $blocking === null
&& $supported
&& (string) $instance->status === 'active'
&& $instance->serviceAvailable(),
'can_detach' => $port !== null && $blocking === null
&& (string) $port->state === 'attached'
&& (string) $instance->status === 'active'
&& $instance->serviceAvailable(),
'can_retry_attach' => $port !== null && $blocking === null
&& $supported
&& (string) $port->state === 'preparing'
&& (string) $instance->status === 'active'
&& $instance->serviceAvailable(),
'can_retry_detach' => $port !== null && $blocking === null
&& (string) $port->state === 'release_pending'
&& (string) $instance->status === 'active'
&& $instance->serviceAvailable(),
'active_task' => $activeVpcJob === null ? null : [
'task_id' => (string) $activeVpcJob->id,
'status_url' => '/api/v1/tasks/' . (int) $activeVpcJob->id,
],
'requires_cold_restart' => true,
];
}
/**
* One database-backed read model for the user VPC management page.
*
* The browser must not fan out one request per instance. This projection
* keeps the same ownership, service-state and resource-fence rules used by
* listing(), while returning every eligible instance in one response.
*/
public static function managementListing(array $actor): array
{
$userId = (int) ($actor['user_id'] ?? 0);
if ($userId < 1) {
throw new CloudError('请先登录', 401);
}
$instances = CloudInstance::query()->where('user_id', $userId)
->where('status', '<>', 'deleted')->orderByDesc('id')->get([
'id', 'uuid', 'host_id', 'user_id', 'resource_pool_id',
'name', 'status', 'service_status', 'network_status',
'expires_at', 'spec_json',
]);
if ($instances->isEmpty()) {
return [];
}
$poolIds = $instances->pluck('resource_pool_id')->filter(
static fn ($id): bool => (int) $id > 0
)->map(static fn ($id): int => (int) $id)->unique()->values()->all();
$regionByPool = $poolIds === [] ? collect() :
CloudResourcePool::query()
->join('cloud_network_lines as vpc_line',
'vpc_line.id', '=', 'cloud_resource_pools.line_id')
->whereIn('cloud_resource_pools.id', $poolIds)
->get([
'cloud_resource_pools.id as pool_id',
'vpc_line.region_id as region_id',
])->mapWithKeys(static fn ($row): array => [
(int) $row->pool_id => (int) $row->region_id,
]);
$instanceIds = $instances->pluck('id')->map(
static fn ($id): int => (int) $id
)->all();
$ports = CloudVpcPort::query()->whereIn('instance_id', $instanceIds)
->where('active_claim', 1)->get()->keyBy('instance_id');
$vpcIds = $ports->pluck('vpc_id')->map(
static fn ($id): int => (int) $id
)->unique()->values()->all();
$vpcs = $vpcIds === [] ? collect() : CloudVpc::query()
->whereIn('id', $vpcIds)->get()->keyBy('id');
$jobs = CloudJobService::blockingQueryForInstances($instanceIds)
->orderByDesc('id')->get([
'id', 'instance_id', 'vpc_port_id', 'kind',
])->groupBy('instance_id');
$result = [];
foreach ($instances as $instance) {
$port = $ports->get((int) $instance->id);
$decoded = json_decode((string) $instance->spec_json, true);
$spec = CloudInstanceMetadata::publicSpec(
is_array($decoded) && !array_is_list($decoded) ? $decoded : []
);
$networkValid = ($spec['public_network_valid'] ?? false) === true;
$supported = $networkValid
&& ($spec['os_source'] ?? null) === 'cloud_template';
$publicIpv4 = $networkValid
? (string) ($spec['public_ipv4'] ?? '') : '';
$publicIpv6 = $networkValid
? (string) ($spec['public_ipv6'] ?? '') : '';
// Keep a historical attached port visible for status and
// diagnosis even if its immutable source metadata is damaged.
if (!$supported && $port === null) {
continue;
}
$regionId = (int) ($regionByPool->get(
(int) ($instance->resource_pool_id ?? 0)
) ?? 0);
if ($regionId < 1 && $port === null) {
continue;
}
$instanceJobs = $jobs->get((int) $instance->id, collect());
$blocking = $instanceJobs->first();
$activeVpcJob = $port === null ? null : $instanceJobs->first(
static fn ($job): bool =>
(int) ($job->vpc_port_id ?? 0) === (int) $port->id
&& in_array((string) $job->kind,
self::JOB_KINDS, true)
);
$serviceAvailable = (string) $instance->status === 'active'
&& $instance->serviceAvailable();
$portState = $port === null ? '' : (string) $port->state;
$reason = '';
if (!$networkValid) {
$reason = $port === null
? '实例公网配置异常,暂不能加入 VPC 私网'
: '实例公网配置异常,仅允许移出 VPC 私网';
} elseif (!$supported) {
$reason = '该实例的系统来源不支持 VPC 私网网卡';
} elseif (!$serviceAvailable) {
$reason = '实例服务已到期或当前不可修改';
} elseif ($blocking !== null) {
$reason = '实例正在执行其他任务';
} elseif ($port === null && $regionId < 1) {
$reason = '实例缺少有效的地区资源池';
}
$vpc = $port === null ? null : $vpcs->get((int) $port->vpc_id);
$result[] = [
'id' => (string) $instance->id,
'name' => (string) $instance->name,
'region_id' => $regionId,
'public_ipv4' => $publicIpv4,
'public_ipv6' => $publicIpv6,
'public_network_valid' => $networkValid,
'instance_status' => (string) $instance->status,
'service_status' => (string) $instance->service_status,
'network_status' => (string) $instance->network_status,
'port' => $port === null ? null : self::portDto($port, $vpc),
'can_attach' => $port === null && $blocking === null
&& $supported && $regionId > 0 && $serviceAvailable,
'can_detach' => $port !== null && $blocking === null
&& $portState === 'attached'
&& $serviceAvailable,
'can_retry_attach' => $port !== null && $blocking === null
&& $supported && $portState === 'preparing'
&& $serviceAvailable,
'can_retry_detach' => $port !== null && $blocking === null
&& $portState === 'release_pending'
&& $serviceAvailable,
'operation_blocked' => $blocking !== null,
'action_unavailable_reason' => $reason,
'active_task' => $activeVpcJob === null ? null : [
'task_id' => (string) $activeVpcJob->id,
'status_url' => '/api/v1/tasks/' . (int) $activeVpcJob->id,
],
];
}
return $result;
}
/** Reserve one IP and enqueue complete host projections. */
public static function attach(CloudInstance $instance, array $actor,
array $input): array
{
CloudInput::only($input, ['vpc_key', 'confirm_restart']);
$vpcKey = $input['vpc_key'] ?? null;
if (!is_string($vpcKey)
|| preg_match('/^[a-f0-9]{32}$/D', $vpcKey) !== 1) {
throw new CloudError('请选择有效的 VPC 私网');
}
if (($input['confirm_restart'] ?? null) !== true) {
throw new CloudError('请确认实例会自动关机并冷启动', 409);
}
$requestKey = self::requestKey($actor);
$connection = (new CloudVpcPort())->getConnection();
return $connection->transaction(static function () use (
$instance, $actor, $vpcKey, $requestKey
): array {
$locked = CloudVpcLockOrder::instance((int) $instance->id);
self::assertOwnerAndInstance($locked, $actor, true);
self::assertPublicNetworkSnapshot($locked);
$vpc = CloudVpcLockOrder::vpcByKey($vpcKey);
if ($vpc === null || (int) $vpc->user_id !== (int) $locked->user_id
|| $vpc->active_claim === null) {
throw new CloudError('VPC 不存在或当前不可用', 404);
}
if ((int) $vpc->region_id !== self::instanceRegionId($locked)) {
throw new CloudError('实例只能加入同一地区的 VPC', 409);
}
$port = CloudVpcPort::query()
->where('instance_id', (int) $locked->id)
->where('active_claim', 1)->lockForUpdate()->first();
if ($port !== null) {
if ((int) $port->vpc_id !== (int) $vpc->id
|| (int) $port->user_id !== (int) $locked->user_id) {
throw new CloudError('实例已连接其他 VPC', 409);
}
if ((string) $port->state === 'attached') {
return ['port' => self::portDto($port), 'job' => null,
'unchanged' => true];
}
if ((string) $port->state !== 'preparing') {
throw new CloudError('VPC 端口正在执行其他操作', 409);
}
$active = self::activePortJobLocked($locked, $port);
if ($active !== null) {
return ['port' => self::portDto($port), 'job' => $active,
'unchanged' => false];
}
self::assertNoInstanceJobLocked($locked);
$job = self::retryPreparingLocked($locked, $vpc, $port);
return ['port' => self::portDto($port->fresh()),
'job' => $job, 'unchanged' => false];
}
if (!in_array((string) $vpc->state,
['active', 'degraded'], true)) {
throw new CloudError('VPC 当前正在执行其他操作', 409);
}
self::assertNoInstanceJobLocked($locked);
self::assertVpcSettledLocked($vpc);
$now = time();
$port = new CloudVpcPort();
$port->port_key = self::portKey(
(int) $locked->user_id, (int) $locked->id,
$vpcKey, $requestKey
);
$existing = CloudVpcPort::query()
->where('port_key', (string) $port->port_key)
->lockForUpdate()->first();
if ($existing !== null) {
throw new CloudError('该请求已结束,请使用新的请求标识重试', 409);
}
$port->vpc_id = (int) $vpc->id;
$port->instance_id = (int) $locked->id;
$port->user_id = (int) $locked->user_id;
$port->private_ip = self::allocateIpLocked($vpc);
$port->mac_address = self::allocateMacLocked(
(string) $port->port_key
);
$port->active_claim = 1;
$port->state = 'preparing';
$revision = (int) $vpc->desired_revision + 1;
$port->desired_revision = $revision;
$port->applied_revision = 0;
$port->last_operation_id = null;
$port->last_error_code = '';
$port->attached_at = null;
$port->released_at = null;
$port->created_at = $port->updated_at = $now;
$port->save();
$vpc->desired_revision = $revision;
$vpc->state = 'provisioning';
$vpc->updated_at = $now;
$vpc->save();
$jobs = self::queueProjectionLocked($vpc, $port, $revision);
if ($jobs === []) {
throw new CloudError('VPC 路由投影没有可执行宿主机', 503);
}
return ['port' => self::portDto($port), 'job' => $jobs[0],
'unchanged' => false];
}, 3);
}
/** Detach the VM port before changing any host route. */
public static function detach(CloudInstance $instance, array $actor,
array $input): array
{
CloudInput::only($input, ['confirm_restart']);
if (($input['confirm_restart'] ?? null) !== true) {
throw new CloudError('请确认实例会自动关机并冷启动', 409);
}
self::requestKey($actor);
$connection = (new CloudVpcPort())->getConnection();
return $connection->transaction(static function () use (
$instance, $actor
): array {
$locked = CloudVpcLockOrder::instance((int) $instance->id);
self::assertOwnerAndInstance($locked, $actor, true);
$reference = CloudVpcPort::query()
->where('instance_id', (int) $locked->id)
->where('active_claim', 1)->first();
if ($reference === null) {
return ['port' => null, 'job' => null, 'unchanged' => true];
}
$vpc = CloudVpcLockOrder::vpcById((int) $reference->vpc_id);
$port = CloudVpcLockOrder::portById((int) $reference->id);
if ($vpc === null || $port === null
|| (int) $vpc->user_id !== (int) $locked->user_id
|| (int) $port->instance_id !== (int) $locked->id
|| (int) $port->vpc_id !== (int) $vpc->id) {
throw new CloudError('VPC 端口账本与所属人不一致', 409);
}
if ((string) $port->state === 'detaching') {
$active = self::activePortJobLocked($locked, $port);
if ($active === null) {
throw new CloudError('VPC 断开任务账本不完整', 409);
}
return ['port' => self::portDto($port), 'job' => $active,
'unchanged' => false];
}
if ((string) $port->state === 'release_pending') {
$active = self::activePortJobLocked($locked, $port);
if ($active !== null) {
return ['port' => self::portDto($port), 'job' => $active,
'unchanged' => false];
}
self::assertNoInstanceJobLocked($locked);
$job = self::retryReleaseLocked($locked, $vpc, $port);
return ['port' => $port->active_claim === null
? null : self::portDto($port->fresh()),
'job' => $job, 'unchanged' => false];
}
if ((string) $port->state !== 'attached') {
throw new CloudError('VPC 端口正在执行其他操作', 409);
}
self::assertNoInstanceJobLocked($locked);
self::assertVpcSettledLocked($vpc);
$hostState = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
->where('host_id', (int) $locked->host_id)
->lockForUpdate()->first();
if (!self::hostReady($hostState)) {
throw new CloudError('实例所在宿主机的 VPC 路由尚未收敛', 409);
}
$port->state = 'detaching';
$port->last_error_code = '';
$port->updated_at = time();
$port->save();
$job = self::queuePortJobLocked(
'vpc_port_detach', $locked, $vpc, $port, $hostState
);
return ['port' => self::portDto($port), 'job' => $job,
'unchanged' => false];
}, 3);
}
/**
* Start route/IP cleanup only after the Agent has proved the VM and every
* managed artifact are absent. The caller already holds the instance row
* lock inside the delete finalizer transaction. No private address is
* released until every resulting host projection has converged.
*
* @return list<CloudJob>
*/
public static function releaseAfterInstanceDeletedLocked(
CloudInstance $instance
): array {
$reference = CloudVpcPort::query()
->where('instance_id', (int) $instance->id)
->where('active_claim', 1)->first();
if ($reference === null) {
return [];
}
$vpc = CloudVpc::query()->where('id', (int) $reference->vpc_id)
->lockForUpdate()->first();
$port = CloudVpcPort::query()->where('id', (int) $reference->id)
->where('active_claim', 1)->lockForUpdate()->first();
if ($vpc === null || $port === null
|| (int) $port->instance_id !== (int) $instance->id
|| (int) $port->user_id !== (int) $instance->user_id
|| (int) $port->vpc_id !== (int) $vpc->id
|| (int) $vpc->user_id !== (int) $instance->user_id) {
throw new CloudError('删除实例的 VPC 端口账本不一致', 409);
}
if (self::activePortJobLocked($instance, $port) !== null) {
throw new CloudError('删除实例仍有 VPC 任务未收敛', 409);
}
if ((string) $port->state === 'release_pending') {
$job = self::retryReleaseLocked($instance, $vpc, $port);
return $job === null ? [] : [$job];
}
if (!in_array((string) $port->state,
['attached', 'preparing'], true)) {
throw new CloudError('删除实例的 VPC 端口状态不允许释放', 409);
}
$now = time();
$port->state = 'release_pending';
$port->last_operation_id = null;
$port->last_error_code = '';
$port->updated_at = $now;
$port->save();
$revision = (int) $vpc->desired_revision + 1;
$vpc->desired_revision = $revision;
$vpc->state = 'provisioning';
$vpc->updated_at = $now;
$vpc->save();
$jobs = self::queueProjectionLocked($vpc, $port, $revision);
if ($jobs === []) {
self::releasePortLocked($vpc, $port);
}
return $jobs;
}
/** Queue new, audited cleanup generations for deleted instances only. */
public static function retryDeletedCleanupDue(int $limit = 10): int
{
$limit = max(1, min(100, $limit));
$candidates = CloudVpcPort::query()
->join('cloud_instances as deleted_instance',
'deleted_instance.id', '=', 'cloud_vpc_ports.instance_id')
->where('cloud_vpc_ports.active_claim', 1)
->where('cloud_vpc_ports.state', 'release_pending')
->where('deleted_instance.status', 'deleted')
->orderBy('cloud_vpc_ports.id')
->limit($limit)
->get([
'cloud_vpc_ports.id as port_id',
'cloud_vpc_ports.instance_id as instance_id',
]);
$queued = 0;
foreach ($candidates as $candidate) {
try {
$queued += (new CloudInstance())->getConnection()
->transaction(static function () use ($candidate): int {
$instance = CloudInstance::query()
->where('id', (int) $candidate->instance_id)
->lockForUpdate()->first();
if ($instance === null
|| (string) $instance->status !== 'deleted') {
return 0;
}
$reference = CloudVpcPort::query()
->where('id', (int) $candidate->port_id)
->where('instance_id', (int) $instance->id)
->where('active_claim', 1)->first();
if ($reference === null
|| (string) $reference->state
!== 'release_pending') {
return 0;
}
if (!self::deletedCleanupRetryReadyLocked(
$reference, time()
)) {
return 0;
}
$jobs = self::releaseAfterInstanceDeletedLocked(
$instance
);
return count($jobs);
}, 3);
} catch (\Throwable $error) {
error_log('cloud VPC deleted-instance cleanup unavailable: '
. get_class($error));
}
}
return $queued;
}
private static function deletedCleanupRetryReadyLocked(
CloudVpcPort $port,
int $now
): bool {
if (self::activePortJob(
(int) $port->instance_id,
(int) $port->id
) !== null) {
return false;
}
$states = CloudVpcHostState::query()
->where('vpc_id', (int) $port->vpc_id)
->orderBy('host_id')->lockForUpdate()->get();
if ($states->every(static fn ($state): bool =>
self::hostReady($state))) {
return true;
}
foreach ($states as $state) {
if (self::hostReady($state)) {
continue;
}
if (self::knownNotAppliedHostFailureLocked(
$state, $now - self::DELETED_CLEANUP_RETRY_DELAY
) === null) {
return false;
}
}
return true;
}
/**
* Return the exact terminal generation only when it proves that no host
* mutation was applied. A failed UI state alone is never replay evidence.
*/
private static function knownNotAppliedHostFailureLocked(
CloudVpcHostState $state,
?int $completedNoLaterThan = null
): ?CloudJob {
if ((string) $state->sync_state !== 'failed'
|| !is_string($state->last_operation_id)
|| $state->last_operation_id === '') {
return null;
}
$job = CloudJob::query()
->where('operation_id', (string) $state->last_operation_id)
->where('vpc_id', (int) $state->vpc_id)
->where('host_id', (int) $state->host_id)
->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])
->lockForUpdate()->first();
if ($job === null
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (string) $job->effect_resolution !== 'not_applied'
|| (bool) $job->effect_may_have_applied
|| !(bool) $job->safe_to_resubmit
|| (bool) $job->finalization_incident) {
return null;
}
if ($completedNoLaterThan !== null
&& max((int) ($job->terminal_at ?? 0),
(int) ($job->updated_at ?? 0)) > $completedNoLaterThan) {
return null;
}
return $job;
}
/** Trusted worker marker for VPC jobs that must finish after acceptance. */
public static function isSystemJob(CloudJob $job, array $payload): bool
{
if (!in_array((string) $job->kind, self::JOB_KINDS, true)) {
return false;
}
if (($payload['system_vpc_orchestration'] ?? null) !== true) {
throw new CloudError('VPC 系统任务缺少可信编排标识', 409);
}
self::assertJobRunnable($job, $payload);
return true;
}
public static function assertJobRunnable(CloudJob $job,
array $payload): bool
{
$kind = (string) $job->kind;
if (!in_array($kind, self::JOB_KINDS, true)) {
return true;
}
$port = $job->vpc_port_id === null ? null
: CloudVpcPort::find((int) $job->vpc_port_id);
$vpc = $job->vpc_id === null ? null
: CloudVpc::find((int) $job->vpc_id);
$instance = $job->instance_id === null ? null
: CloudInstance::find((int) $job->instance_id);
$body = $payload['body'] ?? null;
if ($port === null || $vpc === null || $instance === null
|| !is_array($body) || array_is_list($body)
|| (int) $port->vpc_id !== (int) $vpc->id
|| (int) $port->instance_id !== (int) $instance->id
|| (int) $port->user_id !== (int) $instance->user_id
|| (int) $job->user_id !== (int) $instance->user_id
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id
|| (int) ($payload['vpc_id'] ?? 0) !== (int) $vpc->id
|| (int) ($payload['vpc_port_id'] ?? 0) !== (int) $port->id
|| ($body['operation_id'] ?? null) !== (string) $job->operation_id) {
throw new CloudError('VPC 任务与资源账本不一致', 409);
}
$repair = $job->supersedes_job_id !== null;
$prior = null;
if ($repair) {
$prior = CloudJob::query()->where(
'id', (int) $job->supersedes_job_id
)->first();
$rootId = (int) ($job->vpc_repair_root_job_id ?? 0);
if ($prior === null || $rootId < 1
|| ($payload['vpc_repair'] ?? null) !== true
|| (int) ($payload['vpc_repair_root_job_id'] ?? 0)
!== $rootId
|| ($body['vpc_repair'] ?? null) !== true
|| ($body['supersedes_operation_id'] ?? null)
!== (string) $prior->operation_id
|| (int) $prior->instance_id !== (int) $job->instance_id
|| (int) $prior->vpc_id !== (int) $job->vpc_id
|| (int) $prior->vpc_port_id !== (int) $job->vpc_port_id
|| (int) $prior->host_id !== (int) $job->host_id
|| (string) $prior->kind !== $kind
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'failed'
|| !(bool) $prior->effect_may_have_applied
|| (bool) $prior->safe_to_resubmit
|| (string) $prior->effect_resolution !== 'unknown'
|| (bool) $prior->finalization_incident
|| (int) ($prior->vpc_repair_root_job_id ?? $prior->id)
!== $rootId) {
throw new CloudError('VPC 修复任务继任账本不一致', 409);
}
} elseif (($payload['vpc_repair'] ?? false) === true
|| array_key_exists('vpc_repair', $body)
|| array_key_exists('supersedes_operation_id', $body)) {
throw new CloudError('普通 VPC 任务包含修复字段', 409);
}
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
->where('host_id', (int) $job->host_id)->first();
if ($state === null
|| !hash_equals((string) $state->last_operation_id,
(string) $job->operation_id)) {
throw new CloudError('VPC 宿主机任务已过期或账本不一致', 409);
}
$expected = $kind === 'vpc_host_apply'
? self::hostApplyBody($job, $vpc, $state, $payload)
: self::hostRemoveBody($job, $vpc, $state);
} else {
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
->where('host_id', (int) $instance->host_id)->first();
if (!self::hostReady($state)
|| !hash_equals((string) $port->last_operation_id,
(string) $job->operation_id)
|| (int) $job->host_id !== (int) $instance->host_id
|| ($kind === 'vpc_port_attach'
&& (string) $port->state !== 'preparing')
|| ($kind === 'vpc_port_detach'
&& (string) $port->state !== 'detaching')) {
throw new CloudError('VPC 网卡任务已过期或路由尚未收敛', 409);
}
$expected = self::portBody($job, $vpc, $port, $state);
}
if ($repair) {
$expected['vpc_repair'] = true;
$expected['supersedes_operation_id'] =
(string) $prior->operation_id;
}
$expectedTarget = in_array($kind,
['vpc_host_apply', 'vpc_host_remove'], true)
? '/v1/vpc/networks/' . rawurlencode((string) $vpc->vpc_key)
. ($kind === 'vpc_host_apply' ? '/apply' : '/remove')
. ($repair ? '/repair' : '')
: '/v1/vms/' . rawurlencode(strtolower((string) $instance->uuid))
. '/vpc-port' . ($repair ? '/repair' : '');
$expectedMethod = match ($kind) {
'vpc_host_apply', 'vpc_host_remove', 'vpc_port_attach' => 'POST',
'vpc_port_detach' => 'DELETE',
};
if (($payload['request_target'] ?? null) !== $expectedTarget
|| strtoupper((string) ($payload['method'] ?? '')) !== $expectedMethod
|| !self::sameDocument($body, $expected)) {
throw new CloudError('VPC 任务请求与持久账本不一致', 409);
}
return true;
}
public static function markJobRunning(CloudJob $job): void
{
$kind = (string) $job->kind;
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
CloudVpcHostState::query()
->where('vpc_id', (int) $job->vpc_id)
->where('host_id', (int) $job->host_id)
->where('last_operation_id', (string) $job->operation_id)
->where('sync_state', 'pending')->update([
'sync_state' => 'applying', 'updated_at' => time(),
]);
}
}
public static function finishJobLocked(CloudJob $job, string $status,
$result, array $payload, array $operation): void
{
$kind = (string) $job->kind;
if (!in_array($kind, self::JOB_KINDS, true)) {
return;
}
self::assertJobRunnable($job, $payload);
$confirmed = $operation['operation_id'] ?? null;
if ($job->dispatched_at !== null
&& (!is_string($confirmed)
|| !hash_equals((string) $job->operation_id, $confirmed))) {
throw new CloudError('Agent VPC 结果与任务操作编号不一致', 502);
}
$instance = CloudVpcLockOrder::instance((int) $job->instance_id);
$vpc = CloudVpcLockOrder::vpcById((int) $job->vpc_id);
$port = CloudVpcLockOrder::portById((int) $job->vpc_port_id);
if ($port === null || $vpc === null || $instance === null) {
throw new CloudError('VPC 任务资源账本已丢失', 409);
}
if ($job->supersedes_job_id !== null) {
if ($status === 'succeeded') {
self::settleRepairChainLocked($job);
} elseif ($status === 'failed') {
self::restoreRepairPredecessorLocked(
$job, $vpc, $port
);
return;
}
}
if ($job->vpc_batch_id !== null) {
CloudVpcMembershipBatchService::finishChildLocked(
$job, $status, $result, $operation, $vpc, $port, $instance
);
return;
}
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
self::finishHostJobLocked(
$job, $status, $result, $operation, $vpc, $port, $instance
);
if (isset($payload['maintenance_migration_id'])) {
CloudInstanceMigrationService::finishVpcRoutingJobLocked(
$job, $status, $payload
);
}
return;
}
self::finishPortJobLocked(
$job, $status, $result, $operation, $vpc, $port, $instance
);
}
/** Resolve only VPC safety metadata after the successor proves convergence. */
private static function settleRepairChainLocked(CloudJob $job): void
{
$rootId = (int) ($job->vpc_repair_root_job_id ?? 0);
$priorId = (int) ($job->supersedes_job_id ?? 0);
if ($rootId < 1 || $priorId < 1) {
throw new CloudError('VPC 修复继任链缺失', 409);
}
$seen = [(int) $job->id => true];
$now = time();
for ($depth = 0; $depth < 32; $depth++) {
if ($priorId < 1 || isset($seen[$priorId])) {
throw new CloudError('VPC 修复继任链包含循环', 409);
}
$seen[$priorId] = true;
$prior = CloudJob::query()->where('id', $priorId)
->lockForUpdate()->first();
if ($prior === null
|| (string) $prior->kind !== (string) $job->kind
|| (int) $prior->instance_id !== (int) $job->instance_id
|| (int) $prior->vpc_id !== (int) $job->vpc_id
|| (int) $prior->vpc_port_id !== (int) $job->vpc_port_id
|| (int) $prior->host_id !== (int) $job->host_id
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'failed'
|| !(bool) $prior->effect_may_have_applied
|| (bool) $prior->safe_to_resubmit
|| (string) $prior->effect_resolution !== 'unknown'
|| (bool) $prior->finalization_incident
|| (int) ($prior->vpc_repair_root_job_id ?? $prior->id)
!== $rootId) {
throw new CloudError('VPC 修复继任链已变化', 409);
}
$nextId = (int) ($prior->supersedes_job_id ?? 0);
$prior->effect_resolution = 'applied';
$prior->effect_resolved_at = $now;
$prior->effect_resolved_by = null;
$prior->effect_resolution_reason = substr(
'superseded_by_vpc_repair:' . (int) $job->id . ':'
. (string) $job->operation_id,
0,
500
);
$prior->available_at = 0;
$prior->read_at = null;
$prior->updated_at = $now;
$prior->save();
if ($nextId < 1) {
if ((int) $prior->id !== $rootId) {
throw new CloudError('VPC 修复根任务身份不一致', 409);
}
return;
}
$priorId = $nextId;
}
throw new CloudError('VPC 修复继任链超过安全上限', 409);
}
/** A proved no-effect repair restores the still-fenced predecessor tip. */
private static function restoreRepairPredecessorLocked(
CloudJob $job,
CloudVpc $vpc,
CloudVpcPort $port
): void {
$prior = CloudJob::query()->where(
'id', (int) $job->supersedes_job_id
)->lockForUpdate()->first();
if ($prior === null
|| (string) $prior->kind !== (string) $job->kind
|| (string) $prior->effect_resolution !== 'unknown'
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'failed') {
throw new CloudError('VPC 修复前序任务已变化', 409);
}
$now = time();
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $job->vpc_id)
->where('host_id', (int) $job->host_id)
->lockForUpdate()->first();
if ($state === null || !hash_equals(
(string) $state->last_operation_id,
(string) $job->operation_id
)) {
throw new CloudError('VPC 宿主机修复回退代次不一致', 409);
}
$state->last_operation_id = (string) $prior->operation_id;
$state->sync_state = 'unknown';
$state->last_error_code = 'vpc_result_uncertain';
$state->last_failure_at = $now;
$state->updated_at = $now;
$state->save();
} else {
if (!hash_equals(
(string) $port->last_operation_id,
(string) $job->operation_id
)) {
throw new CloudError('VPC 网卡修复回退代次不一致', 409);
}
$port->last_operation_id = (string) $prior->operation_id;
$port->last_error_code = 'vpc_result_uncertain';
$port->updated_at = $now;
$port->save();
}
if ($job->vpc_batch_id !== null
&& in_array((string) $job->kind,
['vpc_port_attach', 'vpc_port_detach'], true)) {
$item = CloudVpcMembershipBatchItem::query()
->where('batch_id', (int) $job->vpc_batch_id)
->where('instance_id', (int) $job->instance_id)
->where('vpc_port_id', (int) $job->vpc_port_id)
->lockForUpdate()->first();
if ($item === null
|| (int) $item->child_job_id !== (int) $job->id) {
throw new CloudError('VPC 批次修复回退代次不一致', 409);
}
$item->child_job_id = (int) $prior->id;
$item->state = 'unknown';
$item->last_error_code = 'vpc_result_uncertain';
$item->updated_at = $now;
$item->save();
}
$vpc->state = 'degraded';
$vpc->updated_at = $now;
$vpc->save();
}
public static function markJobUncertain(CloudJob $job): void
{
$kind = (string) $job->kind;
if (!in_array($kind, self::JOB_KINDS, true)) {
return;
}
$now = time();
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
CloudVpcHostState::query()
->where('vpc_id', (int) $job->vpc_id)
->where('host_id', (int) $job->host_id)
->where('last_operation_id', (string) $job->operation_id)
->update([
'sync_state' => 'unknown',
'last_error_code' => 'vpc_result_uncertain',
'last_failure_at' => $now, 'updated_at' => $now,
]);
}
CloudVpcPort::query()->where('id', (int) $job->vpc_port_id)
->where('active_claim', 1)->update([
'last_error_code' => 'vpc_result_uncertain',
'updated_at' => $now,
]);
CloudVpc::query()->where('id', (int) $job->vpc_id)->update([
'state' => 'degraded', 'updated_at' => $now,
]);
CloudVpcMembershipBatchService::markChildUncertainLocked($job);
if (!is_string($job->sealed_payload)
|| $job->sealed_payload === '') {
return;
}
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload, 'job-payload'
);
if (is_array($payload)
&& isset($payload['maintenance_migration_id'])) {
CloudInstanceMigrationService::finishVpcRoutingJobLocked(
$job, 'uncertain', $payload
);
}
}
/**
* Queue one new, explicitly linked repair generation for an exact VPC
* operation whose Agent journal is reachable but still inconclusive.
* The failed operation is never replayed.
*/
public static function repairUnknownEffect(
array $actor,
int $jobId,
array $inspection
): CloudJob {
CloudAccess::manage($actor);
$reference = CloudJob::query()->where('id', $jobId)->first();
if ($reference === null) {
throw new CloudError('任务不存在', 404);
}
if (!in_array((string) $reference->kind, self::JOB_KINDS, true)
|| (int) ($reference->instance_id ?? 0) < 1
|| (int) ($reference->vpc_id ?? 0) < 1
|| (int) ($reference->vpc_port_id ?? 0) < 1
|| (int) ($reference->host_id ?? 0) < 1) {
throw new CloudError('该任务不是可修复的 VPC 操作', 409);
}
self::assertRepairInspection($reference, $inspection);
return (new CloudJob())->getConnection()->transaction(
function () use ($reference, $inspection): CloudJob {
$instance = CloudVpcLockOrder::instance(
(int) $reference->instance_id
);
$vpc = CloudVpcLockOrder::vpcById(
(int) $reference->vpc_id
);
$port = CloudVpcLockOrder::portById(
(int) $reference->vpc_port_id
);
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $reference->vpc_id)
->where('host_id', (int) $reference->host_id)
->lockForUpdate()->first();
if ($instance === null || $vpc === null || $port === null
|| $state === null
|| (int) $port->instance_id !== (int) $instance->id
|| (int) $port->vpc_id !== (int) $vpc->id
|| (int) $reference->host_id
!== (int) $state->host_id) {
throw new CloudError('VPC 修复资源账本不完整', 409);
}
$batch = null;
$item = null;
if ($reference->vpc_batch_id !== null) {
$batch = CloudVpcMembershipBatch::query()
->where('id', (int) $reference->vpc_batch_id)
->lockForUpdate()->first();
if ($batch === null
|| (string) $batch->state !== 'blocked'
|| (int) $batch->vpc_id !== (int) $vpc->id) {
throw new CloudError('VPC 批次不处于可修复状态', 409);
}
if (in_array((string) $reference->kind,
['vpc_port_attach', 'vpc_port_detach'], true)) {
$item = CloudVpcMembershipBatchItem::query()
->where('batch_id', (int) $batch->id)
->where('instance_id', (int) $instance->id)
->where('vpc_port_id', (int) $port->id)
->lockForUpdate()->first();
}
}
$prior = CloudJob::query()->where(
'id', (int) $reference->id
)->lockForUpdate()->first();
if ($prior === null) {
throw new CloudError('待修复任务已不存在', 409);
}
self::assertRepairCandidateLocked(
$prior, $instance, $vpc, $port, $state,
$batch, $item, $inspection
);
$rootId = (int) ($prior->vpc_repair_root_job_id
?? $prior->id);
$chain = CloudJob::query()
->where(static function ($query) use ($rootId): void {
$query->where('id', $rootId)
->orWhere('vpc_repair_root_job_id', $rootId);
})->orderBy('id')->lockForUpdate()->get();
$unknownGenerations = $chain->filter(
static fn (CloudJob $candidate): bool =>
(string) $candidate->effect_resolution === 'unknown'
);
$latestUnknown = $unknownGenerations->last();
if ($chain->isEmpty() || $unknownGenerations->count() > 32
|| $chain->contains(static fn (CloudJob $candidate): bool =>
(string) $candidate->phase !== 'terminal')
|| !$latestUnknown instanceof CloudJob
|| (int) $latestUnknown->id !== (int) $prior->id) {
throw new CloudError('VPC 修复链已有其他活动代次', 409);
}
foreach ($chain as $candidate) {
if ((int) $candidate->instance_id !== (int) $instance->id
|| (int) $candidate->vpc_id !== (int) $vpc->id
|| (int) $candidate->vpc_port_id !== (int) $port->id
|| (int) $candidate->host_id !== (int) $prior->host_id
|| (string) $candidate->kind
!== (string) $prior->kind) {
throw new CloudError('VPC 修复链资源身份不一致', 409);
}
}
$payload = CloudSecrets::decrypt(
(string) $prior->sealed_payload, 'job-payload'
);
self::assertJobRunnable($prior, $payload);
$job = self::baseJob(
(string) $prior->kind,
$vpc,
$port,
(int) $prior->host_id,
$prior->vpc_batch_id === null
? null : (int) $prior->vpc_batch_id
);
$job->supersedes_job_id = (int) $prior->id;
$job->vpc_repair_root_job_id = $rootId;
$extra = [];
foreach ([
'maintenance_migration_id', 'vpc_migration_direction',
'vpc_batch_id', 'vpc_batch_revision',
] as $key) {
if (array_key_exists($key, $payload)) {
$extra[$key] = $payload[$key];
}
}
if ((string) $prior->kind === 'vpc_host_apply'
&& is_array($payload['host_projection'] ?? null)
&& !array_is_list($payload['host_projection'])) {
$extra['host_projection'] = $payload['host_projection'];
}
if ($job->vpc_batch_id !== null
&& in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$revision = (int) ($extra['vpc_batch_revision']
?? $batch?->desired_revision ?? 0);
if ($revision < 1) {
throw new CloudError('VPC 批次修复缺少投影代次', 409);
}
$job->request_idempotency_key = substr(
self::batchProjectionRequestPrefix(
(int) $job->vpc_batch_id,
$revision
) . (string) $job->kind . '-'
. (int) $job->host_id . '-repair-'
. (string) $job->operation_id,
0,
128
);
}
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$body = (string) $job->kind === 'vpc_host_apply'
? self::hostApplyBody($job, $vpc, $state, $payload)
: self::hostRemoveBody($job, $vpc, $state);
$target = '/v1/vpc/networks/'
. rawurlencode((string) $vpc->vpc_key)
. ((string) $job->kind === 'vpc_host_apply'
? '/apply/repair' : '/remove/repair');
$method = 'POST';
} else {
if (!self::hostReady($state)) {
throw new CloudError(
'VPC 网卡修复所需宿主机路由未就绪', 409
);
}
$body = self::portBody($job, $vpc, $port, $state);
$target = '/v1/vms/' . rawurlencode(
strtolower((string) $instance->uuid)
) . '/vpc-port/repair';
$method = (string) $job->kind === 'vpc_port_attach'
? 'POST' : 'DELETE';
}
$body['vpc_repair'] = true;
$body['supersedes_operation_id'] =
(string) $prior->operation_id;
self::sealJob(
$job, $vpc, $port, $method, $target, $body,
$extra + [
'vpc_repair' => true,
'vpc_repair_root_job_id' => $rootId,
]
);
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$state->last_operation_id = (string) $job->operation_id;
$state->sync_state = 'pending';
$state->last_error_code = '';
$state->updated_at = time();
$state->save();
} else {
$port->last_operation_id = (string) $job->operation_id;
$port->last_error_code = '';
$port->updated_at = time();
$port->save();
if ($item !== null) {
$item->child_job_id = (int) $job->id;
$item->state = 'queued';
$item->last_error_code = '';
$item->updated_at = time();
$item->save();
}
}
return $job;
},
3
);
}
private static function assertRepairInspection(
CloudJob $job,
array $inspection
): void {
if (($inspection['operation_id'] ?? null)
!== (string) $job->operation_id
|| ($inspection['journal_status'] ?? null) !== 'failed'
|| ($inspection['outcome'] ?? null) !== 'inconclusive'
|| ($inspection['effect_resolution'] ?? null) !== 'inconclusive'
|| ($inspection['fence_state'] ?? null) !== 'active'
|| ($inspection['fence_resolution_required'] ?? null) !== true
|| array_key_exists('completion_delivery_state', $inspection)
|| !is_int($inspection['checked_at'] ?? null)
|| $inspection['checked_at'] < time() - 120
|| $inspection['checked_at'] > time() + 5) {
throw new CloudError(
'宿主机没有提供可执行 VPC 修复的新鲜不一致证据', 409
);
}
}
private static function assertRepairCandidateLocked(
CloudJob $job,
CloudInstance $instance,
CloudVpc $vpc,
CloudVpcPort $port,
CloudVpcHostState $state,
?CloudVpcMembershipBatch $batch,
?CloudVpcMembershipBatchItem $item,
array $inspection
): void {
self::assertRepairInspection($job, $inspection);
if (!in_array((string) $job->kind, self::JOB_KINDS, true)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (int) $job->dispatch_attempts !== 1
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown'
|| (bool) $job->finalization_incident
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->vpc_id !== (int) $vpc->id
|| (int) $job->vpc_port_id !== (int) $port->id
|| (int) $job->host_id !== (int) $state->host_id
|| (int) $job->user_id !== (int) $port->user_id) {
throw new CloudError('VPC 任务已不处于可修复状态', 409);
}
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
if (!hash_equals(
(string) $state->last_operation_id,
(string) $job->operation_id
) || (string) $state->sync_state !== 'unknown') {
throw new CloudError('VPC 宿主机修复代次已过期', 409);
}
} elseif (!hash_equals(
(string) $port->last_operation_id,
(string) $job->operation_id
) || !in_array((string) $port->state,
['preparing', 'detaching'], true)) {
throw new CloudError('VPC 网卡修复代次已过期', 409);
}
if ($job->vpc_batch_id !== null
&& ($batch === null || (string) $batch->state !== 'blocked'
|| (in_array((string) $job->kind,
['vpc_port_attach', 'vpc_port_detach'], true)
&& ($item === null
|| (int) $item->child_job_id !== (int) $job->id
|| (string) $item->state !== 'unknown')))) {
throw new CloudError('VPC 批次修复子任务已过期', 409);
}
}
private static function finishHostJobLocked(CloudJob $job,
string $status, $result, array $operation, CloudVpc $vpc,
CloudVpcPort $port, CloudInstance $instance): void
{
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
->where('host_id', (int) $job->host_id)
->lockForUpdate()->first();
if ($state === null
|| !hash_equals((string) $state->last_operation_id,
(string) $job->operation_id)) {
throw new CloudError('VPC 宿主机结果与当前投影不一致', 409);
}
$now = time();
if ($status === 'succeeded') {
self::assertHostResult($job, $vpc, $state, $result);
if ((string) $job->kind === 'vpc_host_remove') {
$state->delete();
} else {
$state->applied_revision = (int) $state->desired_revision;
$state->applied_digest = (string) $state->desired_digest;
$state->sync_state = 'ready';
$state->last_error_code = '';
$state->last_success_at = $now;
$state->updated_at = $now;
$state->save();
}
self::advancePortLocked($instance, $vpc, $port);
return;
}
$uncertain = $status === 'uncertain';
$code = $uncertain ? 'vpc_result_uncertain'
: (CloudOperationErrorCatalog::workerSummary(
(string) $job->kind, 'failed', $operation
) ?? 'vpc_host_apply_failed');
$state->sync_state = $uncertain ? 'unknown' : 'failed';
$state->last_error_code = substr($code, 0, 64);
$state->last_failure_at = $now;
$state->updated_at = $now;
$state->save();
$port->last_error_code = substr($code, 0, 64);
$port->updated_at = $now;
$port->save();
$vpc->state = 'degraded';
$vpc->updated_at = $now;
$vpc->save();
}
private static function finishPortJobLocked(CloudJob $job,
string $status, $result, array $operation, CloudVpc $vpc,
CloudVpcPort $port, CloudInstance $instance): void
{
$now = time();
if ($status !== 'succeeded') {
$uncertain = $status === 'uncertain';
$code = $uncertain ? 'vpc_result_uncertain'
: (CloudOperationErrorCatalog::workerSummary(
(string) $job->kind, 'failed', $operation
) ?? 'vpc_port_change_failed');
if ((string) $job->kind === 'vpc_port_detach' && !$uncertain) {
$port->state = 'attached';
}
$port->last_error_code = substr($code, 0, 64);
$port->updated_at = $now;
$port->save();
$vpc->state = 'degraded';
$vpc->updated_at = $now;
$vpc->save();
return;
}
self::assertPortResult($job, $vpc, $port, $instance, $result);
if ((string) $job->kind === 'vpc_port_attach') {
$port->state = 'attached';
$port->applied_revision = (int) $port->desired_revision;
$port->last_error_code = '';
$port->attached_at = $now;
$port->updated_at = $now;
$port->save();
$vpc->state = 'active';
$vpc->updated_at = $now;
$vpc->save();
return;
}
$port->state = 'release_pending';
$port->last_error_code = '';
$port->updated_at = $now;
$port->save();
$revision = (int) $vpc->desired_revision + 1;
$vpc->desired_revision = $revision;
$vpc->state = 'provisioning';
$vpc->updated_at = $now;
$vpc->save();
$jobs = self::queueProjectionLocked($vpc, $port, $revision);
if ($jobs === []) {
self::releasePortLocked($vpc, $port);
}
}
private static function advancePortLocked(CloudInstance $instance,
CloudVpc $vpc, CloudVpcPort $port): void
{
$states = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
->lockForUpdate()->get();
if ($states->contains(static fn ($state): bool =>
!self::hostReady($state))) {
return;
}
if ((string) $port->state === 'preparing') {
$hostState = $states->first(static fn ($state): bool =>
(int) $state->host_id === (int) $instance->host_id);
if ($hostState === null) {
throw new CloudError('实例宿主机缺少 VPC 路由投影', 409);
}
self::queuePortJobLocked(
'vpc_port_attach', $instance, $vpc, $port, $hostState
);
return;
}
if ((string) $port->state === 'release_pending') {
self::releasePortLocked($vpc, $port);
}
}
/** @return list<CloudJob> */
private static function beginMigrationProjectionLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
string $direction
): array {
if ($migration->vpc_port_id === null) {
return [];
}
if (!in_array($direction, ['target', 'source'], true)
|| (int) $migration->instance_id !== (int) $instance->id
|| (int) $instance->host_id
!== (int) $migration->source_host_id) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 路由切换实例账本不一致', 409
);
}
$destination = $direction === 'target'
? (int) $migration->target_host_id
: (int) $migration->source_host_id;
[$port, $vpc] = self::migrationPortAndVpcLocked(
(int) $migration->vpc_port_id
);
if ((int) $port->instance_id !== (int) $instance->id
|| (int) $port->user_id !== (int) $instance->user_id
|| (int) $vpc->user_id !== (int) $instance->user_id
|| self::activePortJobLocked($instance, $port) !== null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 端口任务账本不一致或仍未终结', 409
);
}
if ((string) $port->state === 'attached') {
if ($direction !== 'target' || $port->migration_host_id !== null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 端口尚未进入目标切换状态', 409
);
}
self::assertVpcSettledLocked($vpc);
} elseif ((string) $port->state !== 'migrating'
|| !in_array((int) ($port->migration_host_id ?? 0), [
(int) $migration->source_host_id,
(int) $migration->target_host_id,
], true)) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 端口位置状态无效', 409
);
}
$now = time();
$port->state = 'migrating';
$port->migration_host_id = $destination;
$port->last_operation_id = null;
$port->last_error_code = '';
$revision = (int) $vpc->desired_revision + 1;
$port->desired_revision = $revision;
$port->updated_at = $now;
$port->save();
$vpc->desired_revision = $revision;
$vpc->state = 'provisioning';
$vpc->updated_at = $now;
$vpc->save();
$jobs = self::queueProjectionLocked(
$vpc,
$port,
$revision,
[
'maintenance_migration_id' => (int) $migration->id,
'vpc_migration_direction' => $direction,
]
);
if ($jobs === []) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 路由切换没有生成宿主机任务', 409
);
}
return $jobs;
}
/** @return array{0:CloudVpcPort,1:CloudVpc} */
private static function migrationPortAndVpcLocked(
int $portId
): array {
$reference = CloudVpcPort::query()->where('id', $portId)
->where('active_claim', 1)->first();
$vpc = $reference === null ? null : CloudVpc::query()
->where('id', (int) $reference->vpc_id)
->lockForUpdate()->first();
$port = $vpc === null ? null : CloudVpcPort::query()
->where('id', $portId)
->where('vpc_id', (int) $vpc->id)
->where('active_claim', 1)->lockForUpdate()->first();
if ($port === null || $vpc === null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 端口或网络账本缺失', 409
);
}
return [$port, $vpc];
}
/**
* @param array<int,int> $hostOverrides port id => effective host id
* @return array{0:list<array{private_ip:string,mac_address:string,host_id:int}>,1:list<int>}
*/
private static function projectionRowsLocked(
CloudVpc $vpc,
array $hostOverrides = []
): array {
$ports = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
->where('active_claim', 1)
->whereIn('state', [
'preparing', 'attached', 'detaching', 'migrating',
])->orderBy('id')->lockForUpdate()->get();
$instanceIds = $ports->pluck('instance_id')->map(
static fn ($id): int => (int) $id
)->all();
$instances = $instanceIds === [] ? collect()
: CloudInstance::query()->whereIn('id', $instanceIds)
->orderBy('id')->get()->keyBy('id');
if ($instances->count() !== count(array_unique($instanceIds))) {
throw new CloudError('VPC 端口关联的实例账本缺失', 409);
}
$rows = [];
$hostIds = [];
$usedOverrides = [];
foreach ($ports as $port) {
$instance = $instances->get((int) $port->instance_id);
if ($instance === null
|| (int) $instance->user_id !== (int) $vpc->user_id
|| (string) $instance->status === 'deleted') {
throw new CloudError('VPC 端口关联的实例状态无效', 409);
}
$portId = (int) $port->id;
if (isset($hostOverrides[$portId])) {
$hostId = (int) $hostOverrides[$portId];
$usedOverrides[$portId] = true;
} elseif ((string) $port->state === 'migrating') {
$hostId = (int) ($port->migration_host_id ?? 0);
if ($hostId < 1) {
throw new CloudError('迁移 VPC 端口缺少目标宿主机', 409);
}
} else {
if ($port->migration_host_id !== null) {
throw new CloudError('普通 VPC 端口包含迁移位置残留', 409);
}
$hostId = (int) $instance->host_id;
}
if ($hostId < 1) {
throw new CloudError('VPC 端口宿主机账本无效', 409);
}
$hostIds[$hostId] = true;
$rows[] = [
'private_ip' => (string) $port->private_ip,
'mac_address' => (string) $port->mac_address,
'host_id' => $hostId,
];
}
if (count($usedOverrides) !== count($hostOverrides)) {
throw new CloudError('VPC 迁移端口不在当前投影中', 409);
}
$ids = array_map('intval', array_keys($hostIds));
sort($ids, SORT_NUMERIC);
return [$rows, $ids];
}
/** @return list<CloudJob> */
private static function queueProjectionLocked(CloudVpc $vpc,
CloudVpcPort $focusPort, int $revision,
array $jobContext = [],
?string $provenNotAppliedOperationId = null): array
{
[$projectionPorts, $hostIds] = self::projectionRowsLocked($vpc);
$transports = self::readyTransports($hostIds);
$projections = CloudVpcProjection::build([
'vpc_key' => (string) $vpc->vpc_key,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
], $projectionPorts, $transports, $revision);
$states = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
->lockForUpdate()->get()->keyBy('host_id');
$jobs = [];
$now = time();
foreach ($projections as $hostId => $projection) {
$state = $states->get($hostId);
if ($state === null) {
$state = new CloudVpcHostState();
$state->vpc_id = (int) $vpc->id;
$state->host_id = (int) $hostId;
$state->applied_revision = 0;
$state->applied_digest = null;
$state->last_success_at = 0;
$state->last_failure_at = 0;
$state->created_at = $now;
} elseif (!self::hostReady($state)
&& !self::isKnownNotAppliedHostState(
$state,
$provenNotAppliedOperationId
)) {
throw new CloudError(
'旧 VPC 宿主机投影尚未收敛,不能生成新一代', 409
);
}
$state->desired_revision = $revision;
$state->desired_digest = (string) $projection['digest'];
$state->sync_state = 'pending';
$state->last_error_code = '';
$state->updated_at = $now;
$state->save();
$job = self::queueHostJobLocked(
'vpc_host_apply', $vpc, $focusPort, $state,
$projection['network'], $jobContext
);
$jobs[] = $job;
$states->forget($hostId);
}
foreach ($states as $state) {
if (!self::hostReady($state)) {
if (!self::isKnownNotAppliedHostState(
$state,
$provenNotAppliedOperationId
)) {
throw new CloudError(
'旧 VPC 宿主机投影尚未收敛,不能清理', 409
);
}
$hasAppliedProjection = (int) $state->applied_revision > 0
&& preg_match('/^[a-f0-9]{64}$/D',
(string) $state->applied_digest) === 1;
if (!$hasAppliedProjection) {
// A known-not-applied first projection has no host state
// to remove. Dropping only this failed intent is safe.
$state->delete();
continue;
}
}
$state->sync_state = 'pending';
$state->last_error_code = '';
$state->updated_at = $now;
$state->save();
$jobs[] = self::queueHostJobLocked(
'vpc_host_remove', $vpc, $focusPort, $state, null,
$jobContext
);
}
return $jobs;
}
private static function retryPreparingLocked(CloudInstance $instance,
CloudVpc $vpc, CloudVpcPort $port): ?CloudJob
{
$states = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
->lockForUpdate()->get();
if ($states->isEmpty()) {
throw new CloudError('VPC 路由投影账本缺失', 409);
}
if ($states->contains(static fn ($state): bool =>
(string) $state->sync_state === 'unknown')) {
throw new CloudError('VPC 路由结果正在自动核对,请勿重复提交', 409);
}
if ($states->every(static fn ($state): bool => self::hostReady($state))) {
return self::queuePortJobLocked('vpc_port_attach', $instance,
$vpc, $port, $states->firstWhere('host_id', $instance->host_id));
}
$ports = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
->where('active_claim', 1)
->whereIn('state', ['preparing', 'attached', 'detaching'])
->orderBy('id')->get();
$instances = CloudInstance::query()
->whereIn('id', $ports->pluck('instance_id')->all())
->get()->keyBy('id');
$hostIds = [];
$rows = [];
foreach ($ports as $item) {
$bound = $instances->get((int) $item->instance_id);
if ($bound === null) {
throw new CloudError('VPC 端口关联的实例账本缺失', 409);
}
$hostIds[(int) $bound->host_id] = true;
$rows[] = ['private_ip' => (string) $item->private_ip,
'mac_address' => (string) $item->mac_address,
'host_id' => (int) $bound->host_id];
}
$projections = CloudVpcProjection::build([
'vpc_key' => (string) $vpc->vpc_key,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
], $rows, self::readyTransports(array_keys($hostIds)),
(int) $vpc->desired_revision);
foreach ($states as $state) {
if ((string) $state->sync_state !== 'failed') {
continue;
}
if (self::knownNotAppliedHostFailureLocked($state) === null) {
throw new CloudError(
'VPC 路由失败结果尚未证明未生效,不能重试', 409
);
}
$projection = $projections[(int) $state->host_id] ?? null;
if ($projection !== null
&& hash_equals((string) $state->desired_digest,
(string) $projection['digest'])
&& (int) $state->desired_revision
=== (int) $vpc->desired_revision) {
return self::queueHostJobLocked('vpc_host_apply', $vpc,
$port, $state, $projection['network']);
}
if ($projection === null && is_string($state->applied_digest)) {
return self::queueHostJobLocked('vpc_host_remove', $vpc,
$port, $state, null);
}
throw new CloudError('VPC 路由重试投影与持久账本不一致', 409);
}
throw new CloudError('VPC 路由任务仍在执行或等待核对', 409);
}
private static function retryReleaseLocked(CloudInstance $instance,
CloudVpc $vpc, CloudVpcPort $port): ?CloudJob
{
$states = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
->lockForUpdate()->get();
if ($states->contains(static fn ($state): bool =>
(string) $state->sync_state === 'unknown')) {
throw new CloudError('VPC 清理结果正在自动核对,请勿重复提交', 409);
}
if ($states->every(static fn ($state): bool => self::hostReady($state))) {
self::releasePortLocked($vpc, $port);
return null;
}
foreach ($states as $state) {
if ((string) $state->sync_state !== 'failed') {
continue;
}
if (self::knownNotAppliedHostFailureLocked($state) === null) {
throw new CloudError(
'VPC 清理失败结果尚未证明未生效,不能重试', 409
);
}
// Rebuild the desired topology. A state absent from the new
// projection is a remove retry; otherwise queue the exact apply.
$ports = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
->where('active_claim', 1)
->whereIn('state', ['preparing', 'attached', 'detaching'])
->get();
$instances = CloudInstance::query()
->whereIn('id', $ports->pluck('instance_id')->all())
->get()->keyBy('id');
$hostIds = [];
$rows = [];
foreach ($ports as $item) {
$bound = $instances->get((int) $item->instance_id);
if ($bound === null) {
throw new CloudError('VPC 端口关联的实例账本缺失', 409);
}
$hostIds[(int) $bound->host_id] = true;
$rows[] = ['private_ip' => (string) $item->private_ip,
'mac_address' => (string) $item->mac_address,
'host_id' => (int) $bound->host_id];
}
$projections = CloudVpcProjection::build([
'vpc_key' => (string) $vpc->vpc_key,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
], $rows, self::readyTransports(array_keys($hostIds)),
(int) $vpc->desired_revision);
$projection = $projections[(int) $state->host_id] ?? null;
return $projection === null
? self::queueHostJobLocked('vpc_host_remove', $vpc,
$port, $state, null)
: self::queueHostJobLocked('vpc_host_apply', $vpc,
$port, $state, $projection['network']);
}
throw new CloudError('VPC 清理任务仍在执行', 409);
}
private static function queueHostJobLocked(string $kind, CloudVpc $vpc,
CloudVpcPort $port, CloudVpcHostState $state,
?array $network, array $jobContext = []): CloudJob
{
$batchId = isset($jobContext['vpc_batch_id'])
? (int) $jobContext['vpc_batch_id'] : null;
$job = self::baseJob(
$kind, $vpc, $port, (int) $state->host_id, $batchId
);
if ($batchId !== null && isset($jobContext['vpc_batch_revision'])) {
$job->request_idempotency_key = substr(
self::batchProjectionRequestPrefix(
$batchId,
(int) $jobContext['vpc_batch_revision']
) . $kind . '-' . (int) $state->host_id . '-'
. (string) $job->operation_id,
0,
128
);
}
$body = $kind === 'vpc_host_apply' ? [
'operation_id' => (string) $job->operation_id,
'desired_digest' => (string) $state->desired_digest,
'network' => $network,
] : [
'operation_id' => (string) $job->operation_id,
'vpc_key' => (string) $vpc->vpc_key,
'expected_revision' => (int) $state->applied_revision,
'expected_digest' => (string) $state->applied_digest,
];
if ($kind === 'vpc_host_remove'
&& ((int) $state->applied_revision < 1
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $state->applied_digest) !== 1)) {
throw new CloudError('VPC 宿主机删除缺少已应用投影证据', 409);
}
self::sealJob($job, $vpc, $port, 'POST',
'/v1/vpc/networks/' . rawurlencode((string) $vpc->vpc_key)
. ($kind === 'vpc_host_apply' ? '/apply' : '/remove'),
$body, ['host_projection' => $network] + $jobContext);
$state->last_operation_id = (string) $job->operation_id;
$state->sync_state = 'pending';
$state->updated_at = time();
$state->save();
return $job;
}
private static function queuePortJobLocked(string $kind,
CloudInstance $instance, CloudVpc $vpc, CloudVpcPort $port,
?CloudVpcHostState $state, ?int $batchId = null): CloudJob
{
if (!self::hostReady($state)) {
throw new CloudError('实例宿主机的 VPC 路由尚未收敛', 409);
}
$job = self::baseJob(
$kind, $vpc, $port, (int) $instance->host_id, $batchId
);
$body = [
'operation_id' => (string) $job->operation_id,
'port_key' => (string) $port->port_key,
'vpc_key' => (string) $vpc->vpc_key,
'private_ip' => (string) $port->private_ip,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
'mac' => (string) $port->mac_address,
'host_revision' => (int) $state->applied_revision,
'host_digest' => (string) $state->applied_digest,
'confirm_restart' => true,
];
self::sealJob($job, $vpc, $port,
$kind === 'vpc_port_attach' ? 'POST' : 'DELETE',
'/v1/vms/' . rawurlencode(strtolower((string) $instance->uuid))
. '/vpc-port', $body, $batchId === null ? [] : [
'vpc_batch_id' => $batchId,
]);
$port->last_operation_id = (string) $job->operation_id;
$port->last_error_code = '';
$port->updated_at = time();
$port->save();
return $job;
}
private static function baseJob(string $kind, CloudVpc $vpc,
CloudVpcPort $port, int $hostId, ?int $batchId = null): CloudJob
{
$attempt = CloudJob::query()->where('kind', $kind)
->where('vpc_port_id', (int) $port->id)
->where('host_id', $hostId)->lockForUpdate()->count() + 1;
$job = new CloudJob();
$job->operation_id = bin2hex(random_bytes(16));
$job->request_idempotency_key = substr('vpc-' . $kind . '-'
. (int) $port->id . '-' . $hostId . '-' . $attempt, 0, 128);
$job->request_fingerprint = str_repeat('0', 64);
$job->instance_id = (int) $port->instance_id;
$job->vpc_id = (int) $vpc->id;
$job->vpc_port_id = (int) $port->id;
if ($batchId !== null) {
$job->vpc_batch_id = $batchId;
}
$job->user_id = (int) $port->user_id;
$job->host_id = $hostId;
$job->kind = $kind;
$job->execution_mode = 'agent';
$job->status = 'queued';
return $job;
}
/**
* Internal batch boundary. The batch service owns orchestration while this
* service remains the single authority for port allocation and Agent wire
* contracts.
*/
public static function reserveBatchPortLocked(
CloudInstance $instance,
CloudVpc $vpc,
string $batchKey,
int $revision
): CloudVpcPort {
self::assertOwnerAndInstance($instance, [
'user_id' => (int) $vpc->user_id,
], true);
self::assertPublicNetworkSnapshot($instance);
if (self::instanceRegionId($instance) !== (int) $vpc->region_id) {
throw new CloudError('实例只能加入同一地区的 VPC', 409);
}
self::assertNoInstanceJobLocked($instance);
if (CloudVpcPort::query()->where('instance_id', (int) $instance->id)
->where('active_claim', 1)->lockForUpdate()->exists()) {
throw new CloudError('实例已连接 VPC 或正在执行连接任务', 409);
}
$now = time();
$port = new CloudVpcPort();
$port->port_key = substr(hash('sha256', "vpc-batch-port\0"
. $batchKey . "\0" . (int) $instance->id), 0, 32);
$port->vpc_id = (int) $vpc->id;
$port->instance_id = (int) $instance->id;
$port->user_id = (int) $vpc->user_id;
$port->private_ip = self::allocateIpLocked($vpc);
$port->mac_address = self::allocateMacLocked((string) $port->port_key);
$port->active_claim = 1;
$port->state = 'preparing';
$port->desired_revision = $revision;
$port->applied_revision = 0;
$port->last_operation_id = null;
$port->last_error_code = '';
$port->attached_at = null;
$port->released_at = null;
$port->created_at = $port->updated_at = $now;
$port->save();
return $port;
}
public static function assertBatchInstanceLocked(
CloudInstance $instance,
array $actor,
bool $requirePublicNetwork = false
): void {
self::assertOwnerAndInstance($instance, $actor, true);
if ($requirePublicNetwork) {
self::assertPublicNetworkSnapshot($instance);
}
self::assertNoInstanceJobLocked($instance);
}
public static function batchInstanceRegionId(CloudInstance $instance): int
{
return self::instanceRegionId($instance);
}
public static function assertBatchVpcSettledLocked(CloudVpc $vpc): void
{
self::assertVpcSettledLocked($vpc);
}
/** @return list<CloudJob> */
public static function queueBatchProjectionLocked(
CloudVpc $vpc,
CloudVpcPort $focusPort,
int $revision,
int $batchId,
?string $provenNotAppliedOperationId = null
): array {
return self::queueProjectionLocked($vpc, $focusPort, $revision, [
'vpc_batch_id' => $batchId,
'vpc_batch_revision' => $revision,
], $provenNotAppliedOperationId);
}
public static function batchProjectionRequestPrefix(
int $batchId,
int $revision
): string {
if ($batchId < 1 || $revision < 1) {
throw new \InvalidArgumentException(
'VPC batch projection identity must be positive'
);
}
return 'vpc-batch-route-' . $batchId . '-' . $revision . '-';
}
private static function isKnownNotAppliedHostState(
CloudVpcHostState $state,
?string $provenNotAppliedOperationId
): bool {
if ((string) $state->sync_state === 'failed'
&& $provenNotAppliedOperationId !== null
&& preg_match('/^[a-f0-9]{32}$/D',
$provenNotAppliedOperationId) === 1
&& hash_equals(
(string) $state->last_operation_id,
$provenNotAppliedOperationId
)) {
return true;
}
return self::knownNotAppliedHostFailureLocked($state) !== null;
}
public static function queueBatchPortJobLocked(
string $kind,
CloudInstance $instance,
CloudVpc $vpc,
CloudVpcPort $port,
CloudVpcHostState $state,
int $batchId
): CloudJob {
return self::queuePortJobLocked(
$kind, $instance, $vpc, $port, $state, $batchId
);
}
/** Update only the host projection ledger; batch advancement stays outside. */
public static function settleBatchHostJobLocked(
CloudJob $job,
string $status,
$result,
array $operation,
CloudVpc $vpc
): string {
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
->where('host_id', (int) $job->host_id)
->lockForUpdate()->first();
if ($state === null
|| !hash_equals((string) $state->last_operation_id,
(string) $job->operation_id)) {
throw new CloudError('VPC 批次宿主机结果与当前投影不一致', 409);
}
$now = time();
if ($status === 'succeeded') {
self::assertHostResult($job, $vpc, $state, $result);
if ((string) $job->kind === 'vpc_host_remove') {
$state->delete();
} else {
$state->applied_revision = (int) $state->desired_revision;
$state->applied_digest = (string) $state->desired_digest;
$state->sync_state = 'ready';
$state->last_error_code = '';
$state->last_success_at = $now;
$state->updated_at = $now;
$state->save();
}
return '';
}
$uncertain = $status === 'uncertain';
$code = $uncertain ? 'vpc_result_uncertain'
: (CloudOperationErrorCatalog::workerSummary(
(string) $job->kind, 'failed', $operation
) ?? 'vpc_host_apply_failed');
$state->sync_state = $uncertain ? 'unknown' : 'failed';
$state->last_error_code = substr($code, 0, 64);
$state->last_failure_at = $now;
$state->updated_at = $now;
$state->save();
return substr($code, 0, 64);
}
public static function assertBatchPortResult(
CloudJob $job,
CloudVpc $vpc,
CloudVpcPort $port,
CloudInstance $instance,
$result
): void {
self::assertPortResult($job, $vpc, $port, $instance, $result);
}
public static function batchHostReady(?CloudVpcHostState $state): bool
{
return self::hostReady($state);
}
/**
* Continue panel orchestration after a super administrator proves that an
* empty, disabled compute node has been permanently retired. The caller
* has already removed the exact host-state row and resolved every unknown
* predecessor generation; no Agent request is made here.
*/
public static function finishRetiredHostRemovalLocked(
CloudJob $job,
CloudVpc $vpc,
CloudVpcPort $port,
CloudInstance $instance
): void {
if ((string) $job->kind !== 'vpc_host_remove'
|| (int) $job->vpc_id !== (int) $vpc->id
|| (int) $job->vpc_port_id !== (int) $port->id
|| (int) $job->instance_id !== (int) $instance->id
|| CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
->where('host_id', (int) $job->host_id)->exists()) {
throw new CloudError('VPC 宿主机退役收尾账本不一致', 409);
}
if ($job->vpc_batch_id !== null) {
CloudVpcMembershipBatchService::finishRetiredHostRemovalLocked(
$job, $vpc
);
return;
}
self::advancePortLocked($instance, $vpc, $port);
}
private static function sealJob(CloudJob $job, CloudVpc $vpc,
CloudVpcPort $port, string $method, string $target, array $body,
array $extra = []): void
{
$payload = [
'method' => $method,
'request_target' => $target,
'body' => $body,
'owner_id' => (int) $port->user_id,
'can_manage' => true,
'system_vpc_orchestration' => true,
'vpc_id' => (int) $vpc->id,
'vpc_port_id' => (int) $port->id,
] + $extra;
$job->request_fingerprint = hash('sha256', json_encode([
'method' => $method, 'target' => $target, 'body' => $body,
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE));
$job->sealed_payload = CloudSecrets::encrypt($payload, 'job-payload');
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
}
private static function hostApplyBody(CloudJob $job, CloudVpc $vpc,
CloudVpcHostState $state, array $payload): array
{
$network = $payload['host_projection'] ?? null;
if (!is_array($network) || array_is_list($network)
|| !hash_equals((string) $state->desired_digest,
CloudVpcProjection::digest($network))) {
throw new CloudError('VPC 宿主机投影摘要不一致', 409);
}
return ['operation_id' => (string) $job->operation_id,
'desired_digest' => (string) $state->desired_digest,
'network' => $network];
}
private static function hostRemoveBody(CloudJob $job, CloudVpc $vpc,
CloudVpcHostState $state): array
{
return ['operation_id' => (string) $job->operation_id,
'vpc_key' => (string) $vpc->vpc_key,
'expected_revision' => (int) $state->applied_revision,
'expected_digest' => (string) $state->applied_digest];
}
private static function portBody(CloudJob $job, CloudVpc $vpc,
CloudVpcPort $port, CloudVpcHostState $state): array
{
return ['operation_id' => (string) $job->operation_id,
'port_key' => (string) $port->port_key,
'vpc_key' => (string) $vpc->vpc_key,
'private_ip' => (string) $port->private_ip,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
'mac' => (string) $port->mac_address,
'host_revision' => (int) $state->applied_revision,
'host_digest' => (string) $state->applied_digest,
'confirm_restart' => true];
}
private static function assertHostResult(CloudJob $job, CloudVpc $vpc,
CloudVpcHostState $state, $result): void
{
if (!is_array($result) || array_is_list($result)
|| ($result['vpc_key'] ?? null) !== (string) $vpc->vpc_key
|| (int) ($result['desired_revision'] ?? 0)
!== (int) (($job->kind === 'vpc_host_apply')
? $state->desired_revision : $state->applied_revision)
|| !hash_equals((string) (($job->kind === 'vpc_host_apply')
? $state->desired_digest : $state->applied_digest),
(string) ($result['desired_digest'] ?? ''))
|| ($job->kind === 'vpc_host_apply'
&& (($result['applied'] ?? null) !== true
|| ($result['removed'] ?? false) === true
|| ($result['bridge'] ?? null)
!== CloudVpcProjection::bridgeName(
(string) $vpc->vpc_key)))
|| ($job->kind === 'vpc_host_remove'
&& (($result['removed'] ?? null) !== true
|| ($result['applied'] ?? false) === true))) {
throw new CloudError('Agent 成功结果缺少准确的 VPC 路由证据', 502);
}
}
private static function assertPortResult(CloudJob $job, CloudVpc $vpc,
CloudVpcPort $port, CloudInstance $instance, $result): void
{
if (!is_array($result) || array_is_list($result)
|| strtolower((string) ($result['uuid'] ?? ''))
!== strtolower((string) $instance->uuid)
|| ($result['port_key'] ?? null) !== (string) $port->port_key
|| ($result['vpc_key'] ?? null) !== (string) $vpc->vpc_key) {
throw new CloudError('Agent 成功结果缺少准确的 VPC 网卡身份', 502);
}
if ((string) $job->kind === 'vpc_port_detach') {
if (($result['removed'] ?? null) !== true
|| ($result['attached'] ?? false) === true
|| array_key_exists('vpc_port', $result)) {
throw new CloudError('Agent 未确认 VPC 网卡已卸载', 502);
}
return;
}
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
->where('host_id', (int) $instance->host_id)->first();
$metadata = $result['vpc_port'] ?? null;
if (!self::hostReady($state) || ($result['attached'] ?? null) !== true
|| ($result['removed'] ?? false) === true
|| !is_array($metadata) || array_is_list($metadata)
|| !self::sameDocument($metadata, [
'port_key' => (string) $port->port_key,
'vpc_key' => (string) $vpc->vpc_key,
'private_ip' => (string) $port->private_ip,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
'mac' => (string) $port->mac_address,
'bridge' => CloudVpcProjection::bridgeName(
(string) $vpc->vpc_key),
'host_revision' => (int) $state->applied_revision,
'host_digest' => (string) $state->applied_digest,
])) {
throw new CloudError('Agent 未确认准确的 VPC 网卡配置', 502);
}
}
private static function releasePortLocked(CloudVpc $vpc,
CloudVpcPort $port): void
{
$now = time();
$port->state = 'released';
$port->active_claim = null;
$port->last_operation_id = null;
$port->last_error_code = '';
$port->released_at = $now;
$port->updated_at = $now;
$port->save();
$vpc->state = 'active';
$vpc->updated_at = $now;
$vpc->save();
}
private static function assertVpcSettledLocked(CloudVpc $vpc): void
{
if ((string) $vpc->state !== 'active') {
throw new CloudError('该 VPC 尚未收敛到可变更状态', 409);
}
$ports = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
->where('active_claim', 1)->orderBy('id')->lockForUpdate()->get();
if ($ports->contains(static fn ($port): bool =>
(string) $port->state !== 'attached'
|| $port->migration_host_id !== null)) {
throw new CloudError('该 VPC 已有端口变更任务,请等待完成', 409);
}
$states = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
->lockForUpdate()->get();
if ($states->contains(static fn ($state): bool =>
!self::hostReady($state))) {
throw new CloudError('该 VPC 的宿主机路由尚未收敛', 409);
}
}
private static function assertNoInstanceJobLocked(
CloudInstance $instance): void
{
$jobs = CloudJobService::blockingQueryForInstance((int) $instance->id)
->orderBy('id')->lockForUpdate()->get();
if (!$jobs->isEmpty()) {
throw new CloudError('虚拟机已有任务,请等待完成', 409);
}
}
private static function activePortJob(int $instanceId,
int $portId): ?CloudJob
{
return CloudJobService::blockingQueryForInstance($instanceId)
->where('vpc_port_id', $portId)
->whereIn('kind', self::JOB_KINDS)
->orderByDesc('id')->first();
}
private static function activePortJobLocked(CloudInstance $instance,
CloudVpcPort $port): ?CloudJob
{
return CloudJobService::blockingQueryForInstance((int) $instance->id)
->where('vpc_port_id', (int) $port->id)
->whereIn('kind', self::JOB_KINDS)
->orderByDesc('id')->lockForUpdate()->first();
}
private static function assertOwnerAndInstance(?CloudInstance $instance,
array $actor, bool $mutation): void
{
if ($instance === null || (int) $instance->user_id
!== (int) ($actor['user_id'] ?? 0)
|| (string) $instance->status === 'deleted') {
throw new CloudError('虚拟机不存在或无权访问', 404);
}
if ($mutation && ((string) $instance->status !== 'active'
|| !$instance->serviceAvailable())) {
throw new CloudError('实例服务已到期或正在执行策略,不能修改 VPC', 409);
}
}
/**
* A new private NIC may only be attached to an instance whose immutable
* dual-stack public identity still passes the canonical snapshot parser.
* Detach deliberately does not call this guard: damaged metadata must
* never trap an already-attached VPC port or its private address claim.
*/
private static function assertPublicNetworkSnapshot(
CloudInstance $instance
): void {
$decoded = json_decode((string) $instance->spec_json, true);
$spec = CloudInstanceMetadata::publicSpec(
is_array($decoded) && !array_is_list($decoded) ? $decoded : []
);
if (($spec['os_source'] ?? null) !== 'cloud_template') {
throw new CloudError('当前实例不支持 VPC 私网网卡', 409);
}
if (($spec['public_network_valid'] ?? false) !== true) {
throw new CloudError(
'实例公网配置异常,暂不能修改 VPC 私网',
409
);
}
}
private static function instanceRegionId(CloudInstance $instance): int
{
if ($instance->resource_pool_id === null) {
throw new CloudError('实例缺少地区资源池账本', 409);
}
$regionId = CloudResourcePool::query()
->join('cloud_network_lines as line', 'line.id', '=',
'cloud_resource_pools.line_id')
->where('cloud_resource_pools.id', (int) $instance->resource_pool_id)
->value('line.region_id');
if ($regionId === null || (int) $regionId < 1) {
throw new CloudError('实例地区账本无效', 409);
}
return (int) $regionId;
}
/** @return array<int,array> */
private static function readyTransports(array $hostIds): array
{
if ($hostIds === []) {
return [];
}
sort($hostIds, SORT_NUMERIC);
$enabled = CloudHost::query()->whereIn('id', $hostIds)
->where('enabled', 1)->pluck('id')->map(
static fn ($id): int => (int) $id
)->all();
sort($enabled, SORT_NUMERIC);
if ($enabled !== $hostIds) {
throw new CloudError('VPC 包含不可用的计算节点', 409);
}
$rows = CloudHostVpcTransport::query()->whereIn('host_id', $hostIds)
->where('sync_state', 'ready')
->whereColumn('observed_revision', 'desired_revision')
->orderBy('host_id')->get()->keyBy('host_id');
if ($rows->count() !== count($hostIds)) {
throw new CloudError('VPC 计算节点私网传输尚未验证就绪', 409);
}
$result = [];
foreach ($hostIds as $hostId) {
$row = $rows->get($hostId);
$result[$hostId] = [
'transport_address' => (string) $row->transport_address,
'transport_prefix_length' => (int) $row->transport_prefix_length,
'transport_interface' => (string) $row->transport_interface,
'mtu' => (int) $row->mtu,
];
}
return $result;
}
private static function allocateIpLocked(CloudVpc $vpc): string
{
[$network, $prefixText] = explode('/', (string) $vpc->cidr, 2);
$prefix = (int) $prefixText;
$base = self::ipv4Number($network);
$last = $base + (1 << (32 - $prefix)) - 1;
$used = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
->where('active_claim', 1)->orderBy('private_ip')
->lockForUpdate()->pluck('private_ip')->mapWithKeys(
static fn ($ip): array => [(string) $ip => true]
)->all();
for ($number = $base + 2; $number < $last; $number++) {
$candidate = long2ip($number);
if (!isset($used[$candidate])) {
return $candidate;
}
}
throw new CloudError('该 VPC 的私网地址已用尽', 409);
}
private static function allocateMacLocked(string $portKey): string
{
for ($attempt = 0; $attempt < 256; $attempt++) {
$digest = hash('sha256', 'virt-cloud-vpc-port-mac:' . $portKey
. ':' . $attempt, true);
$bytes = [2, ord($digest[0]), ord($digest[1]), ord($digest[2]),
ord($digest[3]), ord($digest[4])];
$mac = implode(':', array_map(static fn (int $byte): string =>
str_pad(dechex($byte), 2, '0', STR_PAD_LEFT), $bytes));
if (!CloudVpcPort::query()->where('mac_address', $mac)
->lockForUpdate()->exists()) {
return $mac;
}
}
throw new CloudError('无法分配唯一的 VPC 网卡地址', 503);
}
private static function requestKey(array $actor): string
{
$key = $actor['request_idempotency_key'] ?? null;
if (!is_string($key)
|| preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) {
throw new CloudError('请求幂等标识无效', 400);
}
return $key;
}
private static function portKey(int $userId, int $instanceId,
string $vpcKey, string $requestKey): string
{
return substr(hash('sha256', "vpc-port\0" . $userId . "\0"
. $instanceId . "\0" . $vpcKey . "\0" . $requestKey), 0, 32);
}
private static function hostReady(?CloudVpcHostState $state): bool
{
return $state !== null && (string) $state->sync_state === 'ready'
&& (int) $state->desired_revision > 0
&& (int) $state->desired_revision === (int) $state->applied_revision
&& is_string($state->desired_digest)
&& is_string($state->applied_digest)
&& hash_equals((string) $state->desired_digest,
(string) $state->applied_digest);
}
private static function portDto(CloudVpcPort $port,
?CloudVpc $vpc = null): array
{
$vpc ??= CloudVpc::find((int) $port->vpc_id);
return [
'port_key' => (string) $port->port_key,
'vpc_key' => $vpc === null ? null : (string) $vpc->vpc_key,
'vpc_name' => $vpc === null ? null : (string) $vpc->name,
'private_ip' => (string) $port->private_ip,
'mac_address' => (string) $port->mac_address,
'state' => (string) $port->state,
'desired_revision' => (int) $port->desired_revision,
'applied_revision' => (int) $port->applied_revision,
'last_error_code' => CloudOperationErrorCatalog::publicCode(
(string) $port->last_error_code
),
'attached_at' => $port->attached_at === null
? null : (int) $port->attached_at,
];
}
private static function sameDocument(array $left, array $right): bool
{
return hash_equals(self::documentDigest($left),
self::documentDigest($right));
}
private static function documentDigest(array $value): string
{
$canonical = static function ($item) use (&$canonical) {
if (!is_array($item)) {
return $item;
}
if (!array_is_list($item)) {
ksort($item, SORT_STRING);
}
foreach ($item as $key => $child) {
$item[$key] = $canonical($child);
}
return $item;
};
return hash('sha256', json_encode($canonical($value),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE));
}
private static function ipv4Number(string $address): int
{
$packed = inet_pton($address);
if ($packed === false || strlen($packed) !== 4) {
throw new CloudError('VPC IPv4 地址账本无效', 503);
}
return (int) unpack('Naddress', $packed)['address'];
}
}