You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2655 lines
112 KiB
PHP
2655 lines
112 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudHost;
|
|
use App\Models\CloudHostVpcTransport;
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceMigration;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudResourcePool;
|
|
use App\Models\CloudVpc;
|
|
use App\Models\CloudVpcHostState;
|
|
use App\Models\CloudVpcMembershipBatch;
|
|
use App\Models\CloudVpcMembershipBatchItem;
|
|
use App\Models\CloudVpcPort;
|
|
|
|
/**
|
|
* Durable VPC attachment orchestration.
|
|
*
|
|
* Attach order is host projections -> cold VM port attach. Detach order is
|
|
* cold VM port detach -> host projection cleanup -> IP release. Every Agent
|
|
* mutation has one operation ID and a row-level fence; an uncertain effect is
|
|
* never replayed.
|
|
*/
|
|
final class CloudVpcPortService
|
|
{
|
|
private const DELETED_CLEANUP_RETRY_DELAY = 300;
|
|
|
|
private const JOB_KINDS = [
|
|
'vpc_host_apply', 'vpc_host_remove',
|
|
'vpc_port_attach', 'vpc_port_detach',
|
|
];
|
|
|
|
/**
|
|
* Freeze the optional logical port before the source VM is stopped. This
|
|
* is a read-only projection preflight: the target transport must already
|
|
* be able to carry the exact port, but routing is not changed yet.
|
|
*/
|
|
public static function migrationPortIdLocked(
|
|
CloudInstance $instance,
|
|
int $targetHostId
|
|
): ?int {
|
|
$reference = CloudVpcPort::query()
|
|
->where('instance_id', (int) $instance->id)
|
|
->where('active_claim', 1)->first();
|
|
if ($reference === null) {
|
|
return null;
|
|
}
|
|
$vpc = CloudVpc::query()->where('id', (int) $reference->vpc_id)
|
|
->lockForUpdate()->first();
|
|
$port = CloudVpcPort::query()->where('id', (int) $reference->id)
|
|
->where('active_claim', 1)->lockForUpdate()->first();
|
|
if ($vpc === null || $port === null
|
|
|| (int) $port->instance_id !== (int) $instance->id
|
|
|| (int) $port->user_id !== (int) $instance->user_id
|
|
|| (int) $vpc->user_id !== (int) $instance->user_id
|
|
|| (string) $port->state !== 'attached'
|
|
|| $port->migration_host_id !== null
|
|
|| self::activePortJobLocked($instance, $port) !== null) {
|
|
throw new CloudError('实例 VPC 端口尚未收敛,不能维护迁移', 409);
|
|
}
|
|
self::assertVpcSettledLocked($vpc);
|
|
[$rows, $hostIds] = self::projectionRowsLocked(
|
|
$vpc, [(int) $port->id => $targetHostId]
|
|
);
|
|
$projections = CloudVpcProjection::build([
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
], $rows, self::readyTransports($hostIds),
|
|
(int) $vpc->desired_revision + 1);
|
|
if (!isset($projections[$targetHostId])) {
|
|
throw new CloudError('目标宿主机不能承载该 VPC 端口', 409);
|
|
}
|
|
return (int) $port->id;
|
|
}
|
|
|
|
/** @return list<CloudJob> */
|
|
public static function beginMigrationToTargetLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance
|
|
): array {
|
|
return self::beginMigrationProjectionLocked(
|
|
$migration, $instance, 'target'
|
|
);
|
|
}
|
|
|
|
/** @return list<CloudJob> */
|
|
public static function beginMigrationToSourceLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance
|
|
): array {
|
|
return self::beginMigrationProjectionLocked(
|
|
$migration, $instance, 'source'
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Mark one migration projection complete only after every affected host
|
|
* has committed the same generation. Source completion also restores the
|
|
* ordinary attached-port state before the source VM may be restarted.
|
|
*/
|
|
public static function finishMigrationProjectionLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
string $direction
|
|
): bool {
|
|
if ($migration->vpc_port_id === null) {
|
|
return true;
|
|
}
|
|
$targetHostId = $direction === 'target'
|
|
? (int) $migration->target_host_id
|
|
: (int) $migration->source_host_id;
|
|
if (!in_array($direction, ['target', 'source'], true)
|
|
|| (int) $migration->instance_id !== (int) $instance->id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 路由方向或实例账本无效', 409
|
|
);
|
|
}
|
|
[$port, $vpc] = self::migrationPortAndVpcLocked(
|
|
(int) $migration->vpc_port_id
|
|
);
|
|
if ((int) $port->instance_id !== (int) $instance->id
|
|
|| (string) $port->state !== 'migrating'
|
|
|| (int) ($port->migration_host_id ?? 0) !== $targetHostId
|
|
|| (int) $port->user_id !== (int) $instance->user_id
|
|
|| (int) $vpc->user_id !== (int) $instance->user_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 端口位置账本不一致', 409
|
|
);
|
|
}
|
|
$states = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
|
|
->lockForUpdate()->get();
|
|
if ($states->isEmpty() || $states->contains(
|
|
static fn ($state): bool => !self::hostReady($state)
|
|
)) {
|
|
return false;
|
|
}
|
|
$targetState = $states->firstWhere('host_id', $targetHostId);
|
|
if ($targetState === null
|
|
|| (int) $targetState->applied_revision
|
|
!== (int) $vpc->desired_revision) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移目标宿主机缺少已收敛的 VPC 投影', 409
|
|
);
|
|
}
|
|
$port->desired_revision = (int) $vpc->desired_revision;
|
|
$port->applied_revision = (int) $vpc->desired_revision;
|
|
if ($direction === 'source') {
|
|
$port->state = 'attached';
|
|
$port->migration_host_id = null;
|
|
}
|
|
$port->last_error_code = '';
|
|
$port->updated_at = time();
|
|
$port->save();
|
|
$vpc->state = 'active';
|
|
$vpc->updated_at = time();
|
|
$vpc->save();
|
|
return true;
|
|
}
|
|
|
|
/** Exact target-side VPC identity injected into PeerMigration receive. */
|
|
public static function migrationTargetMetadataLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance
|
|
): ?array {
|
|
if ($migration->vpc_port_id === null) {
|
|
return null;
|
|
}
|
|
[$port, $vpc] = self::migrationPortAndVpcLocked(
|
|
(int) $migration->vpc_port_id
|
|
);
|
|
$state = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)
|
|
->where('host_id', (int) $migration->target_host_id)
|
|
->lockForUpdate()->first();
|
|
if ((int) $port->instance_id !== (int) $instance->id
|
|
|| !self::hostReady($state)
|
|
|| (string) $port->state !== 'migrating'
|
|
|| (int) ($port->migration_host_id ?? 0)
|
|
!== (int) $migration->target_host_id
|
|
|| (int) $port->applied_revision
|
|
!== (int) $vpc->desired_revision
|
|
|| (int) $port->user_id !== (int) $instance->user_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移目标 VPC 端口尚未收敛', 409
|
|
);
|
|
}
|
|
return [
|
|
'port_key' => (string) $port->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'private_ip' => (string) $port->private_ip,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
'mac' => (string) $port->mac_address,
|
|
'bridge' => CloudVpcProjection::bridgeName(
|
|
(string) $vpc->vpc_key
|
|
),
|
|
'host_revision' => (int) $state->applied_revision,
|
|
'host_digest' => (string) $state->applied_digest,
|
|
];
|
|
}
|
|
|
|
/** Atomically hand the logical port back to normal placement at cutover. */
|
|
public static function finalizeMigrationCutoverLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance
|
|
): void {
|
|
if ($migration->vpc_port_id === null) {
|
|
return;
|
|
}
|
|
self::migrationTargetMetadataLocked($migration, $instance);
|
|
$port = CloudVpcPort::query()
|
|
->where('id', (int) $migration->vpc_port_id)
|
|
->where('active_claim', 1)->lockForUpdate()->firstOrFail();
|
|
$port->state = 'attached';
|
|
$port->migration_host_id = null;
|
|
$port->last_error_code = '';
|
|
$port->updated_at = time();
|
|
$port->save();
|
|
}
|
|
|
|
public static function listing(CloudInstance $instance,
|
|
array $actor): array
|
|
{
|
|
self::assertOwnerAndInstance($instance, $actor, false);
|
|
$decoded = json_decode((string) $instance->spec_json, true);
|
|
$publicSpec = CloudInstanceMetadata::publicSpec(
|
|
is_array($decoded) && !array_is_list($decoded) ? $decoded : []
|
|
);
|
|
$networkValid =
|
|
($publicSpec['public_network_valid'] ?? false) === true;
|
|
$supported = $networkValid
|
|
&& ($publicSpec['os_source'] ?? null) === 'cloud_template';
|
|
$regionId = self::instanceRegionId($instance);
|
|
$port = CloudVpcPort::query()
|
|
->where('instance_id', (int) $instance->id)
|
|
->where('active_claim', 1)->first();
|
|
$vpcs = CloudVpc::query()
|
|
->where('user_id', (int) $instance->user_id)
|
|
->where('region_id', $regionId)->where('active_claim', 1)
|
|
->whereIn('state', ['active', 'degraded'])
|
|
->orderBy('name')->orderBy('id')->get();
|
|
$blocking = CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->orderBy('id')->first();
|
|
$activeVpcJob = $port === null ? null
|
|
: self::activePortJob((int) $instance->id, (int) $port->id);
|
|
return [
|
|
'port' => $port === null ? null : self::portDto($port),
|
|
'vpcs' => $vpcs->map(static fn (CloudVpc $vpc): array => [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'name' => (string) $vpc->name,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
'state' => (string) $vpc->state,
|
|
'revision' => (int) $vpc->desired_revision,
|
|
])->all(),
|
|
'public_network_valid' => $networkValid,
|
|
'can_attach' => $port === null && $blocking === null
|
|
&& $supported
|
|
&& (string) $instance->status === 'active'
|
|
&& $instance->serviceAvailable(),
|
|
'can_detach' => $port !== null && $blocking === null
|
|
&& (string) $port->state === 'attached'
|
|
&& (string) $instance->status === 'active'
|
|
&& $instance->serviceAvailable(),
|
|
'can_retry_attach' => $port !== null && $blocking === null
|
|
&& $supported
|
|
&& (string) $port->state === 'preparing'
|
|
&& (string) $instance->status === 'active'
|
|
&& $instance->serviceAvailable(),
|
|
'can_retry_detach' => $port !== null && $blocking === null
|
|
&& (string) $port->state === 'release_pending'
|
|
&& (string) $instance->status === 'active'
|
|
&& $instance->serviceAvailable(),
|
|
'active_task' => $activeVpcJob === null ? null : [
|
|
'task_id' => (string) $activeVpcJob->id,
|
|
'status_url' => '/api/v1/tasks/' . (int) $activeVpcJob->id,
|
|
],
|
|
'requires_cold_restart' => true,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* One database-backed read model for the user VPC management page.
|
|
*
|
|
* The browser must not fan out one request per instance. This projection
|
|
* keeps the same ownership, service-state and resource-fence rules used by
|
|
* listing(), while returning every eligible instance in one response.
|
|
*/
|
|
public static function managementListing(array $actor): array
|
|
{
|
|
$userId = (int) ($actor['user_id'] ?? 0);
|
|
if ($userId < 1) {
|
|
throw new CloudError('请先登录', 401);
|
|
}
|
|
$instances = CloudInstance::query()->where('user_id', $userId)
|
|
->where('status', '<>', 'deleted')->orderByDesc('id')->get([
|
|
'id', 'uuid', 'host_id', 'user_id', 'resource_pool_id',
|
|
'name', 'status', 'service_status', 'network_status',
|
|
'expires_at', 'spec_json',
|
|
]);
|
|
if ($instances->isEmpty()) {
|
|
return [];
|
|
}
|
|
|
|
$poolIds = $instances->pluck('resource_pool_id')->filter(
|
|
static fn ($id): bool => (int) $id > 0
|
|
)->map(static fn ($id): int => (int) $id)->unique()->values()->all();
|
|
$regionByPool = $poolIds === [] ? collect() :
|
|
CloudResourcePool::query()
|
|
->join('cloud_network_lines as vpc_line',
|
|
'vpc_line.id', '=', 'cloud_resource_pools.line_id')
|
|
->whereIn('cloud_resource_pools.id', $poolIds)
|
|
->get([
|
|
'cloud_resource_pools.id as pool_id',
|
|
'vpc_line.region_id as region_id',
|
|
])->mapWithKeys(static fn ($row): array => [
|
|
(int) $row->pool_id => (int) $row->region_id,
|
|
]);
|
|
$instanceIds = $instances->pluck('id')->map(
|
|
static fn ($id): int => (int) $id
|
|
)->all();
|
|
$ports = CloudVpcPort::query()->whereIn('instance_id', $instanceIds)
|
|
->where('active_claim', 1)->get()->keyBy('instance_id');
|
|
$vpcIds = $ports->pluck('vpc_id')->map(
|
|
static fn ($id): int => (int) $id
|
|
)->unique()->values()->all();
|
|
$vpcs = $vpcIds === [] ? collect() : CloudVpc::query()
|
|
->whereIn('id', $vpcIds)->get()->keyBy('id');
|
|
$jobs = CloudJobService::blockingQueryForInstances($instanceIds)
|
|
->orderByDesc('id')->get([
|
|
'id', 'instance_id', 'vpc_port_id', 'kind',
|
|
])->groupBy('instance_id');
|
|
|
|
$result = [];
|
|
foreach ($instances as $instance) {
|
|
$port = $ports->get((int) $instance->id);
|
|
$decoded = json_decode((string) $instance->spec_json, true);
|
|
$spec = CloudInstanceMetadata::publicSpec(
|
|
is_array($decoded) && !array_is_list($decoded) ? $decoded : []
|
|
);
|
|
$networkValid = ($spec['public_network_valid'] ?? false) === true;
|
|
$supported = $networkValid
|
|
&& ($spec['os_source'] ?? null) === 'cloud_template';
|
|
$publicIpv4 = $networkValid
|
|
? (string) ($spec['public_ipv4'] ?? '') : '';
|
|
$publicIpv6 = $networkValid
|
|
? (string) ($spec['public_ipv6'] ?? '') : '';
|
|
// Keep a historical attached port visible for status and
|
|
// diagnosis even if its immutable source metadata is damaged.
|
|
if (!$supported && $port === null) {
|
|
continue;
|
|
}
|
|
$regionId = (int) ($regionByPool->get(
|
|
(int) ($instance->resource_pool_id ?? 0)
|
|
) ?? 0);
|
|
if ($regionId < 1 && $port === null) {
|
|
continue;
|
|
}
|
|
$instanceJobs = $jobs->get((int) $instance->id, collect());
|
|
$blocking = $instanceJobs->first();
|
|
$activeVpcJob = $port === null ? null : $instanceJobs->first(
|
|
static fn ($job): bool =>
|
|
(int) ($job->vpc_port_id ?? 0) === (int) $port->id
|
|
&& in_array((string) $job->kind,
|
|
self::JOB_KINDS, true)
|
|
);
|
|
$serviceAvailable = (string) $instance->status === 'active'
|
|
&& $instance->serviceAvailable();
|
|
$portState = $port === null ? '' : (string) $port->state;
|
|
$reason = '';
|
|
if (!$networkValid) {
|
|
$reason = $port === null
|
|
? '实例公网配置异常,暂不能加入 VPC 私网'
|
|
: '实例公网配置异常,仅允许移出 VPC 私网';
|
|
} elseif (!$supported) {
|
|
$reason = '该实例的系统来源不支持 VPC 私网网卡';
|
|
} elseif (!$serviceAvailable) {
|
|
$reason = '实例服务已到期或当前不可修改';
|
|
} elseif ($blocking !== null) {
|
|
$reason = '实例正在执行其他任务';
|
|
} elseif ($port === null && $regionId < 1) {
|
|
$reason = '实例缺少有效的地区资源池';
|
|
}
|
|
$vpc = $port === null ? null : $vpcs->get((int) $port->vpc_id);
|
|
$result[] = [
|
|
'id' => (string) $instance->id,
|
|
'name' => (string) $instance->name,
|
|
'region_id' => $regionId,
|
|
'public_ipv4' => $publicIpv4,
|
|
'public_ipv6' => $publicIpv6,
|
|
'public_network_valid' => $networkValid,
|
|
'instance_status' => (string) $instance->status,
|
|
'service_status' => (string) $instance->service_status,
|
|
'network_status' => (string) $instance->network_status,
|
|
'port' => $port === null ? null : self::portDto($port, $vpc),
|
|
'can_attach' => $port === null && $blocking === null
|
|
&& $supported && $regionId > 0 && $serviceAvailable,
|
|
'can_detach' => $port !== null && $blocking === null
|
|
&& $portState === 'attached'
|
|
&& $serviceAvailable,
|
|
'can_retry_attach' => $port !== null && $blocking === null
|
|
&& $supported && $portState === 'preparing'
|
|
&& $serviceAvailable,
|
|
'can_retry_detach' => $port !== null && $blocking === null
|
|
&& $portState === 'release_pending'
|
|
&& $serviceAvailable,
|
|
'operation_blocked' => $blocking !== null,
|
|
'action_unavailable_reason' => $reason,
|
|
'active_task' => $activeVpcJob === null ? null : [
|
|
'task_id' => (string) $activeVpcJob->id,
|
|
'status_url' => '/api/v1/tasks/' . (int) $activeVpcJob->id,
|
|
],
|
|
];
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
/** Reserve one IP and enqueue complete host projections. */
|
|
public static function attach(CloudInstance $instance, array $actor,
|
|
array $input): array
|
|
{
|
|
CloudInput::only($input, ['vpc_key', 'confirm_restart']);
|
|
$vpcKey = $input['vpc_key'] ?? null;
|
|
if (!is_string($vpcKey)
|
|
|| preg_match('/^[a-f0-9]{32}$/D', $vpcKey) !== 1) {
|
|
throw new CloudError('请选择有效的 VPC 私网');
|
|
}
|
|
if (($input['confirm_restart'] ?? null) !== true) {
|
|
throw new CloudError('请确认实例会自动关机并冷启动', 409);
|
|
}
|
|
$requestKey = self::requestKey($actor);
|
|
$connection = (new CloudVpcPort())->getConnection();
|
|
return $connection->transaction(static function () use (
|
|
$instance, $actor, $vpcKey, $requestKey
|
|
): array {
|
|
$locked = CloudVpcLockOrder::instance((int) $instance->id);
|
|
self::assertOwnerAndInstance($locked, $actor, true);
|
|
self::assertPublicNetworkSnapshot($locked);
|
|
$vpc = CloudVpcLockOrder::vpcByKey($vpcKey);
|
|
if ($vpc === null || (int) $vpc->user_id !== (int) $locked->user_id
|
|
|| $vpc->active_claim === null) {
|
|
throw new CloudError('VPC 不存在或当前不可用', 404);
|
|
}
|
|
if ((int) $vpc->region_id !== self::instanceRegionId($locked)) {
|
|
throw new CloudError('实例只能加入同一地区的 VPC', 409);
|
|
}
|
|
|
|
$port = CloudVpcPort::query()
|
|
->where('instance_id', (int) $locked->id)
|
|
->where('active_claim', 1)->lockForUpdate()->first();
|
|
if ($port !== null) {
|
|
if ((int) $port->vpc_id !== (int) $vpc->id
|
|
|| (int) $port->user_id !== (int) $locked->user_id) {
|
|
throw new CloudError('实例已连接其他 VPC', 409);
|
|
}
|
|
if ((string) $port->state === 'attached') {
|
|
return ['port' => self::portDto($port), 'job' => null,
|
|
'unchanged' => true];
|
|
}
|
|
if ((string) $port->state !== 'preparing') {
|
|
throw new CloudError('VPC 端口正在执行其他操作', 409);
|
|
}
|
|
$active = self::activePortJobLocked($locked, $port);
|
|
if ($active !== null) {
|
|
return ['port' => self::portDto($port), 'job' => $active,
|
|
'unchanged' => false];
|
|
}
|
|
self::assertNoInstanceJobLocked($locked);
|
|
$job = self::retryPreparingLocked($locked, $vpc, $port);
|
|
return ['port' => self::portDto($port->fresh()),
|
|
'job' => $job, 'unchanged' => false];
|
|
}
|
|
|
|
if (!in_array((string) $vpc->state,
|
|
['active', 'degraded'], true)) {
|
|
throw new CloudError('VPC 当前正在执行其他操作', 409);
|
|
}
|
|
self::assertNoInstanceJobLocked($locked);
|
|
self::assertVpcSettledLocked($vpc);
|
|
$now = time();
|
|
$port = new CloudVpcPort();
|
|
$port->port_key = self::portKey(
|
|
(int) $locked->user_id, (int) $locked->id,
|
|
$vpcKey, $requestKey
|
|
);
|
|
$existing = CloudVpcPort::query()
|
|
->where('port_key', (string) $port->port_key)
|
|
->lockForUpdate()->first();
|
|
if ($existing !== null) {
|
|
throw new CloudError('该请求已结束,请使用新的请求标识重试', 409);
|
|
}
|
|
$port->vpc_id = (int) $vpc->id;
|
|
$port->instance_id = (int) $locked->id;
|
|
$port->user_id = (int) $locked->user_id;
|
|
$port->private_ip = self::allocateIpLocked($vpc);
|
|
$port->mac_address = self::allocateMacLocked(
|
|
(string) $port->port_key
|
|
);
|
|
$port->active_claim = 1;
|
|
$port->state = 'preparing';
|
|
$revision = (int) $vpc->desired_revision + 1;
|
|
$port->desired_revision = $revision;
|
|
$port->applied_revision = 0;
|
|
$port->last_operation_id = null;
|
|
$port->last_error_code = '';
|
|
$port->attached_at = null;
|
|
$port->released_at = null;
|
|
$port->created_at = $port->updated_at = $now;
|
|
$port->save();
|
|
|
|
$vpc->desired_revision = $revision;
|
|
$vpc->state = 'provisioning';
|
|
$vpc->updated_at = $now;
|
|
$vpc->save();
|
|
$jobs = self::queueProjectionLocked($vpc, $port, $revision);
|
|
if ($jobs === []) {
|
|
throw new CloudError('VPC 路由投影没有可执行宿主机', 503);
|
|
}
|
|
return ['port' => self::portDto($port), 'job' => $jobs[0],
|
|
'unchanged' => false];
|
|
}, 3);
|
|
}
|
|
|
|
/** Detach the VM port before changing any host route. */
|
|
public static function detach(CloudInstance $instance, array $actor,
|
|
array $input): array
|
|
{
|
|
CloudInput::only($input, ['confirm_restart']);
|
|
if (($input['confirm_restart'] ?? null) !== true) {
|
|
throw new CloudError('请确认实例会自动关机并冷启动', 409);
|
|
}
|
|
self::requestKey($actor);
|
|
$connection = (new CloudVpcPort())->getConnection();
|
|
return $connection->transaction(static function () use (
|
|
$instance, $actor
|
|
): array {
|
|
$locked = CloudVpcLockOrder::instance((int) $instance->id);
|
|
self::assertOwnerAndInstance($locked, $actor, true);
|
|
$reference = CloudVpcPort::query()
|
|
->where('instance_id', (int) $locked->id)
|
|
->where('active_claim', 1)->first();
|
|
if ($reference === null) {
|
|
return ['port' => null, 'job' => null, 'unchanged' => true];
|
|
}
|
|
$vpc = CloudVpcLockOrder::vpcById((int) $reference->vpc_id);
|
|
$port = CloudVpcLockOrder::portById((int) $reference->id);
|
|
if ($vpc === null || $port === null
|
|
|| (int) $vpc->user_id !== (int) $locked->user_id
|
|
|| (int) $port->instance_id !== (int) $locked->id
|
|
|| (int) $port->vpc_id !== (int) $vpc->id) {
|
|
throw new CloudError('VPC 端口账本与所属人不一致', 409);
|
|
}
|
|
if ((string) $port->state === 'detaching') {
|
|
$active = self::activePortJobLocked($locked, $port);
|
|
if ($active === null) {
|
|
throw new CloudError('VPC 断开任务账本不完整', 409);
|
|
}
|
|
return ['port' => self::portDto($port), 'job' => $active,
|
|
'unchanged' => false];
|
|
}
|
|
if ((string) $port->state === 'release_pending') {
|
|
$active = self::activePortJobLocked($locked, $port);
|
|
if ($active !== null) {
|
|
return ['port' => self::portDto($port), 'job' => $active,
|
|
'unchanged' => false];
|
|
}
|
|
self::assertNoInstanceJobLocked($locked);
|
|
$job = self::retryReleaseLocked($locked, $vpc, $port);
|
|
return ['port' => $port->active_claim === null
|
|
? null : self::portDto($port->fresh()),
|
|
'job' => $job, 'unchanged' => false];
|
|
}
|
|
if ((string) $port->state !== 'attached') {
|
|
throw new CloudError('VPC 端口正在执行其他操作', 409);
|
|
}
|
|
self::assertNoInstanceJobLocked($locked);
|
|
self::assertVpcSettledLocked($vpc);
|
|
$hostState = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)
|
|
->where('host_id', (int) $locked->host_id)
|
|
->lockForUpdate()->first();
|
|
if (!self::hostReady($hostState)) {
|
|
throw new CloudError('实例所在宿主机的 VPC 路由尚未收敛', 409);
|
|
}
|
|
$port->state = 'detaching';
|
|
$port->last_error_code = '';
|
|
$port->updated_at = time();
|
|
$port->save();
|
|
$job = self::queuePortJobLocked(
|
|
'vpc_port_detach', $locked, $vpc, $port, $hostState
|
|
);
|
|
return ['port' => self::portDto($port), 'job' => $job,
|
|
'unchanged' => false];
|
|
}, 3);
|
|
}
|
|
|
|
/**
|
|
* Start route/IP cleanup only after the Agent has proved the VM and every
|
|
* managed artifact are absent. The caller already holds the instance row
|
|
* lock inside the delete finalizer transaction. No private address is
|
|
* released until every resulting host projection has converged.
|
|
*
|
|
* @return list<CloudJob>
|
|
*/
|
|
public static function releaseAfterInstanceDeletedLocked(
|
|
CloudInstance $instance
|
|
): array {
|
|
$reference = CloudVpcPort::query()
|
|
->where('instance_id', (int) $instance->id)
|
|
->where('active_claim', 1)->first();
|
|
if ($reference === null) {
|
|
return [];
|
|
}
|
|
$vpc = CloudVpc::query()->where('id', (int) $reference->vpc_id)
|
|
->lockForUpdate()->first();
|
|
$port = CloudVpcPort::query()->where('id', (int) $reference->id)
|
|
->where('active_claim', 1)->lockForUpdate()->first();
|
|
if ($vpc === null || $port === null
|
|
|| (int) $port->instance_id !== (int) $instance->id
|
|
|| (int) $port->user_id !== (int) $instance->user_id
|
|
|| (int) $port->vpc_id !== (int) $vpc->id
|
|
|| (int) $vpc->user_id !== (int) $instance->user_id) {
|
|
throw new CloudError('删除实例的 VPC 端口账本不一致', 409);
|
|
}
|
|
if (self::activePortJobLocked($instance, $port) !== null) {
|
|
throw new CloudError('删除实例仍有 VPC 任务未收敛', 409);
|
|
}
|
|
if ((string) $port->state === 'release_pending') {
|
|
$job = self::retryReleaseLocked($instance, $vpc, $port);
|
|
return $job === null ? [] : [$job];
|
|
}
|
|
if (!in_array((string) $port->state,
|
|
['attached', 'preparing'], true)) {
|
|
throw new CloudError('删除实例的 VPC 端口状态不允许释放', 409);
|
|
}
|
|
|
|
$now = time();
|
|
$port->state = 'release_pending';
|
|
$port->last_operation_id = null;
|
|
$port->last_error_code = '';
|
|
$port->updated_at = $now;
|
|
$port->save();
|
|
$revision = (int) $vpc->desired_revision + 1;
|
|
$vpc->desired_revision = $revision;
|
|
$vpc->state = 'provisioning';
|
|
$vpc->updated_at = $now;
|
|
$vpc->save();
|
|
$jobs = self::queueProjectionLocked($vpc, $port, $revision);
|
|
if ($jobs === []) {
|
|
self::releasePortLocked($vpc, $port);
|
|
}
|
|
return $jobs;
|
|
}
|
|
|
|
/** Queue new, audited cleanup generations for deleted instances only. */
|
|
public static function retryDeletedCleanupDue(int $limit = 10): int
|
|
{
|
|
$limit = max(1, min(100, $limit));
|
|
$candidates = CloudVpcPort::query()
|
|
->join('cloud_instances as deleted_instance',
|
|
'deleted_instance.id', '=', 'cloud_vpc_ports.instance_id')
|
|
->where('cloud_vpc_ports.active_claim', 1)
|
|
->where('cloud_vpc_ports.state', 'release_pending')
|
|
->where('deleted_instance.status', 'deleted')
|
|
->orderBy('cloud_vpc_ports.id')
|
|
->limit($limit)
|
|
->get([
|
|
'cloud_vpc_ports.id as port_id',
|
|
'cloud_vpc_ports.instance_id as instance_id',
|
|
]);
|
|
$queued = 0;
|
|
foreach ($candidates as $candidate) {
|
|
try {
|
|
$queued += (new CloudInstance())->getConnection()
|
|
->transaction(static function () use ($candidate): int {
|
|
$instance = CloudInstance::query()
|
|
->where('id', (int) $candidate->instance_id)
|
|
->lockForUpdate()->first();
|
|
if ($instance === null
|
|
|| (string) $instance->status !== 'deleted') {
|
|
return 0;
|
|
}
|
|
$reference = CloudVpcPort::query()
|
|
->where('id', (int) $candidate->port_id)
|
|
->where('instance_id', (int) $instance->id)
|
|
->where('active_claim', 1)->first();
|
|
if ($reference === null
|
|
|| (string) $reference->state
|
|
!== 'release_pending') {
|
|
return 0;
|
|
}
|
|
if (!self::deletedCleanupRetryReadyLocked(
|
|
$reference, time()
|
|
)) {
|
|
return 0;
|
|
}
|
|
$jobs = self::releaseAfterInstanceDeletedLocked(
|
|
$instance
|
|
);
|
|
return count($jobs);
|
|
}, 3);
|
|
} catch (\Throwable $error) {
|
|
error_log('cloud VPC deleted-instance cleanup unavailable: '
|
|
. get_class($error));
|
|
}
|
|
}
|
|
return $queued;
|
|
}
|
|
|
|
private static function deletedCleanupRetryReadyLocked(
|
|
CloudVpcPort $port,
|
|
int $now
|
|
): bool {
|
|
if (self::activePortJob(
|
|
(int) $port->instance_id,
|
|
(int) $port->id
|
|
) !== null) {
|
|
return false;
|
|
}
|
|
$states = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $port->vpc_id)
|
|
->orderBy('host_id')->lockForUpdate()->get();
|
|
if ($states->every(static fn ($state): bool =>
|
|
self::hostReady($state))) {
|
|
return true;
|
|
}
|
|
foreach ($states as $state) {
|
|
if (self::hostReady($state)) {
|
|
continue;
|
|
}
|
|
if (self::knownNotAppliedHostFailureLocked(
|
|
$state, $now - self::DELETED_CLEANUP_RETRY_DELAY
|
|
) === null) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Return the exact terminal generation only when it proves that no host
|
|
* mutation was applied. A failed UI state alone is never replay evidence.
|
|
*/
|
|
private static function knownNotAppliedHostFailureLocked(
|
|
CloudVpcHostState $state,
|
|
?int $completedNoLaterThan = null
|
|
): ?CloudJob {
|
|
if ((string) $state->sync_state !== 'failed'
|
|
|| !is_string($state->last_operation_id)
|
|
|| $state->last_operation_id === '') {
|
|
return null;
|
|
}
|
|
$job = CloudJob::query()
|
|
->where('operation_id', (string) $state->last_operation_id)
|
|
->where('vpc_id', (int) $state->vpc_id)
|
|
->where('host_id', (int) $state->host_id)
|
|
->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])
|
|
->lockForUpdate()->first();
|
|
if ($job === null
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| (string) $job->effect_resolution !== 'not_applied'
|
|
|| (bool) $job->effect_may_have_applied
|
|
|| !(bool) $job->safe_to_resubmit
|
|
|| (bool) $job->finalization_incident) {
|
|
return null;
|
|
}
|
|
if ($completedNoLaterThan !== null
|
|
&& max((int) ($job->terminal_at ?? 0),
|
|
(int) ($job->updated_at ?? 0)) > $completedNoLaterThan) {
|
|
return null;
|
|
}
|
|
return $job;
|
|
}
|
|
|
|
/** Trusted worker marker for VPC jobs that must finish after acceptance. */
|
|
public static function isSystemJob(CloudJob $job, array $payload): bool
|
|
{
|
|
if (!in_array((string) $job->kind, self::JOB_KINDS, true)) {
|
|
return false;
|
|
}
|
|
if (($payload['system_vpc_orchestration'] ?? null) !== true) {
|
|
throw new CloudError('VPC 系统任务缺少可信编排标识', 409);
|
|
}
|
|
self::assertJobRunnable($job, $payload);
|
|
return true;
|
|
}
|
|
|
|
public static function assertJobRunnable(CloudJob $job,
|
|
array $payload): bool
|
|
{
|
|
$kind = (string) $job->kind;
|
|
if (!in_array($kind, self::JOB_KINDS, true)) {
|
|
return true;
|
|
}
|
|
$port = $job->vpc_port_id === null ? null
|
|
: CloudVpcPort::find((int) $job->vpc_port_id);
|
|
$vpc = $job->vpc_id === null ? null
|
|
: CloudVpc::find((int) $job->vpc_id);
|
|
$instance = $job->instance_id === null ? null
|
|
: CloudInstance::find((int) $job->instance_id);
|
|
$body = $payload['body'] ?? null;
|
|
if ($port === null || $vpc === null || $instance === null
|
|
|| !is_array($body) || array_is_list($body)
|
|
|| (int) $port->vpc_id !== (int) $vpc->id
|
|
|| (int) $port->instance_id !== (int) $instance->id
|
|
|| (int) $port->user_id !== (int) $instance->user_id
|
|
|| (int) $job->user_id !== (int) $instance->user_id
|
|
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id
|
|
|| (int) ($payload['vpc_id'] ?? 0) !== (int) $vpc->id
|
|
|| (int) ($payload['vpc_port_id'] ?? 0) !== (int) $port->id
|
|
|| ($body['operation_id'] ?? null) !== (string) $job->operation_id) {
|
|
throw new CloudError('VPC 任务与资源账本不一致', 409);
|
|
}
|
|
$repair = $job->supersedes_job_id !== null;
|
|
$prior = null;
|
|
if ($repair) {
|
|
$prior = CloudJob::query()->where(
|
|
'id', (int) $job->supersedes_job_id
|
|
)->first();
|
|
$rootId = (int) ($job->vpc_repair_root_job_id ?? 0);
|
|
if ($prior === null || $rootId < 1
|
|
|| ($payload['vpc_repair'] ?? null) !== true
|
|
|| (int) ($payload['vpc_repair_root_job_id'] ?? 0)
|
|
!== $rootId
|
|
|| ($body['vpc_repair'] ?? null) !== true
|
|
|| ($body['supersedes_operation_id'] ?? null)
|
|
!== (string) $prior->operation_id
|
|
|| (int) $prior->instance_id !== (int) $job->instance_id
|
|
|| (int) $prior->vpc_id !== (int) $job->vpc_id
|
|
|| (int) $prior->vpc_port_id !== (int) $job->vpc_port_id
|
|
|| (int) $prior->host_id !== (int) $job->host_id
|
|
|| (string) $prior->kind !== $kind
|
|
|| (string) $prior->phase !== 'terminal'
|
|
|| (string) $prior->outcome !== 'failed'
|
|
|| !(bool) $prior->effect_may_have_applied
|
|
|| (bool) $prior->safe_to_resubmit
|
|
|| (string) $prior->effect_resolution !== 'unknown'
|
|
|| (bool) $prior->finalization_incident
|
|
|| (int) ($prior->vpc_repair_root_job_id ?? $prior->id)
|
|
!== $rootId) {
|
|
throw new CloudError('VPC 修复任务继任账本不一致', 409);
|
|
}
|
|
} elseif (($payload['vpc_repair'] ?? false) === true
|
|
|| array_key_exists('vpc_repair', $body)
|
|
|| array_key_exists('supersedes_operation_id', $body)) {
|
|
throw new CloudError('普通 VPC 任务包含修复字段', 409);
|
|
}
|
|
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
$state = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)
|
|
->where('host_id', (int) $job->host_id)->first();
|
|
if ($state === null
|
|
|| !hash_equals((string) $state->last_operation_id,
|
|
(string) $job->operation_id)) {
|
|
throw new CloudError('VPC 宿主机任务已过期或账本不一致', 409);
|
|
}
|
|
$expected = $kind === 'vpc_host_apply'
|
|
? self::hostApplyBody($job, $vpc, $state, $payload)
|
|
: self::hostRemoveBody($job, $vpc, $state);
|
|
} else {
|
|
$state = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)
|
|
->where('host_id', (int) $instance->host_id)->first();
|
|
if (!self::hostReady($state)
|
|
|| !hash_equals((string) $port->last_operation_id,
|
|
(string) $job->operation_id)
|
|
|| (int) $job->host_id !== (int) $instance->host_id
|
|
|| ($kind === 'vpc_port_attach'
|
|
&& (string) $port->state !== 'preparing')
|
|
|| ($kind === 'vpc_port_detach'
|
|
&& (string) $port->state !== 'detaching')) {
|
|
throw new CloudError('VPC 网卡任务已过期或路由尚未收敛', 409);
|
|
}
|
|
$expected = self::portBody($job, $vpc, $port, $state);
|
|
}
|
|
if ($repair) {
|
|
$expected['vpc_repair'] = true;
|
|
$expected['supersedes_operation_id'] =
|
|
(string) $prior->operation_id;
|
|
}
|
|
$expectedTarget = in_array($kind,
|
|
['vpc_host_apply', 'vpc_host_remove'], true)
|
|
? '/v1/vpc/networks/' . rawurlencode((string) $vpc->vpc_key)
|
|
. ($kind === 'vpc_host_apply' ? '/apply' : '/remove')
|
|
. ($repair ? '/repair' : '')
|
|
: '/v1/vms/' . rawurlencode(strtolower((string) $instance->uuid))
|
|
. '/vpc-port' . ($repair ? '/repair' : '');
|
|
$expectedMethod = match ($kind) {
|
|
'vpc_host_apply', 'vpc_host_remove', 'vpc_port_attach' => 'POST',
|
|
'vpc_port_detach' => 'DELETE',
|
|
};
|
|
if (($payload['request_target'] ?? null) !== $expectedTarget
|
|
|| strtoupper((string) ($payload['method'] ?? '')) !== $expectedMethod
|
|
|| !self::sameDocument($body, $expected)) {
|
|
throw new CloudError('VPC 任务请求与持久账本不一致', 409);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
public static function markJobRunning(CloudJob $job): void
|
|
{
|
|
$kind = (string) $job->kind;
|
|
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $job->vpc_id)
|
|
->where('host_id', (int) $job->host_id)
|
|
->where('last_operation_id', (string) $job->operation_id)
|
|
->where('sync_state', 'pending')->update([
|
|
'sync_state' => 'applying', 'updated_at' => time(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
public static function finishJobLocked(CloudJob $job, string $status,
|
|
$result, array $payload, array $operation): void
|
|
{
|
|
$kind = (string) $job->kind;
|
|
if (!in_array($kind, self::JOB_KINDS, true)) {
|
|
return;
|
|
}
|
|
self::assertJobRunnable($job, $payload);
|
|
$confirmed = $operation['operation_id'] ?? null;
|
|
if ($job->dispatched_at !== null
|
|
&& (!is_string($confirmed)
|
|
|| !hash_equals((string) $job->operation_id, $confirmed))) {
|
|
throw new CloudError('Agent VPC 结果与任务操作编号不一致', 502);
|
|
}
|
|
$instance = CloudVpcLockOrder::instance((int) $job->instance_id);
|
|
$vpc = CloudVpcLockOrder::vpcById((int) $job->vpc_id);
|
|
$port = CloudVpcLockOrder::portById((int) $job->vpc_port_id);
|
|
if ($port === null || $vpc === null || $instance === null) {
|
|
throw new CloudError('VPC 任务资源账本已丢失', 409);
|
|
}
|
|
if ($job->supersedes_job_id !== null) {
|
|
if ($status === 'succeeded') {
|
|
self::settleRepairChainLocked($job);
|
|
} elseif ($status === 'failed') {
|
|
self::restoreRepairPredecessorLocked(
|
|
$job, $vpc, $port
|
|
);
|
|
return;
|
|
}
|
|
}
|
|
if ($job->vpc_batch_id !== null) {
|
|
CloudVpcMembershipBatchService::finishChildLocked(
|
|
$job, $status, $result, $operation, $vpc, $port, $instance
|
|
);
|
|
return;
|
|
}
|
|
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
self::finishHostJobLocked(
|
|
$job, $status, $result, $operation, $vpc, $port, $instance
|
|
);
|
|
if (isset($payload['maintenance_migration_id'])) {
|
|
CloudInstanceMigrationService::finishVpcRoutingJobLocked(
|
|
$job, $status, $payload
|
|
);
|
|
}
|
|
return;
|
|
}
|
|
self::finishPortJobLocked(
|
|
$job, $status, $result, $operation, $vpc, $port, $instance
|
|
);
|
|
}
|
|
|
|
/** Resolve only VPC safety metadata after the successor proves convergence. */
|
|
private static function settleRepairChainLocked(CloudJob $job): void
|
|
{
|
|
$rootId = (int) ($job->vpc_repair_root_job_id ?? 0);
|
|
$priorId = (int) ($job->supersedes_job_id ?? 0);
|
|
if ($rootId < 1 || $priorId < 1) {
|
|
throw new CloudError('VPC 修复继任链缺失', 409);
|
|
}
|
|
$seen = [(int) $job->id => true];
|
|
$now = time();
|
|
for ($depth = 0; $depth < 32; $depth++) {
|
|
if ($priorId < 1 || isset($seen[$priorId])) {
|
|
throw new CloudError('VPC 修复继任链包含循环', 409);
|
|
}
|
|
$seen[$priorId] = true;
|
|
$prior = CloudJob::query()->where('id', $priorId)
|
|
->lockForUpdate()->first();
|
|
if ($prior === null
|
|
|| (string) $prior->kind !== (string) $job->kind
|
|
|| (int) $prior->instance_id !== (int) $job->instance_id
|
|
|| (int) $prior->vpc_id !== (int) $job->vpc_id
|
|
|| (int) $prior->vpc_port_id !== (int) $job->vpc_port_id
|
|
|| (int) $prior->host_id !== (int) $job->host_id
|
|
|| (string) $prior->phase !== 'terminal'
|
|
|| (string) $prior->outcome !== 'failed'
|
|
|| !(bool) $prior->effect_may_have_applied
|
|
|| (bool) $prior->safe_to_resubmit
|
|
|| (string) $prior->effect_resolution !== 'unknown'
|
|
|| (bool) $prior->finalization_incident
|
|
|| (int) ($prior->vpc_repair_root_job_id ?? $prior->id)
|
|
!== $rootId) {
|
|
throw new CloudError('VPC 修复继任链已变化', 409);
|
|
}
|
|
$nextId = (int) ($prior->supersedes_job_id ?? 0);
|
|
$prior->effect_resolution = 'applied';
|
|
$prior->effect_resolved_at = $now;
|
|
$prior->effect_resolved_by = null;
|
|
$prior->effect_resolution_reason = substr(
|
|
'superseded_by_vpc_repair:' . (int) $job->id . ':'
|
|
. (string) $job->operation_id,
|
|
0,
|
|
500
|
|
);
|
|
$prior->available_at = 0;
|
|
$prior->read_at = null;
|
|
$prior->updated_at = $now;
|
|
$prior->save();
|
|
if ($nextId < 1) {
|
|
if ((int) $prior->id !== $rootId) {
|
|
throw new CloudError('VPC 修复根任务身份不一致', 409);
|
|
}
|
|
return;
|
|
}
|
|
$priorId = $nextId;
|
|
}
|
|
throw new CloudError('VPC 修复继任链超过安全上限', 409);
|
|
}
|
|
|
|
/** A proved no-effect repair restores the still-fenced predecessor tip. */
|
|
private static function restoreRepairPredecessorLocked(
|
|
CloudJob $job,
|
|
CloudVpc $vpc,
|
|
CloudVpcPort $port
|
|
): void {
|
|
$prior = CloudJob::query()->where(
|
|
'id', (int) $job->supersedes_job_id
|
|
)->lockForUpdate()->first();
|
|
if ($prior === null
|
|
|| (string) $prior->kind !== (string) $job->kind
|
|
|| (string) $prior->effect_resolution !== 'unknown'
|
|
|| (string) $prior->phase !== 'terminal'
|
|
|| (string) $prior->outcome !== 'failed') {
|
|
throw new CloudError('VPC 修复前序任务已变化', 409);
|
|
}
|
|
$now = time();
|
|
if (in_array((string) $job->kind,
|
|
['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
$state = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $job->vpc_id)
|
|
->where('host_id', (int) $job->host_id)
|
|
->lockForUpdate()->first();
|
|
if ($state === null || !hash_equals(
|
|
(string) $state->last_operation_id,
|
|
(string) $job->operation_id
|
|
)) {
|
|
throw new CloudError('VPC 宿主机修复回退代次不一致', 409);
|
|
}
|
|
$state->last_operation_id = (string) $prior->operation_id;
|
|
$state->sync_state = 'unknown';
|
|
$state->last_error_code = 'vpc_result_uncertain';
|
|
$state->last_failure_at = $now;
|
|
$state->updated_at = $now;
|
|
$state->save();
|
|
} else {
|
|
if (!hash_equals(
|
|
(string) $port->last_operation_id,
|
|
(string) $job->operation_id
|
|
)) {
|
|
throw new CloudError('VPC 网卡修复回退代次不一致', 409);
|
|
}
|
|
$port->last_operation_id = (string) $prior->operation_id;
|
|
$port->last_error_code = 'vpc_result_uncertain';
|
|
$port->updated_at = $now;
|
|
$port->save();
|
|
}
|
|
if ($job->vpc_batch_id !== null
|
|
&& in_array((string) $job->kind,
|
|
['vpc_port_attach', 'vpc_port_detach'], true)) {
|
|
$item = CloudVpcMembershipBatchItem::query()
|
|
->where('batch_id', (int) $job->vpc_batch_id)
|
|
->where('instance_id', (int) $job->instance_id)
|
|
->where('vpc_port_id', (int) $job->vpc_port_id)
|
|
->lockForUpdate()->first();
|
|
if ($item === null
|
|
|| (int) $item->child_job_id !== (int) $job->id) {
|
|
throw new CloudError('VPC 批次修复回退代次不一致', 409);
|
|
}
|
|
$item->child_job_id = (int) $prior->id;
|
|
$item->state = 'unknown';
|
|
$item->last_error_code = 'vpc_result_uncertain';
|
|
$item->updated_at = $now;
|
|
$item->save();
|
|
}
|
|
$vpc->state = 'degraded';
|
|
$vpc->updated_at = $now;
|
|
$vpc->save();
|
|
}
|
|
|
|
public static function markJobUncertain(CloudJob $job): void
|
|
{
|
|
$kind = (string) $job->kind;
|
|
if (!in_array($kind, self::JOB_KINDS, true)) {
|
|
return;
|
|
}
|
|
$now = time();
|
|
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $job->vpc_id)
|
|
->where('host_id', (int) $job->host_id)
|
|
->where('last_operation_id', (string) $job->operation_id)
|
|
->update([
|
|
'sync_state' => 'unknown',
|
|
'last_error_code' => 'vpc_result_uncertain',
|
|
'last_failure_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
}
|
|
CloudVpcPort::query()->where('id', (int) $job->vpc_port_id)
|
|
->where('active_claim', 1)->update([
|
|
'last_error_code' => 'vpc_result_uncertain',
|
|
'updated_at' => $now,
|
|
]);
|
|
CloudVpc::query()->where('id', (int) $job->vpc_id)->update([
|
|
'state' => 'degraded', 'updated_at' => $now,
|
|
]);
|
|
CloudVpcMembershipBatchService::markChildUncertainLocked($job);
|
|
if (!is_string($job->sealed_payload)
|
|
|| $job->sealed_payload === '') {
|
|
return;
|
|
}
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload, 'job-payload'
|
|
);
|
|
if (is_array($payload)
|
|
&& isset($payload['maintenance_migration_id'])) {
|
|
CloudInstanceMigrationService::finishVpcRoutingJobLocked(
|
|
$job, 'uncertain', $payload
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Queue one new, explicitly linked repair generation for an exact VPC
|
|
* operation whose Agent journal is reachable but still inconclusive.
|
|
* The failed operation is never replayed.
|
|
*/
|
|
public static function repairUnknownEffect(
|
|
array $actor,
|
|
int $jobId,
|
|
array $inspection
|
|
): CloudJob {
|
|
CloudAccess::manage($actor);
|
|
$reference = CloudJob::query()->where('id', $jobId)->first();
|
|
if ($reference === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
if (!in_array((string) $reference->kind, self::JOB_KINDS, true)
|
|
|| (int) ($reference->instance_id ?? 0) < 1
|
|
|| (int) ($reference->vpc_id ?? 0) < 1
|
|
|| (int) ($reference->vpc_port_id ?? 0) < 1
|
|
|| (int) ($reference->host_id ?? 0) < 1) {
|
|
throw new CloudError('该任务不是可修复的 VPC 操作', 409);
|
|
}
|
|
self::assertRepairInspection($reference, $inspection);
|
|
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
function () use ($reference, $inspection): CloudJob {
|
|
$instance = CloudVpcLockOrder::instance(
|
|
(int) $reference->instance_id
|
|
);
|
|
$vpc = CloudVpcLockOrder::vpcById(
|
|
(int) $reference->vpc_id
|
|
);
|
|
$port = CloudVpcLockOrder::portById(
|
|
(int) $reference->vpc_port_id
|
|
);
|
|
$state = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $reference->vpc_id)
|
|
->where('host_id', (int) $reference->host_id)
|
|
->lockForUpdate()->first();
|
|
if ($instance === null || $vpc === null || $port === null
|
|
|| $state === null
|
|
|| (int) $port->instance_id !== (int) $instance->id
|
|
|| (int) $port->vpc_id !== (int) $vpc->id
|
|
|| (int) $reference->host_id
|
|
!== (int) $state->host_id) {
|
|
throw new CloudError('VPC 修复资源账本不完整', 409);
|
|
}
|
|
|
|
$batch = null;
|
|
$item = null;
|
|
if ($reference->vpc_batch_id !== null) {
|
|
$batch = CloudVpcMembershipBatch::query()
|
|
->where('id', (int) $reference->vpc_batch_id)
|
|
->lockForUpdate()->first();
|
|
if ($batch === null
|
|
|| (string) $batch->state !== 'blocked'
|
|
|| (int) $batch->vpc_id !== (int) $vpc->id) {
|
|
throw new CloudError('VPC 批次不处于可修复状态', 409);
|
|
}
|
|
if (in_array((string) $reference->kind,
|
|
['vpc_port_attach', 'vpc_port_detach'], true)) {
|
|
$item = CloudVpcMembershipBatchItem::query()
|
|
->where('batch_id', (int) $batch->id)
|
|
->where('instance_id', (int) $instance->id)
|
|
->where('vpc_port_id', (int) $port->id)
|
|
->lockForUpdate()->first();
|
|
}
|
|
}
|
|
|
|
$prior = CloudJob::query()->where(
|
|
'id', (int) $reference->id
|
|
)->lockForUpdate()->first();
|
|
if ($prior === null) {
|
|
throw new CloudError('待修复任务已不存在', 409);
|
|
}
|
|
self::assertRepairCandidateLocked(
|
|
$prior, $instance, $vpc, $port, $state,
|
|
$batch, $item, $inspection
|
|
);
|
|
$rootId = (int) ($prior->vpc_repair_root_job_id
|
|
?? $prior->id);
|
|
$chain = CloudJob::query()
|
|
->where(static function ($query) use ($rootId): void {
|
|
$query->where('id', $rootId)
|
|
->orWhere('vpc_repair_root_job_id', $rootId);
|
|
})->orderBy('id')->lockForUpdate()->get();
|
|
$unknownGenerations = $chain->filter(
|
|
static fn (CloudJob $candidate): bool =>
|
|
(string) $candidate->effect_resolution === 'unknown'
|
|
);
|
|
$latestUnknown = $unknownGenerations->last();
|
|
if ($chain->isEmpty() || $unknownGenerations->count() > 32
|
|
|| $chain->contains(static fn (CloudJob $candidate): bool =>
|
|
(string) $candidate->phase !== 'terminal')
|
|
|| !$latestUnknown instanceof CloudJob
|
|
|| (int) $latestUnknown->id !== (int) $prior->id) {
|
|
throw new CloudError('VPC 修复链已有其他活动代次', 409);
|
|
}
|
|
foreach ($chain as $candidate) {
|
|
if ((int) $candidate->instance_id !== (int) $instance->id
|
|
|| (int) $candidate->vpc_id !== (int) $vpc->id
|
|
|| (int) $candidate->vpc_port_id !== (int) $port->id
|
|
|| (int) $candidate->host_id !== (int) $prior->host_id
|
|
|| (string) $candidate->kind
|
|
!== (string) $prior->kind) {
|
|
throw new CloudError('VPC 修复链资源身份不一致', 409);
|
|
}
|
|
}
|
|
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $prior->sealed_payload, 'job-payload'
|
|
);
|
|
self::assertJobRunnable($prior, $payload);
|
|
$job = self::baseJob(
|
|
(string) $prior->kind,
|
|
$vpc,
|
|
$port,
|
|
(int) $prior->host_id,
|
|
$prior->vpc_batch_id === null
|
|
? null : (int) $prior->vpc_batch_id
|
|
);
|
|
$job->supersedes_job_id = (int) $prior->id;
|
|
$job->vpc_repair_root_job_id = $rootId;
|
|
$extra = [];
|
|
foreach ([
|
|
'maintenance_migration_id', 'vpc_migration_direction',
|
|
'vpc_batch_id', 'vpc_batch_revision',
|
|
] as $key) {
|
|
if (array_key_exists($key, $payload)) {
|
|
$extra[$key] = $payload[$key];
|
|
}
|
|
}
|
|
if ((string) $prior->kind === 'vpc_host_apply'
|
|
&& is_array($payload['host_projection'] ?? null)
|
|
&& !array_is_list($payload['host_projection'])) {
|
|
$extra['host_projection'] = $payload['host_projection'];
|
|
}
|
|
if ($job->vpc_batch_id !== null
|
|
&& in_array((string) $job->kind,
|
|
['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
$revision = (int) ($extra['vpc_batch_revision']
|
|
?? $batch?->desired_revision ?? 0);
|
|
if ($revision < 1) {
|
|
throw new CloudError('VPC 批次修复缺少投影代次', 409);
|
|
}
|
|
$job->request_idempotency_key = substr(
|
|
self::batchProjectionRequestPrefix(
|
|
(int) $job->vpc_batch_id,
|
|
$revision
|
|
) . (string) $job->kind . '-'
|
|
. (int) $job->host_id . '-repair-'
|
|
. (string) $job->operation_id,
|
|
0,
|
|
128
|
|
);
|
|
}
|
|
if (in_array((string) $job->kind,
|
|
['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
$body = (string) $job->kind === 'vpc_host_apply'
|
|
? self::hostApplyBody($job, $vpc, $state, $payload)
|
|
: self::hostRemoveBody($job, $vpc, $state);
|
|
$target = '/v1/vpc/networks/'
|
|
. rawurlencode((string) $vpc->vpc_key)
|
|
. ((string) $job->kind === 'vpc_host_apply'
|
|
? '/apply/repair' : '/remove/repair');
|
|
$method = 'POST';
|
|
} else {
|
|
if (!self::hostReady($state)) {
|
|
throw new CloudError(
|
|
'VPC 网卡修复所需宿主机路由未就绪', 409
|
|
);
|
|
}
|
|
$body = self::portBody($job, $vpc, $port, $state);
|
|
$target = '/v1/vms/' . rawurlencode(
|
|
strtolower((string) $instance->uuid)
|
|
) . '/vpc-port/repair';
|
|
$method = (string) $job->kind === 'vpc_port_attach'
|
|
? 'POST' : 'DELETE';
|
|
}
|
|
$body['vpc_repair'] = true;
|
|
$body['supersedes_operation_id'] =
|
|
(string) $prior->operation_id;
|
|
self::sealJob(
|
|
$job, $vpc, $port, $method, $target, $body,
|
|
$extra + [
|
|
'vpc_repair' => true,
|
|
'vpc_repair_root_job_id' => $rootId,
|
|
]
|
|
);
|
|
if (in_array((string) $job->kind,
|
|
['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
$state->last_operation_id = (string) $job->operation_id;
|
|
$state->sync_state = 'pending';
|
|
$state->last_error_code = '';
|
|
$state->updated_at = time();
|
|
$state->save();
|
|
} else {
|
|
$port->last_operation_id = (string) $job->operation_id;
|
|
$port->last_error_code = '';
|
|
$port->updated_at = time();
|
|
$port->save();
|
|
if ($item !== null) {
|
|
$item->child_job_id = (int) $job->id;
|
|
$item->state = 'queued';
|
|
$item->last_error_code = '';
|
|
$item->updated_at = time();
|
|
$item->save();
|
|
}
|
|
}
|
|
return $job;
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
private static function assertRepairInspection(
|
|
CloudJob $job,
|
|
array $inspection
|
|
): void {
|
|
if (($inspection['operation_id'] ?? null)
|
|
!== (string) $job->operation_id
|
|
|| ($inspection['journal_status'] ?? null) !== 'failed'
|
|
|| ($inspection['outcome'] ?? null) !== 'inconclusive'
|
|
|| ($inspection['effect_resolution'] ?? null) !== 'inconclusive'
|
|
|| ($inspection['fence_state'] ?? null) !== 'active'
|
|
|| ($inspection['fence_resolution_required'] ?? null) !== true
|
|
|| array_key_exists('completion_delivery_state', $inspection)
|
|
|| !is_int($inspection['checked_at'] ?? null)
|
|
|| $inspection['checked_at'] < time() - 120
|
|
|| $inspection['checked_at'] > time() + 5) {
|
|
throw new CloudError(
|
|
'宿主机没有提供可执行 VPC 修复的新鲜不一致证据', 409
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function assertRepairCandidateLocked(
|
|
CloudJob $job,
|
|
CloudInstance $instance,
|
|
CloudVpc $vpc,
|
|
CloudVpcPort $port,
|
|
CloudVpcHostState $state,
|
|
?CloudVpcMembershipBatch $batch,
|
|
?CloudVpcMembershipBatchItem $item,
|
|
array $inspection
|
|
): void {
|
|
self::assertRepairInspection($job, $inspection);
|
|
if (!in_array((string) $job->kind, self::JOB_KINDS, true)
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| (int) $job->dispatch_attempts !== 1
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown'
|
|
|| (bool) $job->finalization_incident
|
|
|| (int) $job->instance_id !== (int) $instance->id
|
|
|| (int) $job->vpc_id !== (int) $vpc->id
|
|
|| (int) $job->vpc_port_id !== (int) $port->id
|
|
|| (int) $job->host_id !== (int) $state->host_id
|
|
|| (int) $job->user_id !== (int) $port->user_id) {
|
|
throw new CloudError('VPC 任务已不处于可修复状态', 409);
|
|
}
|
|
if (in_array((string) $job->kind,
|
|
['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
if (!hash_equals(
|
|
(string) $state->last_operation_id,
|
|
(string) $job->operation_id
|
|
) || (string) $state->sync_state !== 'unknown') {
|
|
throw new CloudError('VPC 宿主机修复代次已过期', 409);
|
|
}
|
|
} elseif (!hash_equals(
|
|
(string) $port->last_operation_id,
|
|
(string) $job->operation_id
|
|
) || !in_array((string) $port->state,
|
|
['preparing', 'detaching'], true)) {
|
|
throw new CloudError('VPC 网卡修复代次已过期', 409);
|
|
}
|
|
if ($job->vpc_batch_id !== null
|
|
&& ($batch === null || (string) $batch->state !== 'blocked'
|
|
|| (in_array((string) $job->kind,
|
|
['vpc_port_attach', 'vpc_port_detach'], true)
|
|
&& ($item === null
|
|
|| (int) $item->child_job_id !== (int) $job->id
|
|
|| (string) $item->state !== 'unknown')))) {
|
|
throw new CloudError('VPC 批次修复子任务已过期', 409);
|
|
}
|
|
}
|
|
|
|
private static function finishHostJobLocked(CloudJob $job,
|
|
string $status, $result, array $operation, CloudVpc $vpc,
|
|
CloudVpcPort $port, CloudInstance $instance): void
|
|
{
|
|
$state = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)
|
|
->where('host_id', (int) $job->host_id)
|
|
->lockForUpdate()->first();
|
|
if ($state === null
|
|
|| !hash_equals((string) $state->last_operation_id,
|
|
(string) $job->operation_id)) {
|
|
throw new CloudError('VPC 宿主机结果与当前投影不一致', 409);
|
|
}
|
|
$now = time();
|
|
if ($status === 'succeeded') {
|
|
self::assertHostResult($job, $vpc, $state, $result);
|
|
if ((string) $job->kind === 'vpc_host_remove') {
|
|
$state->delete();
|
|
} else {
|
|
$state->applied_revision = (int) $state->desired_revision;
|
|
$state->applied_digest = (string) $state->desired_digest;
|
|
$state->sync_state = 'ready';
|
|
$state->last_error_code = '';
|
|
$state->last_success_at = $now;
|
|
$state->updated_at = $now;
|
|
$state->save();
|
|
}
|
|
self::advancePortLocked($instance, $vpc, $port);
|
|
return;
|
|
}
|
|
$uncertain = $status === 'uncertain';
|
|
$code = $uncertain ? 'vpc_result_uncertain'
|
|
: (CloudOperationErrorCatalog::workerSummary(
|
|
(string) $job->kind, 'failed', $operation
|
|
) ?? 'vpc_host_apply_failed');
|
|
$state->sync_state = $uncertain ? 'unknown' : 'failed';
|
|
$state->last_error_code = substr($code, 0, 64);
|
|
$state->last_failure_at = $now;
|
|
$state->updated_at = $now;
|
|
$state->save();
|
|
$port->last_error_code = substr($code, 0, 64);
|
|
$port->updated_at = $now;
|
|
$port->save();
|
|
$vpc->state = 'degraded';
|
|
$vpc->updated_at = $now;
|
|
$vpc->save();
|
|
}
|
|
|
|
private static function finishPortJobLocked(CloudJob $job,
|
|
string $status, $result, array $operation, CloudVpc $vpc,
|
|
CloudVpcPort $port, CloudInstance $instance): void
|
|
{
|
|
$now = time();
|
|
if ($status !== 'succeeded') {
|
|
$uncertain = $status === 'uncertain';
|
|
$code = $uncertain ? 'vpc_result_uncertain'
|
|
: (CloudOperationErrorCatalog::workerSummary(
|
|
(string) $job->kind, 'failed', $operation
|
|
) ?? 'vpc_port_change_failed');
|
|
if ((string) $job->kind === 'vpc_port_detach' && !$uncertain) {
|
|
$port->state = 'attached';
|
|
}
|
|
$port->last_error_code = substr($code, 0, 64);
|
|
$port->updated_at = $now;
|
|
$port->save();
|
|
$vpc->state = 'degraded';
|
|
$vpc->updated_at = $now;
|
|
$vpc->save();
|
|
return;
|
|
}
|
|
self::assertPortResult($job, $vpc, $port, $instance, $result);
|
|
if ((string) $job->kind === 'vpc_port_attach') {
|
|
$port->state = 'attached';
|
|
$port->applied_revision = (int) $port->desired_revision;
|
|
$port->last_error_code = '';
|
|
$port->attached_at = $now;
|
|
$port->updated_at = $now;
|
|
$port->save();
|
|
$vpc->state = 'active';
|
|
$vpc->updated_at = $now;
|
|
$vpc->save();
|
|
return;
|
|
}
|
|
|
|
$port->state = 'release_pending';
|
|
$port->last_error_code = '';
|
|
$port->updated_at = $now;
|
|
$port->save();
|
|
$revision = (int) $vpc->desired_revision + 1;
|
|
$vpc->desired_revision = $revision;
|
|
$vpc->state = 'provisioning';
|
|
$vpc->updated_at = $now;
|
|
$vpc->save();
|
|
$jobs = self::queueProjectionLocked($vpc, $port, $revision);
|
|
if ($jobs === []) {
|
|
self::releasePortLocked($vpc, $port);
|
|
}
|
|
}
|
|
|
|
private static function advancePortLocked(CloudInstance $instance,
|
|
CloudVpc $vpc, CloudVpcPort $port): void
|
|
{
|
|
$states = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
|
|
->lockForUpdate()->get();
|
|
if ($states->contains(static fn ($state): bool =>
|
|
!self::hostReady($state))) {
|
|
return;
|
|
}
|
|
if ((string) $port->state === 'preparing') {
|
|
$hostState = $states->first(static fn ($state): bool =>
|
|
(int) $state->host_id === (int) $instance->host_id);
|
|
if ($hostState === null) {
|
|
throw new CloudError('实例宿主机缺少 VPC 路由投影', 409);
|
|
}
|
|
self::queuePortJobLocked(
|
|
'vpc_port_attach', $instance, $vpc, $port, $hostState
|
|
);
|
|
return;
|
|
}
|
|
if ((string) $port->state === 'release_pending') {
|
|
self::releasePortLocked($vpc, $port);
|
|
}
|
|
}
|
|
|
|
/** @return list<CloudJob> */
|
|
private static function beginMigrationProjectionLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
string $direction
|
|
): array {
|
|
if ($migration->vpc_port_id === null) {
|
|
return [];
|
|
}
|
|
if (!in_array($direction, ['target', 'source'], true)
|
|
|| (int) $migration->instance_id !== (int) $instance->id
|
|
|| (int) $instance->host_id
|
|
!== (int) $migration->source_host_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 路由切换实例账本不一致', 409
|
|
);
|
|
}
|
|
$destination = $direction === 'target'
|
|
? (int) $migration->target_host_id
|
|
: (int) $migration->source_host_id;
|
|
[$port, $vpc] = self::migrationPortAndVpcLocked(
|
|
(int) $migration->vpc_port_id
|
|
);
|
|
if ((int) $port->instance_id !== (int) $instance->id
|
|
|| (int) $port->user_id !== (int) $instance->user_id
|
|
|| (int) $vpc->user_id !== (int) $instance->user_id
|
|
|| self::activePortJobLocked($instance, $port) !== null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 端口任务账本不一致或仍未终结', 409
|
|
);
|
|
}
|
|
if ((string) $port->state === 'attached') {
|
|
if ($direction !== 'target' || $port->migration_host_id !== null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 端口尚未进入目标切换状态', 409
|
|
);
|
|
}
|
|
self::assertVpcSettledLocked($vpc);
|
|
} elseif ((string) $port->state !== 'migrating'
|
|
|| !in_array((int) ($port->migration_host_id ?? 0), [
|
|
(int) $migration->source_host_id,
|
|
(int) $migration->target_host_id,
|
|
], true)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 端口位置状态无效', 409
|
|
);
|
|
}
|
|
|
|
$now = time();
|
|
$port->state = 'migrating';
|
|
$port->migration_host_id = $destination;
|
|
$port->last_operation_id = null;
|
|
$port->last_error_code = '';
|
|
$revision = (int) $vpc->desired_revision + 1;
|
|
$port->desired_revision = $revision;
|
|
$port->updated_at = $now;
|
|
$port->save();
|
|
$vpc->desired_revision = $revision;
|
|
$vpc->state = 'provisioning';
|
|
$vpc->updated_at = $now;
|
|
$vpc->save();
|
|
$jobs = self::queueProjectionLocked(
|
|
$vpc,
|
|
$port,
|
|
$revision,
|
|
[
|
|
'maintenance_migration_id' => (int) $migration->id,
|
|
'vpc_migration_direction' => $direction,
|
|
]
|
|
);
|
|
if ($jobs === []) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 路由切换没有生成宿主机任务', 409
|
|
);
|
|
}
|
|
return $jobs;
|
|
}
|
|
|
|
/** @return array{0:CloudVpcPort,1:CloudVpc} */
|
|
private static function migrationPortAndVpcLocked(
|
|
int $portId
|
|
): array {
|
|
$reference = CloudVpcPort::query()->where('id', $portId)
|
|
->where('active_claim', 1)->first();
|
|
$vpc = $reference === null ? null : CloudVpc::query()
|
|
->where('id', (int) $reference->vpc_id)
|
|
->lockForUpdate()->first();
|
|
$port = $vpc === null ? null : CloudVpcPort::query()
|
|
->where('id', $portId)
|
|
->where('vpc_id', (int) $vpc->id)
|
|
->where('active_claim', 1)->lockForUpdate()->first();
|
|
if ($port === null || $vpc === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 端口或网络账本缺失', 409
|
|
);
|
|
}
|
|
return [$port, $vpc];
|
|
}
|
|
|
|
/**
|
|
* @param array<int,int> $hostOverrides port id => effective host id
|
|
* @return array{0:list<array{private_ip:string,mac_address:string,host_id:int}>,1:list<int>}
|
|
*/
|
|
private static function projectionRowsLocked(
|
|
CloudVpc $vpc,
|
|
array $hostOverrides = []
|
|
): array {
|
|
$ports = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
|
|
->where('active_claim', 1)
|
|
->whereIn('state', [
|
|
'preparing', 'attached', 'detaching', 'migrating',
|
|
])->orderBy('id')->lockForUpdate()->get();
|
|
$instanceIds = $ports->pluck('instance_id')->map(
|
|
static fn ($id): int => (int) $id
|
|
)->all();
|
|
$instances = $instanceIds === [] ? collect()
|
|
: CloudInstance::query()->whereIn('id', $instanceIds)
|
|
->orderBy('id')->get()->keyBy('id');
|
|
if ($instances->count() !== count(array_unique($instanceIds))) {
|
|
throw new CloudError('VPC 端口关联的实例账本缺失', 409);
|
|
}
|
|
$rows = [];
|
|
$hostIds = [];
|
|
$usedOverrides = [];
|
|
foreach ($ports as $port) {
|
|
$instance = $instances->get((int) $port->instance_id);
|
|
if ($instance === null
|
|
|| (int) $instance->user_id !== (int) $vpc->user_id
|
|
|| (string) $instance->status === 'deleted') {
|
|
throw new CloudError('VPC 端口关联的实例状态无效', 409);
|
|
}
|
|
$portId = (int) $port->id;
|
|
if (isset($hostOverrides[$portId])) {
|
|
$hostId = (int) $hostOverrides[$portId];
|
|
$usedOverrides[$portId] = true;
|
|
} elseif ((string) $port->state === 'migrating') {
|
|
$hostId = (int) ($port->migration_host_id ?? 0);
|
|
if ($hostId < 1) {
|
|
throw new CloudError('迁移 VPC 端口缺少目标宿主机', 409);
|
|
}
|
|
} else {
|
|
if ($port->migration_host_id !== null) {
|
|
throw new CloudError('普通 VPC 端口包含迁移位置残留', 409);
|
|
}
|
|
$hostId = (int) $instance->host_id;
|
|
}
|
|
if ($hostId < 1) {
|
|
throw new CloudError('VPC 端口宿主机账本无效', 409);
|
|
}
|
|
$hostIds[$hostId] = true;
|
|
$rows[] = [
|
|
'private_ip' => (string) $port->private_ip,
|
|
'mac_address' => (string) $port->mac_address,
|
|
'host_id' => $hostId,
|
|
];
|
|
}
|
|
if (count($usedOverrides) !== count($hostOverrides)) {
|
|
throw new CloudError('VPC 迁移端口不在当前投影中', 409);
|
|
}
|
|
$ids = array_map('intval', array_keys($hostIds));
|
|
sort($ids, SORT_NUMERIC);
|
|
return [$rows, $ids];
|
|
}
|
|
|
|
/** @return list<CloudJob> */
|
|
private static function queueProjectionLocked(CloudVpc $vpc,
|
|
CloudVpcPort $focusPort, int $revision,
|
|
array $jobContext = [],
|
|
?string $provenNotAppliedOperationId = null): array
|
|
{
|
|
[$projectionPorts, $hostIds] = self::projectionRowsLocked($vpc);
|
|
$transports = self::readyTransports($hostIds);
|
|
$projections = CloudVpcProjection::build([
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
], $projectionPorts, $transports, $revision);
|
|
|
|
$states = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
|
|
->lockForUpdate()->get()->keyBy('host_id');
|
|
$jobs = [];
|
|
$now = time();
|
|
foreach ($projections as $hostId => $projection) {
|
|
$state = $states->get($hostId);
|
|
if ($state === null) {
|
|
$state = new CloudVpcHostState();
|
|
$state->vpc_id = (int) $vpc->id;
|
|
$state->host_id = (int) $hostId;
|
|
$state->applied_revision = 0;
|
|
$state->applied_digest = null;
|
|
$state->last_success_at = 0;
|
|
$state->last_failure_at = 0;
|
|
$state->created_at = $now;
|
|
} elseif (!self::hostReady($state)
|
|
&& !self::isKnownNotAppliedHostState(
|
|
$state,
|
|
$provenNotAppliedOperationId
|
|
)) {
|
|
throw new CloudError(
|
|
'旧 VPC 宿主机投影尚未收敛,不能生成新一代', 409
|
|
);
|
|
}
|
|
$state->desired_revision = $revision;
|
|
$state->desired_digest = (string) $projection['digest'];
|
|
$state->sync_state = 'pending';
|
|
$state->last_error_code = '';
|
|
$state->updated_at = $now;
|
|
$state->save();
|
|
$job = self::queueHostJobLocked(
|
|
'vpc_host_apply', $vpc, $focusPort, $state,
|
|
$projection['network'], $jobContext
|
|
);
|
|
$jobs[] = $job;
|
|
$states->forget($hostId);
|
|
}
|
|
foreach ($states as $state) {
|
|
if (!self::hostReady($state)) {
|
|
if (!self::isKnownNotAppliedHostState(
|
|
$state,
|
|
$provenNotAppliedOperationId
|
|
)) {
|
|
throw new CloudError(
|
|
'旧 VPC 宿主机投影尚未收敛,不能清理', 409
|
|
);
|
|
}
|
|
$hasAppliedProjection = (int) $state->applied_revision > 0
|
|
&& preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $state->applied_digest) === 1;
|
|
if (!$hasAppliedProjection) {
|
|
// A known-not-applied first projection has no host state
|
|
// to remove. Dropping only this failed intent is safe.
|
|
$state->delete();
|
|
continue;
|
|
}
|
|
}
|
|
$state->sync_state = 'pending';
|
|
$state->last_error_code = '';
|
|
$state->updated_at = $now;
|
|
$state->save();
|
|
$jobs[] = self::queueHostJobLocked(
|
|
'vpc_host_remove', $vpc, $focusPort, $state, null,
|
|
$jobContext
|
|
);
|
|
}
|
|
return $jobs;
|
|
}
|
|
|
|
private static function retryPreparingLocked(CloudInstance $instance,
|
|
CloudVpc $vpc, CloudVpcPort $port): ?CloudJob
|
|
{
|
|
$states = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
|
|
->lockForUpdate()->get();
|
|
if ($states->isEmpty()) {
|
|
throw new CloudError('VPC 路由投影账本缺失', 409);
|
|
}
|
|
if ($states->contains(static fn ($state): bool =>
|
|
(string) $state->sync_state === 'unknown')) {
|
|
throw new CloudError('VPC 路由结果正在自动核对,请勿重复提交', 409);
|
|
}
|
|
if ($states->every(static fn ($state): bool => self::hostReady($state))) {
|
|
return self::queuePortJobLocked('vpc_port_attach', $instance,
|
|
$vpc, $port, $states->firstWhere('host_id', $instance->host_id));
|
|
}
|
|
$ports = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
|
|
->where('active_claim', 1)
|
|
->whereIn('state', ['preparing', 'attached', 'detaching'])
|
|
->orderBy('id')->get();
|
|
$instances = CloudInstance::query()
|
|
->whereIn('id', $ports->pluck('instance_id')->all())
|
|
->get()->keyBy('id');
|
|
$hostIds = [];
|
|
$rows = [];
|
|
foreach ($ports as $item) {
|
|
$bound = $instances->get((int) $item->instance_id);
|
|
if ($bound === null) {
|
|
throw new CloudError('VPC 端口关联的实例账本缺失', 409);
|
|
}
|
|
$hostIds[(int) $bound->host_id] = true;
|
|
$rows[] = ['private_ip' => (string) $item->private_ip,
|
|
'mac_address' => (string) $item->mac_address,
|
|
'host_id' => (int) $bound->host_id];
|
|
}
|
|
$projections = CloudVpcProjection::build([
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
], $rows, self::readyTransports(array_keys($hostIds)),
|
|
(int) $vpc->desired_revision);
|
|
foreach ($states as $state) {
|
|
if ((string) $state->sync_state !== 'failed') {
|
|
continue;
|
|
}
|
|
if (self::knownNotAppliedHostFailureLocked($state) === null) {
|
|
throw new CloudError(
|
|
'VPC 路由失败结果尚未证明未生效,不能重试', 409
|
|
);
|
|
}
|
|
$projection = $projections[(int) $state->host_id] ?? null;
|
|
if ($projection !== null
|
|
&& hash_equals((string) $state->desired_digest,
|
|
(string) $projection['digest'])
|
|
&& (int) $state->desired_revision
|
|
=== (int) $vpc->desired_revision) {
|
|
return self::queueHostJobLocked('vpc_host_apply', $vpc,
|
|
$port, $state, $projection['network']);
|
|
}
|
|
if ($projection === null && is_string($state->applied_digest)) {
|
|
return self::queueHostJobLocked('vpc_host_remove', $vpc,
|
|
$port, $state, null);
|
|
}
|
|
throw new CloudError('VPC 路由重试投影与持久账本不一致', 409);
|
|
}
|
|
throw new CloudError('VPC 路由任务仍在执行或等待核对', 409);
|
|
}
|
|
|
|
private static function retryReleaseLocked(CloudInstance $instance,
|
|
CloudVpc $vpc, CloudVpcPort $port): ?CloudJob
|
|
{
|
|
$states = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
|
|
->lockForUpdate()->get();
|
|
if ($states->contains(static fn ($state): bool =>
|
|
(string) $state->sync_state === 'unknown')) {
|
|
throw new CloudError('VPC 清理结果正在自动核对,请勿重复提交', 409);
|
|
}
|
|
if ($states->every(static fn ($state): bool => self::hostReady($state))) {
|
|
self::releasePortLocked($vpc, $port);
|
|
return null;
|
|
}
|
|
foreach ($states as $state) {
|
|
if ((string) $state->sync_state !== 'failed') {
|
|
continue;
|
|
}
|
|
if (self::knownNotAppliedHostFailureLocked($state) === null) {
|
|
throw new CloudError(
|
|
'VPC 清理失败结果尚未证明未生效,不能重试', 409
|
|
);
|
|
}
|
|
// Rebuild the desired topology. A state absent from the new
|
|
// projection is a remove retry; otherwise queue the exact apply.
|
|
$ports = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
|
|
->where('active_claim', 1)
|
|
->whereIn('state', ['preparing', 'attached', 'detaching'])
|
|
->get();
|
|
$instances = CloudInstance::query()
|
|
->whereIn('id', $ports->pluck('instance_id')->all())
|
|
->get()->keyBy('id');
|
|
$hostIds = [];
|
|
$rows = [];
|
|
foreach ($ports as $item) {
|
|
$bound = $instances->get((int) $item->instance_id);
|
|
if ($bound === null) {
|
|
throw new CloudError('VPC 端口关联的实例账本缺失', 409);
|
|
}
|
|
$hostIds[(int) $bound->host_id] = true;
|
|
$rows[] = ['private_ip' => (string) $item->private_ip,
|
|
'mac_address' => (string) $item->mac_address,
|
|
'host_id' => (int) $bound->host_id];
|
|
}
|
|
$projections = CloudVpcProjection::build([
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
], $rows, self::readyTransports(array_keys($hostIds)),
|
|
(int) $vpc->desired_revision);
|
|
$projection = $projections[(int) $state->host_id] ?? null;
|
|
return $projection === null
|
|
? self::queueHostJobLocked('vpc_host_remove', $vpc,
|
|
$port, $state, null)
|
|
: self::queueHostJobLocked('vpc_host_apply', $vpc,
|
|
$port, $state, $projection['network']);
|
|
}
|
|
throw new CloudError('VPC 清理任务仍在执行', 409);
|
|
}
|
|
|
|
private static function queueHostJobLocked(string $kind, CloudVpc $vpc,
|
|
CloudVpcPort $port, CloudVpcHostState $state,
|
|
?array $network, array $jobContext = []): CloudJob
|
|
{
|
|
$batchId = isset($jobContext['vpc_batch_id'])
|
|
? (int) $jobContext['vpc_batch_id'] : null;
|
|
$job = self::baseJob(
|
|
$kind, $vpc, $port, (int) $state->host_id, $batchId
|
|
);
|
|
if ($batchId !== null && isset($jobContext['vpc_batch_revision'])) {
|
|
$job->request_idempotency_key = substr(
|
|
self::batchProjectionRequestPrefix(
|
|
$batchId,
|
|
(int) $jobContext['vpc_batch_revision']
|
|
) . $kind . '-' . (int) $state->host_id . '-'
|
|
. (string) $job->operation_id,
|
|
0,
|
|
128
|
|
);
|
|
}
|
|
$body = $kind === 'vpc_host_apply' ? [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'desired_digest' => (string) $state->desired_digest,
|
|
'network' => $network,
|
|
] : [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'expected_revision' => (int) $state->applied_revision,
|
|
'expected_digest' => (string) $state->applied_digest,
|
|
];
|
|
if ($kind === 'vpc_host_remove'
|
|
&& ((int) $state->applied_revision < 1
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $state->applied_digest) !== 1)) {
|
|
throw new CloudError('VPC 宿主机删除缺少已应用投影证据', 409);
|
|
}
|
|
self::sealJob($job, $vpc, $port, 'POST',
|
|
'/v1/vpc/networks/' . rawurlencode((string) $vpc->vpc_key)
|
|
. ($kind === 'vpc_host_apply' ? '/apply' : '/remove'),
|
|
$body, ['host_projection' => $network] + $jobContext);
|
|
$state->last_operation_id = (string) $job->operation_id;
|
|
$state->sync_state = 'pending';
|
|
$state->updated_at = time();
|
|
$state->save();
|
|
return $job;
|
|
}
|
|
|
|
private static function queuePortJobLocked(string $kind,
|
|
CloudInstance $instance, CloudVpc $vpc, CloudVpcPort $port,
|
|
?CloudVpcHostState $state, ?int $batchId = null): CloudJob
|
|
{
|
|
if (!self::hostReady($state)) {
|
|
throw new CloudError('实例宿主机的 VPC 路由尚未收敛', 409);
|
|
}
|
|
$job = self::baseJob(
|
|
$kind, $vpc, $port, (int) $instance->host_id, $batchId
|
|
);
|
|
$body = [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'port_key' => (string) $port->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'private_ip' => (string) $port->private_ip,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
'mac' => (string) $port->mac_address,
|
|
'host_revision' => (int) $state->applied_revision,
|
|
'host_digest' => (string) $state->applied_digest,
|
|
'confirm_restart' => true,
|
|
];
|
|
self::sealJob($job, $vpc, $port,
|
|
$kind === 'vpc_port_attach' ? 'POST' : 'DELETE',
|
|
'/v1/vms/' . rawurlencode(strtolower((string) $instance->uuid))
|
|
. '/vpc-port', $body, $batchId === null ? [] : [
|
|
'vpc_batch_id' => $batchId,
|
|
]);
|
|
$port->last_operation_id = (string) $job->operation_id;
|
|
$port->last_error_code = '';
|
|
$port->updated_at = time();
|
|
$port->save();
|
|
return $job;
|
|
}
|
|
|
|
private static function baseJob(string $kind, CloudVpc $vpc,
|
|
CloudVpcPort $port, int $hostId, ?int $batchId = null): CloudJob
|
|
{
|
|
$attempt = CloudJob::query()->where('kind', $kind)
|
|
->where('vpc_port_id', (int) $port->id)
|
|
->where('host_id', $hostId)->lockForUpdate()->count() + 1;
|
|
$job = new CloudJob();
|
|
$job->operation_id = bin2hex(random_bytes(16));
|
|
$job->request_idempotency_key = substr('vpc-' . $kind . '-'
|
|
. (int) $port->id . '-' . $hostId . '-' . $attempt, 0, 128);
|
|
$job->request_fingerprint = str_repeat('0', 64);
|
|
$job->instance_id = (int) $port->instance_id;
|
|
$job->vpc_id = (int) $vpc->id;
|
|
$job->vpc_port_id = (int) $port->id;
|
|
if ($batchId !== null) {
|
|
$job->vpc_batch_id = $batchId;
|
|
}
|
|
$job->user_id = (int) $port->user_id;
|
|
$job->host_id = $hostId;
|
|
$job->kind = $kind;
|
|
$job->execution_mode = 'agent';
|
|
$job->status = 'queued';
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* Internal batch boundary. The batch service owns orchestration while this
|
|
* service remains the single authority for port allocation and Agent wire
|
|
* contracts.
|
|
*/
|
|
public static function reserveBatchPortLocked(
|
|
CloudInstance $instance,
|
|
CloudVpc $vpc,
|
|
string $batchKey,
|
|
int $revision
|
|
): CloudVpcPort {
|
|
self::assertOwnerAndInstance($instance, [
|
|
'user_id' => (int) $vpc->user_id,
|
|
], true);
|
|
self::assertPublicNetworkSnapshot($instance);
|
|
if (self::instanceRegionId($instance) !== (int) $vpc->region_id) {
|
|
throw new CloudError('实例只能加入同一地区的 VPC', 409);
|
|
}
|
|
self::assertNoInstanceJobLocked($instance);
|
|
if (CloudVpcPort::query()->where('instance_id', (int) $instance->id)
|
|
->where('active_claim', 1)->lockForUpdate()->exists()) {
|
|
throw new CloudError('实例已连接 VPC 或正在执行连接任务', 409);
|
|
}
|
|
$now = time();
|
|
$port = new CloudVpcPort();
|
|
$port->port_key = substr(hash('sha256', "vpc-batch-port\0"
|
|
. $batchKey . "\0" . (int) $instance->id), 0, 32);
|
|
$port->vpc_id = (int) $vpc->id;
|
|
$port->instance_id = (int) $instance->id;
|
|
$port->user_id = (int) $vpc->user_id;
|
|
$port->private_ip = self::allocateIpLocked($vpc);
|
|
$port->mac_address = self::allocateMacLocked((string) $port->port_key);
|
|
$port->active_claim = 1;
|
|
$port->state = 'preparing';
|
|
$port->desired_revision = $revision;
|
|
$port->applied_revision = 0;
|
|
$port->last_operation_id = null;
|
|
$port->last_error_code = '';
|
|
$port->attached_at = null;
|
|
$port->released_at = null;
|
|
$port->created_at = $port->updated_at = $now;
|
|
$port->save();
|
|
return $port;
|
|
}
|
|
|
|
public static function assertBatchInstanceLocked(
|
|
CloudInstance $instance,
|
|
array $actor,
|
|
bool $requirePublicNetwork = false
|
|
): void {
|
|
self::assertOwnerAndInstance($instance, $actor, true);
|
|
if ($requirePublicNetwork) {
|
|
self::assertPublicNetworkSnapshot($instance);
|
|
}
|
|
self::assertNoInstanceJobLocked($instance);
|
|
}
|
|
|
|
public static function batchInstanceRegionId(CloudInstance $instance): int
|
|
{
|
|
return self::instanceRegionId($instance);
|
|
}
|
|
|
|
public static function assertBatchVpcSettledLocked(CloudVpc $vpc): void
|
|
{
|
|
self::assertVpcSettledLocked($vpc);
|
|
}
|
|
|
|
/** @return list<CloudJob> */
|
|
public static function queueBatchProjectionLocked(
|
|
CloudVpc $vpc,
|
|
CloudVpcPort $focusPort,
|
|
int $revision,
|
|
int $batchId,
|
|
?string $provenNotAppliedOperationId = null
|
|
): array {
|
|
return self::queueProjectionLocked($vpc, $focusPort, $revision, [
|
|
'vpc_batch_id' => $batchId,
|
|
'vpc_batch_revision' => $revision,
|
|
], $provenNotAppliedOperationId);
|
|
}
|
|
|
|
public static function batchProjectionRequestPrefix(
|
|
int $batchId,
|
|
int $revision
|
|
): string {
|
|
if ($batchId < 1 || $revision < 1) {
|
|
throw new \InvalidArgumentException(
|
|
'VPC batch projection identity must be positive'
|
|
);
|
|
}
|
|
return 'vpc-batch-route-' . $batchId . '-' . $revision . '-';
|
|
}
|
|
|
|
private static function isKnownNotAppliedHostState(
|
|
CloudVpcHostState $state,
|
|
?string $provenNotAppliedOperationId
|
|
): bool {
|
|
if ((string) $state->sync_state === 'failed'
|
|
&& $provenNotAppliedOperationId !== null
|
|
&& preg_match('/^[a-f0-9]{32}$/D',
|
|
$provenNotAppliedOperationId) === 1
|
|
&& hash_equals(
|
|
(string) $state->last_operation_id,
|
|
$provenNotAppliedOperationId
|
|
)) {
|
|
return true;
|
|
}
|
|
return self::knownNotAppliedHostFailureLocked($state) !== null;
|
|
}
|
|
|
|
public static function queueBatchPortJobLocked(
|
|
string $kind,
|
|
CloudInstance $instance,
|
|
CloudVpc $vpc,
|
|
CloudVpcPort $port,
|
|
CloudVpcHostState $state,
|
|
int $batchId
|
|
): CloudJob {
|
|
return self::queuePortJobLocked(
|
|
$kind, $instance, $vpc, $port, $state, $batchId
|
|
);
|
|
}
|
|
|
|
/** Update only the host projection ledger; batch advancement stays outside. */
|
|
public static function settleBatchHostJobLocked(
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $operation,
|
|
CloudVpc $vpc
|
|
): string {
|
|
$state = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)
|
|
->where('host_id', (int) $job->host_id)
|
|
->lockForUpdate()->first();
|
|
if ($state === null
|
|
|| !hash_equals((string) $state->last_operation_id,
|
|
(string) $job->operation_id)) {
|
|
throw new CloudError('VPC 批次宿主机结果与当前投影不一致', 409);
|
|
}
|
|
$now = time();
|
|
if ($status === 'succeeded') {
|
|
self::assertHostResult($job, $vpc, $state, $result);
|
|
if ((string) $job->kind === 'vpc_host_remove') {
|
|
$state->delete();
|
|
} else {
|
|
$state->applied_revision = (int) $state->desired_revision;
|
|
$state->applied_digest = (string) $state->desired_digest;
|
|
$state->sync_state = 'ready';
|
|
$state->last_error_code = '';
|
|
$state->last_success_at = $now;
|
|
$state->updated_at = $now;
|
|
$state->save();
|
|
}
|
|
return '';
|
|
}
|
|
$uncertain = $status === 'uncertain';
|
|
$code = $uncertain ? 'vpc_result_uncertain'
|
|
: (CloudOperationErrorCatalog::workerSummary(
|
|
(string) $job->kind, 'failed', $operation
|
|
) ?? 'vpc_host_apply_failed');
|
|
$state->sync_state = $uncertain ? 'unknown' : 'failed';
|
|
$state->last_error_code = substr($code, 0, 64);
|
|
$state->last_failure_at = $now;
|
|
$state->updated_at = $now;
|
|
$state->save();
|
|
return substr($code, 0, 64);
|
|
}
|
|
|
|
public static function assertBatchPortResult(
|
|
CloudJob $job,
|
|
CloudVpc $vpc,
|
|
CloudVpcPort $port,
|
|
CloudInstance $instance,
|
|
$result
|
|
): void {
|
|
self::assertPortResult($job, $vpc, $port, $instance, $result);
|
|
}
|
|
|
|
public static function batchHostReady(?CloudVpcHostState $state): bool
|
|
{
|
|
return self::hostReady($state);
|
|
}
|
|
|
|
/**
|
|
* Continue panel orchestration after a super administrator proves that an
|
|
* empty, disabled compute node has been permanently retired. The caller
|
|
* has already removed the exact host-state row and resolved every unknown
|
|
* predecessor generation; no Agent request is made here.
|
|
*/
|
|
public static function finishRetiredHostRemovalLocked(
|
|
CloudJob $job,
|
|
CloudVpc $vpc,
|
|
CloudVpcPort $port,
|
|
CloudInstance $instance
|
|
): void {
|
|
if ((string) $job->kind !== 'vpc_host_remove'
|
|
|| (int) $job->vpc_id !== (int) $vpc->id
|
|
|| (int) $job->vpc_port_id !== (int) $port->id
|
|
|| (int) $job->instance_id !== (int) $instance->id
|
|
|| CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)
|
|
->where('host_id', (int) $job->host_id)->exists()) {
|
|
throw new CloudError('VPC 宿主机退役收尾账本不一致', 409);
|
|
}
|
|
if ($job->vpc_batch_id !== null) {
|
|
CloudVpcMembershipBatchService::finishRetiredHostRemovalLocked(
|
|
$job, $vpc
|
|
);
|
|
return;
|
|
}
|
|
self::advancePortLocked($instance, $vpc, $port);
|
|
}
|
|
|
|
private static function sealJob(CloudJob $job, CloudVpc $vpc,
|
|
CloudVpcPort $port, string $method, string $target, array $body,
|
|
array $extra = []): void
|
|
{
|
|
$payload = [
|
|
'method' => $method,
|
|
'request_target' => $target,
|
|
'body' => $body,
|
|
'owner_id' => (int) $port->user_id,
|
|
'can_manage' => true,
|
|
'system_vpc_orchestration' => true,
|
|
'vpc_id' => (int) $vpc->id,
|
|
'vpc_port_id' => (int) $port->id,
|
|
] + $extra;
|
|
$job->request_fingerprint = hash('sha256', json_encode([
|
|
'method' => $method, 'target' => $target, 'body' => $body,
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE));
|
|
$job->sealed_payload = CloudSecrets::encrypt($payload, 'job-payload');
|
|
$job->available_at = $job->created_at = $job->updated_at = time();
|
|
$job->save();
|
|
}
|
|
|
|
private static function hostApplyBody(CloudJob $job, CloudVpc $vpc,
|
|
CloudVpcHostState $state, array $payload): array
|
|
{
|
|
$network = $payload['host_projection'] ?? null;
|
|
if (!is_array($network) || array_is_list($network)
|
|
|| !hash_equals((string) $state->desired_digest,
|
|
CloudVpcProjection::digest($network))) {
|
|
throw new CloudError('VPC 宿主机投影摘要不一致', 409);
|
|
}
|
|
return ['operation_id' => (string) $job->operation_id,
|
|
'desired_digest' => (string) $state->desired_digest,
|
|
'network' => $network];
|
|
}
|
|
|
|
private static function hostRemoveBody(CloudJob $job, CloudVpc $vpc,
|
|
CloudVpcHostState $state): array
|
|
{
|
|
return ['operation_id' => (string) $job->operation_id,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'expected_revision' => (int) $state->applied_revision,
|
|
'expected_digest' => (string) $state->applied_digest];
|
|
}
|
|
|
|
private static function portBody(CloudJob $job, CloudVpc $vpc,
|
|
CloudVpcPort $port, CloudVpcHostState $state): array
|
|
{
|
|
return ['operation_id' => (string) $job->operation_id,
|
|
'port_key' => (string) $port->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'private_ip' => (string) $port->private_ip,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
'mac' => (string) $port->mac_address,
|
|
'host_revision' => (int) $state->applied_revision,
|
|
'host_digest' => (string) $state->applied_digest,
|
|
'confirm_restart' => true];
|
|
}
|
|
|
|
private static function assertHostResult(CloudJob $job, CloudVpc $vpc,
|
|
CloudVpcHostState $state, $result): void
|
|
{
|
|
if (!is_array($result) || array_is_list($result)
|
|
|| ($result['vpc_key'] ?? null) !== (string) $vpc->vpc_key
|
|
|| (int) ($result['desired_revision'] ?? 0)
|
|
!== (int) (($job->kind === 'vpc_host_apply')
|
|
? $state->desired_revision : $state->applied_revision)
|
|
|| !hash_equals((string) (($job->kind === 'vpc_host_apply')
|
|
? $state->desired_digest : $state->applied_digest),
|
|
(string) ($result['desired_digest'] ?? ''))
|
|
|| ($job->kind === 'vpc_host_apply'
|
|
&& (($result['applied'] ?? null) !== true
|
|
|| ($result['removed'] ?? false) === true
|
|
|| ($result['bridge'] ?? null)
|
|
!== CloudVpcProjection::bridgeName(
|
|
(string) $vpc->vpc_key)))
|
|
|| ($job->kind === 'vpc_host_remove'
|
|
&& (($result['removed'] ?? null) !== true
|
|
|| ($result['applied'] ?? false) === true))) {
|
|
throw new CloudError('Agent 成功结果缺少准确的 VPC 路由证据', 502);
|
|
}
|
|
}
|
|
|
|
private static function assertPortResult(CloudJob $job, CloudVpc $vpc,
|
|
CloudVpcPort $port, CloudInstance $instance, $result): void
|
|
{
|
|
if (!is_array($result) || array_is_list($result)
|
|
|| strtolower((string) ($result['uuid'] ?? ''))
|
|
!== strtolower((string) $instance->uuid)
|
|
|| ($result['port_key'] ?? null) !== (string) $port->port_key
|
|
|| ($result['vpc_key'] ?? null) !== (string) $vpc->vpc_key) {
|
|
throw new CloudError('Agent 成功结果缺少准确的 VPC 网卡身份', 502);
|
|
}
|
|
if ((string) $job->kind === 'vpc_port_detach') {
|
|
if (($result['removed'] ?? null) !== true
|
|
|| ($result['attached'] ?? false) === true
|
|
|| array_key_exists('vpc_port', $result)) {
|
|
throw new CloudError('Agent 未确认 VPC 网卡已卸载', 502);
|
|
}
|
|
return;
|
|
}
|
|
$state = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)
|
|
->where('host_id', (int) $instance->host_id)->first();
|
|
$metadata = $result['vpc_port'] ?? null;
|
|
if (!self::hostReady($state) || ($result['attached'] ?? null) !== true
|
|
|| ($result['removed'] ?? false) === true
|
|
|| !is_array($metadata) || array_is_list($metadata)
|
|
|| !self::sameDocument($metadata, [
|
|
'port_key' => (string) $port->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'private_ip' => (string) $port->private_ip,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
'mac' => (string) $port->mac_address,
|
|
'bridge' => CloudVpcProjection::bridgeName(
|
|
(string) $vpc->vpc_key),
|
|
'host_revision' => (int) $state->applied_revision,
|
|
'host_digest' => (string) $state->applied_digest,
|
|
])) {
|
|
throw new CloudError('Agent 未确认准确的 VPC 网卡配置', 502);
|
|
}
|
|
}
|
|
|
|
private static function releasePortLocked(CloudVpc $vpc,
|
|
CloudVpcPort $port): void
|
|
{
|
|
$now = time();
|
|
$port->state = 'released';
|
|
$port->active_claim = null;
|
|
$port->last_operation_id = null;
|
|
$port->last_error_code = '';
|
|
$port->released_at = $now;
|
|
$port->updated_at = $now;
|
|
$port->save();
|
|
$vpc->state = 'active';
|
|
$vpc->updated_at = $now;
|
|
$vpc->save();
|
|
}
|
|
|
|
private static function assertVpcSettledLocked(CloudVpc $vpc): void
|
|
{
|
|
if ((string) $vpc->state !== 'active') {
|
|
throw new CloudError('该 VPC 尚未收敛到可变更状态', 409);
|
|
}
|
|
$ports = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
|
|
->where('active_claim', 1)->orderBy('id')->lockForUpdate()->get();
|
|
if ($ports->contains(static fn ($port): bool =>
|
|
(string) $port->state !== 'attached'
|
|
|| $port->migration_host_id !== null)) {
|
|
throw new CloudError('该 VPC 已有端口变更任务,请等待完成', 409);
|
|
}
|
|
$states = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $vpc->id)->orderBy('host_id')
|
|
->lockForUpdate()->get();
|
|
if ($states->contains(static fn ($state): bool =>
|
|
!self::hostReady($state))) {
|
|
throw new CloudError('该 VPC 的宿主机路由尚未收敛', 409);
|
|
}
|
|
}
|
|
|
|
private static function assertNoInstanceJobLocked(
|
|
CloudInstance $instance): void
|
|
{
|
|
$jobs = CloudJobService::blockingQueryForInstance((int) $instance->id)
|
|
->orderBy('id')->lockForUpdate()->get();
|
|
if (!$jobs->isEmpty()) {
|
|
throw new CloudError('虚拟机已有任务,请等待完成', 409);
|
|
}
|
|
}
|
|
|
|
private static function activePortJob(int $instanceId,
|
|
int $portId): ?CloudJob
|
|
{
|
|
return CloudJobService::blockingQueryForInstance($instanceId)
|
|
->where('vpc_port_id', $portId)
|
|
->whereIn('kind', self::JOB_KINDS)
|
|
->orderByDesc('id')->first();
|
|
}
|
|
|
|
private static function activePortJobLocked(CloudInstance $instance,
|
|
CloudVpcPort $port): ?CloudJob
|
|
{
|
|
return CloudJobService::blockingQueryForInstance((int) $instance->id)
|
|
->where('vpc_port_id', (int) $port->id)
|
|
->whereIn('kind', self::JOB_KINDS)
|
|
->orderByDesc('id')->lockForUpdate()->first();
|
|
}
|
|
|
|
private static function assertOwnerAndInstance(?CloudInstance $instance,
|
|
array $actor, bool $mutation): void
|
|
{
|
|
if ($instance === null || (int) $instance->user_id
|
|
!== (int) ($actor['user_id'] ?? 0)
|
|
|| (string) $instance->status === 'deleted') {
|
|
throw new CloudError('虚拟机不存在或无权访问', 404);
|
|
}
|
|
if ($mutation && ((string) $instance->status !== 'active'
|
|
|| !$instance->serviceAvailable())) {
|
|
throw new CloudError('实例服务已到期或正在执行策略,不能修改 VPC', 409);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A new private NIC may only be attached to an instance whose immutable
|
|
* dual-stack public identity still passes the canonical snapshot parser.
|
|
* Detach deliberately does not call this guard: damaged metadata must
|
|
* never trap an already-attached VPC port or its private address claim.
|
|
*/
|
|
private static function assertPublicNetworkSnapshot(
|
|
CloudInstance $instance
|
|
): void {
|
|
$decoded = json_decode((string) $instance->spec_json, true);
|
|
$spec = CloudInstanceMetadata::publicSpec(
|
|
is_array($decoded) && !array_is_list($decoded) ? $decoded : []
|
|
);
|
|
if (($spec['os_source'] ?? null) !== 'cloud_template') {
|
|
throw new CloudError('当前实例不支持 VPC 私网网卡', 409);
|
|
}
|
|
if (($spec['public_network_valid'] ?? false) !== true) {
|
|
throw new CloudError(
|
|
'实例公网配置异常,暂不能修改 VPC 私网',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function instanceRegionId(CloudInstance $instance): int
|
|
{
|
|
if ($instance->resource_pool_id === null) {
|
|
throw new CloudError('实例缺少地区资源池账本', 409);
|
|
}
|
|
$regionId = CloudResourcePool::query()
|
|
->join('cloud_network_lines as line', 'line.id', '=',
|
|
'cloud_resource_pools.line_id')
|
|
->where('cloud_resource_pools.id', (int) $instance->resource_pool_id)
|
|
->value('line.region_id');
|
|
if ($regionId === null || (int) $regionId < 1) {
|
|
throw new CloudError('实例地区账本无效', 409);
|
|
}
|
|
return (int) $regionId;
|
|
}
|
|
|
|
/** @return array<int,array> */
|
|
private static function readyTransports(array $hostIds): array
|
|
{
|
|
if ($hostIds === []) {
|
|
return [];
|
|
}
|
|
sort($hostIds, SORT_NUMERIC);
|
|
$enabled = CloudHost::query()->whereIn('id', $hostIds)
|
|
->where('enabled', 1)->pluck('id')->map(
|
|
static fn ($id): int => (int) $id
|
|
)->all();
|
|
sort($enabled, SORT_NUMERIC);
|
|
if ($enabled !== $hostIds) {
|
|
throw new CloudError('VPC 包含不可用的计算节点', 409);
|
|
}
|
|
$rows = CloudHostVpcTransport::query()->whereIn('host_id', $hostIds)
|
|
->where('sync_state', 'ready')
|
|
->whereColumn('observed_revision', 'desired_revision')
|
|
->orderBy('host_id')->get()->keyBy('host_id');
|
|
if ($rows->count() !== count($hostIds)) {
|
|
throw new CloudError('VPC 计算节点私网传输尚未验证就绪', 409);
|
|
}
|
|
$result = [];
|
|
foreach ($hostIds as $hostId) {
|
|
$row = $rows->get($hostId);
|
|
$result[$hostId] = [
|
|
'transport_address' => (string) $row->transport_address,
|
|
'transport_prefix_length' => (int) $row->transport_prefix_length,
|
|
'transport_interface' => (string) $row->transport_interface,
|
|
'mtu' => (int) $row->mtu,
|
|
];
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
private static function allocateIpLocked(CloudVpc $vpc): string
|
|
{
|
|
[$network, $prefixText] = explode('/', (string) $vpc->cidr, 2);
|
|
$prefix = (int) $prefixText;
|
|
$base = self::ipv4Number($network);
|
|
$last = $base + (1 << (32 - $prefix)) - 1;
|
|
$used = CloudVpcPort::query()->where('vpc_id', (int) $vpc->id)
|
|
->where('active_claim', 1)->orderBy('private_ip')
|
|
->lockForUpdate()->pluck('private_ip')->mapWithKeys(
|
|
static fn ($ip): array => [(string) $ip => true]
|
|
)->all();
|
|
for ($number = $base + 2; $number < $last; $number++) {
|
|
$candidate = long2ip($number);
|
|
if (!isset($used[$candidate])) {
|
|
return $candidate;
|
|
}
|
|
}
|
|
throw new CloudError('该 VPC 的私网地址已用尽', 409);
|
|
}
|
|
|
|
private static function allocateMacLocked(string $portKey): string
|
|
{
|
|
for ($attempt = 0; $attempt < 256; $attempt++) {
|
|
$digest = hash('sha256', 'virt-cloud-vpc-port-mac:' . $portKey
|
|
. ':' . $attempt, true);
|
|
$bytes = [2, ord($digest[0]), ord($digest[1]), ord($digest[2]),
|
|
ord($digest[3]), ord($digest[4])];
|
|
$mac = implode(':', array_map(static fn (int $byte): string =>
|
|
str_pad(dechex($byte), 2, '0', STR_PAD_LEFT), $bytes));
|
|
if (!CloudVpcPort::query()->where('mac_address', $mac)
|
|
->lockForUpdate()->exists()) {
|
|
return $mac;
|
|
}
|
|
}
|
|
throw new CloudError('无法分配唯一的 VPC 网卡地址', 503);
|
|
}
|
|
|
|
private static function requestKey(array $actor): string
|
|
{
|
|
$key = $actor['request_idempotency_key'] ?? null;
|
|
if (!is_string($key)
|
|
|| preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) {
|
|
throw new CloudError('请求幂等标识无效', 400);
|
|
}
|
|
return $key;
|
|
}
|
|
|
|
private static function portKey(int $userId, int $instanceId,
|
|
string $vpcKey, string $requestKey): string
|
|
{
|
|
return substr(hash('sha256', "vpc-port\0" . $userId . "\0"
|
|
. $instanceId . "\0" . $vpcKey . "\0" . $requestKey), 0, 32);
|
|
}
|
|
|
|
private static function hostReady(?CloudVpcHostState $state): bool
|
|
{
|
|
return $state !== null && (string) $state->sync_state === 'ready'
|
|
&& (int) $state->desired_revision > 0
|
|
&& (int) $state->desired_revision === (int) $state->applied_revision
|
|
&& is_string($state->desired_digest)
|
|
&& is_string($state->applied_digest)
|
|
&& hash_equals((string) $state->desired_digest,
|
|
(string) $state->applied_digest);
|
|
}
|
|
|
|
private static function portDto(CloudVpcPort $port,
|
|
?CloudVpc $vpc = null): array
|
|
{
|
|
$vpc ??= CloudVpc::find((int) $port->vpc_id);
|
|
return [
|
|
'port_key' => (string) $port->port_key,
|
|
'vpc_key' => $vpc === null ? null : (string) $vpc->vpc_key,
|
|
'vpc_name' => $vpc === null ? null : (string) $vpc->name,
|
|
'private_ip' => (string) $port->private_ip,
|
|
'mac_address' => (string) $port->mac_address,
|
|
'state' => (string) $port->state,
|
|
'desired_revision' => (int) $port->desired_revision,
|
|
'applied_revision' => (int) $port->applied_revision,
|
|
'last_error_code' => CloudOperationErrorCatalog::publicCode(
|
|
(string) $port->last_error_code
|
|
),
|
|
'attached_at' => $port->attached_at === null
|
|
? null : (int) $port->attached_at,
|
|
];
|
|
}
|
|
|
|
private static function sameDocument(array $left, array $right): bool
|
|
{
|
|
return hash_equals(self::documentDigest($left),
|
|
self::documentDigest($right));
|
|
}
|
|
|
|
private static function documentDigest(array $value): string
|
|
{
|
|
$canonical = static function ($item) use (&$canonical) {
|
|
if (!is_array($item)) {
|
|
return $item;
|
|
}
|
|
if (!array_is_list($item)) {
|
|
ksort($item, SORT_STRING);
|
|
}
|
|
foreach ($item as $key => $child) {
|
|
$item[$key] = $canonical($child);
|
|
}
|
|
return $item;
|
|
};
|
|
return hash('sha256', json_encode($canonical($value),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE));
|
|
}
|
|
|
|
private static function ipv4Number(string $address): int
|
|
{
|
|
$packed = inet_pton($address);
|
|
if ($packed === false || strlen($packed) !== 4) {
|
|
throw new CloudError('VPC IPv4 地址账本无效', 503);
|
|
}
|
|
return (int) unpack('Naddress', $packed)['address'];
|
|
}
|
|
}
|