You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudInstanceMigrationServi...

2844 lines
114 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudBackupRestore;
use App\Models\CloudInstance;
use App\Models\CloudInstanceFirewall;
use App\Models\CloudInstanceMigration;
use App\Models\CloudJob;
use App\Models\CloudResourceReservation;
/** Durable cold migration coordinated over the PeerMigration V3 data plane. */
final class CloudInstanceMigrationService
{
private const GIB = 1073741824;
private const PROTOCOL_VERSION = 3;
private const CHUNK_SIZE = 8388608;
private const TICKET_LIFETIME_SECONDS = 604800;
/**
* Queue one plan-bound cleanup for residue left by an interrupted target
* receive. The original receive remains fenced and is never replayed; its
* ordinary automatic recovery will prove absence and repair every PHP
* placement/migration ledger after this successor succeeds.
*/
public static function ensureTargetResidueCleanup(
CloudJob $receiveJob,
array $inspection
): bool {
$cleanup = $inspection['cleanup'] ?? null;
$planId = is_array($cleanup) ? ($cleanup['plan_id'] ?? null) : null;
$steps = is_array($cleanup) ? ($cleanup['steps'] ?? null) : null;
$allowedSteps = [
'delete_managed_volumes', 'delete_seed_volume',
'undefine_network_filter', 'delete_nvram',
'remove_traffic_state',
];
if ((string) $receiveJob->kind !== 'instance_migration_transfer'
|| ($cleanup['available'] ?? false) !== true
|| !is_string($planId)
|| preg_match('/^[a-f0-9]{64}$/D', strtolower($planId)) !== 1
|| !is_array($steps) || !array_is_list($steps) || $steps === []
|| array_diff($steps, $allowedSteps) !== []
|| count(array_unique($steps)) !== count($steps)) {
return false;
}
return $receiveJob->getConnection()->transaction(
static function () use ($receiveJob, $planId, $steps): bool {
$lockedJob = CloudJob::where(
'id', (int) $receiveJob->id
)->lockForUpdate()->first();
$migration = CloudInstanceMigration::where(
'target_receive_job_id', (int) $receiveJob->id
)->lockForUpdate()->first();
$instance = $migration === null ? null : CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
$restore = $migration === null ? null : CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
if ($lockedJob === null || $migration === null
|| $instance === null || $restore === null
|| (string) $lockedJob->kind
!== 'instance_migration_transfer'
|| (string) $lockedJob->phase !== 'terminal'
|| (string) $lockedJob->outcome !== 'failed'
|| (string) $lockedJob->effect_resolution !== 'unknown'
|| (int) $lockedJob->effect_may_have_applied !== 1
|| (int) $lockedJob->safe_to_resubmit !== 0
|| (int) $lockedJob->host_id
!== (int) $migration->target_host_id
|| (int) $lockedJob->instance_id !== (int) $instance->id
|| (int) $lockedJob->backup_restore_id
!== (int) $restore->id
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| !in_array((string) $restore->state,
['restoring', 'uncertain'], true)
|| !hash_equals((string) $lockedJob->operation_id,
(string) $restore->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标残留清理账本不一致', 409
);
}
$priorCleanups = CloudJob::where(
'parent_job_id', (int) $lockedJob->id
)->where('kind', 'instance_migration_target_clean')
->orderBy('id')->lockForUpdate()->get();
foreach ($priorCleanups as $priorCleanup) {
if ((string) $priorCleanup->phase !== 'terminal'
|| (string) $priorCleanup->outcome === 'succeeded'
|| (string) $priorCleanup->effect_resolution
!== 'not_applied') {
return true;
}
}
if (count($priorCleanups) >= 3) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标残留自动清理已达到安全重试上限',
409
);
}
$generation = count($priorCleanups) + 1;
$target = '/v1/vms/'
. rawurlencode((string) $migration->target_uuid)
. '/peer-migration/cleanup-target';
$body = [
'migration_id' => (string) $migration->migration_key,
'prior_operation_id' =>
(string) $lockedJob->operation_id,
'cleanup_plan_id' => strtolower($planId),
];
[$cleanupJob, $created] = CloudJobService::
reserveStandaloneAgentMutation(
[
'user_id' => (int) $lockedJob->user_id,
'request_idempotency_key' =>
'migration-target-clean-'
. (string) $migration->migration_key
. '-' . $generation,
],
(int) $migration->target_host_id,
'instance_migration_target_clean',
'POST',
$target,
$body,
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'maintenance_migration_id' =>
(int) $migration->id,
'prior_job_id' => (int) $lockedJob->id,
'cleanup_plan_id' => strtolower($planId),
'cleanup_steps' => array_values($steps),
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'prior_job_id' => (int) $lockedJob->id,
'prior_operation_id' =>
(string) $lockedJob->operation_id,
'cleanup_plan_id' => strtolower($planId),
'cleanup_steps' => array_values($steps),
],
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id
. '/target-residue-cleanup/' . $generation
);
if (!$created) {
return true;
}
$cleanupJob->parent_job_id = (int) $lockedJob->id;
$cleanupJob->instance_id = (int) $instance->id;
$cleanupJob->backup_restore_id =
(int) $migration->restore_id;
$cleanupJob->save();
self::setState(
$migration,
'attention_required',
'target_residue_cleanup_running',
'reconcile_transfer'
);
return true;
},
3
);
}
public static function start(
CloudInstance $instance,
array $actor,
array $input
): array {
CloudAccess::manage($actor);
CloudInput::only($input, [
'target_host_id', 'target_resource_pool_id', 'cleanup_mode',
'confirm_maintenance_migration',
]);
$targetHostId = filter_var($input['target_host_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$targetResourcePoolId = filter_var(
$input['target_resource_pool_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$cleanupMode = $input['cleanup_mode'] ?? null;
if ($targetHostId === false || $targetResourcePoolId === false
|| !in_array($cleanupMode, ['retain', 'delete'], true)) {
throw new CloudError(
'请选择目标宿主机、资源池和源副本处理方式'
);
}
if (($input['confirm_maintenance_migration'] ?? false) !== true) {
throw new CloudError(
'请确认维护迁移会关闭源实例并更换公网 IPv4 和 IPv6'
);
}
$requestKey = $actor['request_idempotency_key'] ?? null;
if (!is_string($requestKey) || $requestKey === '') {
throw new CloudError('维护迁移必须提供请求幂等标识', 400);
}
$sourceHostId = (int) $instance->host_id;
$target = '/v1/vms/' . rawurlencode((string) $instance->uuid)
. '/peer-migration/prepare';
$canonical = [
'instance_id' => (int) $instance->id,
'target_host_id' => (int) $targetHostId,
'target_resource_pool_id' => (int) $targetResourcePoolId,
'cleanup_mode' => (string) $cleanupMode,
'confirm_maintenance_migration' => true,
];
$fingerprint = CloudJobService::standaloneRequestFingerprint(
$sourceHostId,
'instance_migration_prepare',
'POST',
$target,
$canonical
);
$prior = CloudJob::where('user_id', (int) $actor['user_id'])
->where('request_idempotency_key', $requestKey)->first();
if ($prior !== null) {
if ((string) $prior->kind !== 'instance_migration_prepare'
|| (int) $prior->instance_id !== (int) $instance->id
|| !hash_equals((string) $prior->request_fingerprint,
$fingerprint)
|| !self::ownsPrepareJob((int) $prior->id)) {
throw new CloudError('同一请求标识不能用于不同的任务', 409);
}
return ['job' => $prior];
}
$sourceCapability = self::peerCapability($sourceHostId, $actor);
$targetCapability = self::peerCapability((int) $targetHostId, $actor);
if (hash_equals($sourceCapability['identity_sha256'],
$targetCapability['identity_sha256'])) {
throw new CloudError('两台宿主机不能共用同一个 PeerMigration 自动身份', 409);
}
return $instance->getConnection()->transaction(function () use (
$instance,
$actor,
$targetHostId,
$targetResourcePoolId,
$cleanupMode,
$sourceCapability,
$target,
$canonical,
$fingerprint
): array {
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->first();
if ($locked === null || (string) $locked->status !== 'active') {
throw new CloudError('只有活动实例可以维护迁移', 409);
}
if ((int) $locked->host_id !== (int) $instance->host_id
|| !hash_equals(strtolower((string) $locked->uuid),
strtolower((string) $instance->uuid))) {
throw new CloudError('实例位置已经变化,请刷新后重试', 409);
}
$prior = CloudJob::where('user_id', (int) $actor['user_id'])
->where('request_idempotency_key',
(string) $actor['request_idempotency_key'])
->lockForUpdate()->first();
if ($prior !== null) {
if ((string) $prior->kind !== 'instance_migration_prepare'
|| (int) $prior->instance_id !== (int) $locked->id
|| !hash_equals((string) $prior->request_fingerprint,
$fingerprint)
|| !self::ownsPrepareJob((int) $prior->id)) {
throw new CloudError('同一请求标识不能用于不同的任务', 409);
}
return ['job' => $prior];
}
$spec = self::decodeSpec((string) $locked->spec_json);
if (($spec['managed'] ?? null) !== true) {
throw new CloudError('只有面板托管的实例可以维护迁移', 409);
}
if (CloudJobService::blockingQueryForInstance(
(int) $locked->id
)->lockForUpdate()->exists()) {
throw new CloudError('实例已有未完成任务', 409);
}
$snapshots = CloudPeerMigrationArtifactService::
freezeSnapshotsLocked($locked);
$resources = self::resources($spec);
self::assertStartAvailable(
$locked,
(int) $targetHostId,
(int) $targetResourcePoolId,
$resources,
CloudPeerMigrationArtifactService::additionalDiskGib($snapshots)
);
$vpcPortId = CloudVpcPortService::migrationPortIdLocked(
$locked,
(int) $targetHostId
);
$migrationKey = bin2hex(random_bytes(16));
$targetUuid = self::uuidV4();
$restore = new CloudBackupRestore();
$restore->restore_key = bin2hex(random_bytes(16));
$restore->backup_id = null;
$restore->target_host_id = (int) $targetHostId;
$restore->target_instance_id = null;
$restore->reservation_id = null;
$restore->job_id = null;
$restore->operation_id = bin2hex(random_bytes(16));
$restore->new_uuid = $targetUuid;
$restore->new_name = (string) $locked->name;
$restore->state = 'reserved';
$restore->error_code = '';
$restore->ready_at = null;
$restore->created_at = $restore->updated_at = time();
$restore->save();
$placement = CloudPlacementService::
reserveMaintenanceMigrationTarget(
$restore,
(int) $locked->user_id,
$resources,
(int) $targetResourcePoolId,
CloudPeerMigrationArtifactService::additionalDiskGib($snapshots)
);
$sourceReservation = CloudResourceReservation::where(
'id',
(int) $locked->reservation_id
)->lockForUpdate()->first();
if ($sourceReservation === null
|| (string) $sourceReservation->state !== 'committed'
|| (int) $sourceReservation->host_id !== (int) $locked->host_id
|| (int) $sourceReservation->instance_id !== (int) $locked->id
|| (int) $sourceReservation->resource_pool_id
!== (int) $locked->resource_pool_id
|| (int) ($sourceReservation->order_id ?? 0)
!== (int) $locked->order_id) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源资源预留与实例所属资源池不一致',
409
);
}
CloudPublicNetworkAllocationService::currentClaimSet(
(int) $sourceReservation->id,
'committed',
(int) $locked->id
);
$reservation = $placement['reservation'];
$restore->reservation_id = (int) $reservation->id;
$restore->updated_at = time();
$restore->save();
$migration = new CloudInstanceMigration();
$migration->migration_key = $migrationKey;
$migration->instance_id = (int) $locked->id;
$migration->vpc_port_id = $vpcPortId;
$migration->restore_id = (int) $restore->id;
$migration->source_host_id = (int) $locked->host_id;
$migration->source_uuid = strtolower((string) $locked->uuid);
$migration->source_reservation_id = (int) $locked->reservation_id;
$migration->target_host_id = (int) $targetHostId;
$migration->target_uuid = $targetUuid;
$migration->target_reservation_id = (int) $reservation->id;
$migration->source_prepare_job_id = null;
$migration->target_receive_job_id = null;
$migration->firewall_job_id = null;
$migration->activate_job_id = null;
$migration->rollback_job_id = null;
$migration->cleanup_job_id = null;
$migration->cleanup_mode = (string) $cleanupMode;
$migration->state = 'preparing_source';
$migration->recovery_action = '';
$migration->retry_generation = 0;
$migration->source_endpoint = self::encodePeerEndpoints(
$sourceCapability['endpoints']
);
$migration->peer_trust_id =
$sourceCapability['identity_sha256'];
$migration->manifest_digest = '';
$migration->source_was_running = null;
$migration->source_autostart_enabled = null;
$migration->source_service_status = (string) $locked->service_status;
$migration->source_network_status = (string) $locked->network_status;
$migration->error_code = '';
$migration->cutover_at = null;
$migration->completed_at = null;
$migration->created_at = $migration->updated_at = time();
$migration->save();
CloudPeerMigrationArtifactService::persistLocked(
$migration,
$snapshots,
$resources['disk_gib'] * self::GIB
);
$ticket = bin2hex(random_bytes(32));
$expiresAt = time() + self::TICKET_LIFETIME_SECONDS;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
$actor,
(int) $locked->host_id,
'instance_migration_prepare',
'POST',
$target,
[
'migration_id' => $migrationKey,
'ticket' => $ticket,
'expires_at' => $expiresAt,
'artifacts' => CloudPeerMigrationArtifactService::
sourceRequestArtifacts($migration),
],
[
'owner_id' => (int) $locked->user_id,
'can_manage' => true,
'maintenance_migration_id' => (int) $migration->id,
],
$canonical,
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/prepare'
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'维护迁移源准备任务幂等冲突', 409
);
}
$job->instance_id = (int) $locked->id;
$job->save();
$migration->source_prepare_job_id = (int) $job->id;
$migration->updated_at = time();
$migration->save();
return ['job' => $job];
}, 3);
}
public static function assertStartAvailable(
CloudInstance $instance,
int $targetHostId,
int $targetResourcePoolId,
array $resources,
int $snapshotDiskGib = 0,
?int $excludeMigrationId = null
): void {
if (!in_array((string) $instance->service_status,
['active', 'exempt', 'expired_grace', 'suspended_expired'], true)
|| !in_array((string) $instance->network_status,
['active', 'blocked_traffic', 'blocked_expired'], true)) {
throw new CloudError('实例服务或网络状态正在变化,暂不能维护迁移', 409);
}
if ($targetHostId === (int) $instance->host_id) {
throw new CloudError('维护迁移必须选择其他宿主机', 409);
}
if ((int) ($instance->reservation_id ?? 0) < 1
|| (int) ($instance->order_id ?? 0) < 1
|| (int) ($instance->resource_pool_id ?? 0) < 1) {
throw new CloudError('只有具备完整订单和资源预留的实例可以维护迁移', 409);
}
if ($targetResourcePoolId !== (int) $instance->resource_pool_id) {
throw new CloudError('维护迁移必须保持原资源池身份', 409);
}
$query = CloudInstanceMigration::where(
'instance_id', (int) $instance->id
)->whereIn('state', array_merge(
CloudInstanceMigrationWorkflow::blockingStates(),
['source_retained']
));
if ($excludeMigrationId !== null) {
$query->where('id', '<>', $excludeMigrationId);
}
if ($query->exists()) {
throw new CloudError('实例已有未完成迁移或待清理源副本', 409);
}
$allowed = array_filter(
CloudPlacementService::maintenanceMigrationTargets(
$instance,
$resources,
$snapshotDiskGib
),
static fn (array $target): bool =>
(int) ($target['id'] ?? 0) === $targetHostId
&& (int) ($target['resource_pool_id'] ?? 0)
=== $targetResourcePoolId
);
if ($allowed === []) {
throw new CloudError(
'目标宿主机必须具备原资源池的活动成员关系、可用容量、已验证网络接入和公网 IP',
409
);
}
CloudFirewallService::assertMaintenanceMigrationReady($instance);
}
public static function display(CloudInstanceMigration $migration): array
{
$state = (string) $migration->state;
$action = (string) $migration->recovery_action;
$targetReservation = CloudResourceReservation::find(
(int) $migration->target_reservation_id
);
$hasAbortEvidence = self::hasAbortPrepareEvidence($migration);
$recoveryOptions = CloudInstanceMigrationWorkflow::recoveryOptions(
$migration,
$hasAbortEvidence
);
$display = [
'id' => (string) $migration->id,
'state' => $state,
'cleanup_mode' => (string) $migration->cleanup_mode,
'source_host_id' => (string) $migration->source_host_id,
'target_host_id' => (string) $migration->target_host_id,
'target_resource_pool_id' => $targetReservation === null
? null : (string) $targetReservation->resource_pool_id,
'source_retained' => $state === 'source_retained',
'can_cleanup' => $state === 'source_retained'
&& $action !== 'reconcile_cleanup',
'recovery_action' => $action === '' ? null : $action,
'can_recover' => $recoveryOptions !== [],
'recovery_actions' => array_column(
$recoveryOptions,
'action'
),
'recovery_options' => $recoveryOptions,
'progress' => CloudInstanceMigrationWorkflow::progress(
$migration
),
'error_code' => (string) $migration->error_code,
'cutover_at' => $migration->cutover_at === null
? null : (int) $migration->cutover_at,
'completed_at' => $migration->completed_at === null
? null : (int) $migration->completed_at,
'created_at' => (int) $migration->created_at,
'updated_at' => (int) $migration->updated_at,
];
foreach (CloudInstanceMigrationWorkflow::stages() as $stage) {
if ($stage['task_key'] === null) {
continue;
}
$field = (string) $stage['job_field'];
$display[(string) $stage['task_key']] =
$migration->{$field} === null
? null : (string) $migration->{$field};
}
return $display;
}
private static function hasAbortPrepareEvidence(
CloudInstanceMigration $migration
): bool {
try {
self::abortPrepareEvidence($migration);
return true;
} catch (\Throwable $error) {
return false;
}
}
/**
* @return array{job:CloudJob,event:CloudAgentCompletionEvent,result:array}
*/
private static function abortPrepareEvidence(
CloudInstanceMigration $migration
): array {
if ((string) $migration->state !== 'attention_required'
|| !in_array((string) $migration->recovery_action,
['reconcile_prepare', 'abort_prepare'], true)
|| $migration->target_receive_job_id !== null
|| $migration->firewall_job_id !== null
|| $migration->activate_job_id !== null
|| $migration->rollback_job_id !== null
|| $migration->cleanup_job_id !== null
|| $migration->cutover_at !== null) {
throw new CloudRecoveryInstanceStateConflict(
'当前迁移不处于可安全中止的源准备阶段', 409
);
}
$job = CloudJob::find((int) $migration->source_prepare_job_id);
if ($job === null
|| (string) $job->kind !== 'instance_migration_prepare'
|| (string) $job->execution_mode !== 'agent'
|| (int) $job->instance_id !== (int) $migration->instance_id
|| (int) $job->host_id !== (int) $migration->source_host_id
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (int) $job->dispatch_attempts !== 1
|| !CloudDispatchLedger::isDispatched($job)
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown') {
throw new CloudRecoveryInstanceStateConflict(
'源准备任务不处于可补偿的效果核对状态', 409
);
}
$event = CloudOperationCompletionService::pendingForJob($job);
if ($event === null
|| !CloudOperationCompletionService::isExactStoredEvent($event)
|| (string) $event->outcome !== 'succeeded'
|| (bool) $event->effect_may_have_applied
|| (bool) $event->safe_to_resubmit
|| ($job->completion_event_id !== null
&& !hash_equals(
(string) $job->completion_event_id,
(string) $event->event_id
))) {
throw new CloudRecoveryInstanceStateConflict(
'源准备任务尚无可安全补偿的精确成功回执', 409
);
}
CloudOperationCompletionService::assertEventIdentityForJob(
$job,
$event
);
$operation = CloudOperationCompletionService::operation($event);
$result = $operation['result'] ?? null;
self::assertAbortPrepareResult($migration, $result);
return ['job' => $job, 'event' => $event, 'result' => $result];
}
private static function assertAbortPrepareResult(
CloudInstanceMigration $migration,
$result
): void {
if (!is_array($result) || array_is_list($result)
|| !hash_equals(
(string) $migration->migration_key,
(string) ($result['migration_id'] ?? '')
)
|| ($result['source_stopped'] ?? null) !== true
|| ($result['source_autostart_disabled'] ?? null) !== true
|| !is_bool($result['source_was_running'] ?? null)
|| !is_bool($result['source_autostart_enabled'] ?? null)
|| !is_string($result['manifest_digest'] ?? null)
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $result['manifest_digest']) !== 1
|| !is_array($result['artifacts'] ?? null)
|| !array_is_list($result['artifacts'])
|| $result['artifacts'] === []) {
throw new CloudRecoveryReceiptInvalid(
'源准备成功回执缺少可安全恢复实例的证据', 409
);
}
}
public static function ownsPrepareJob(int $jobId): bool
{
return self::catalogAvailable()
&& CloudInstanceMigration::where(
'source_prepare_job_id', $jobId
)->exists();
}
public static function hasOpenMigration(int $instanceId): bool
{
return $instanceId > 0
&& self::catalogAvailable()
&& CloudInstanceMigration::where('instance_id', $instanceId)
->whereIn(
'state',
CloudInstanceMigrationWorkflow::blockingStates()
)->exists();
}
public static function assertInstanceMutationAvailable(
CloudInstance $instance,
?string $operationKind = null
): void {
if (self::hasOpenMigration((int) $instance->id)) {
throw new CloudError('实例正在维护迁移,请先等待迁移收敛', 409);
}
if ($operationKind !== null
&& in_array($operationKind, [
'delete', 'delete_force', 'expiry_delete',
], true)
&& self::catalogAvailable()
&& CloudInstanceMigration::where(
'instance_id', (int) $instance->id
)->where('state', 'source_retained')->exists()) {
throw new CloudError(
'迁移源副本尚未释放,请先完成迁移清理后再删除实例',
409
);
}
}
public static function finishJobLocked(
CloudJob $job,
string $status,
$result,
array $payload
): void {
if (!self::catalogAvailable()) {
return;
}
$stage = CloudInstanceMigrationWorkflow::job((string) $job->kind);
if ($stage === null) {
return;
}
$field = (string) $stage['job_field'];
$migration = CloudInstanceMigration::where($field, (int) $job->id)
->lockForUpdate()->first();
if ($migration === null) {
return;
}
$handler = (string) $stage['finish_handler'];
self::{$handler}($migration, $job, $status, $result, $payload);
}
public static function markJobUncertain(CloudJob $job): void
{
if (!self::catalogAvailable()) {
return;
}
$stage = CloudInstanceMigrationWorkflow::job((string) $job->kind);
if ($stage === null) {
return;
}
$field = (string) $stage['job_field'];
$migration = CloudInstanceMigration::where($field, (int) $job->id)
->lockForUpdate()->first();
if ($migration === null) {
return;
}
$action = (string) $stage['reconcile_action'];
if ((string) $job->kind === 'instance_migration_prepare'
&& (string) $migration->recovery_action === 'abort_prepare') {
$action = 'abort_prepare';
}
if ((string) $stage['key'] === 'cleanup') {
CloudInstanceMigrationSettlement::markCleanupAttentionLocked(
$migration,
$job,
'finalization_incident'
);
self::setState($migration, 'source_retained',
'finalization_incident', $action);
return;
}
self::setState($migration, 'attention_required',
'finalization_incident', $action);
}
/**
* Join the multi-host VPC projection back into the migration state
* machine. Every host job carries the same durable migration generation;
* only the last converged result may advance transfer or source rollback.
*/
public static function finishVpcRoutingJobLocked(
CloudJob $job,
string $status,
array $payload
): void {
$migrationId = filter_var(
$payload['maintenance_migration_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$direction = $payload['vpc_migration_direction'] ?? null;
if ($migrationId === false
|| !in_array($direction, ['target', 'source'], true)
|| !in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 宿主机任务缺少完整编排身份', 409
);
}
$reference = CloudInstanceMigration::where(
'id', (int) $migrationId
)->first();
$instance = $reference === null ? null : CloudInstance::where(
'id', (int) $reference->instance_id
)->lockForUpdate()->first();
$migration = $instance === null ? null : CloudInstanceMigration::where(
'id', (int) $migrationId
)->lockForUpdate()->first();
if ($migration === null || $instance === null
|| $migration->vpc_port_id === null
|| (int) $migration->vpc_port_id !== (int) $job->vpc_port_id
|| (int) $migration->instance_id !== (int) $job->instance_id
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(
strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid)
)) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 任务与实例位置账本不一致', 409
);
}
$runningState = $direction === 'target'
? 'vpc_target_routing' : 'vpc_source_routing';
$reconcileAction = $direction === 'target'
? 'reconcile_vpc_target' : 'reconcile_vpc_source';
$failureAction = $direction === 'target'
? 'retry_vpc_target' : 'retry_vpc_source';
if ($direction === 'target'
&& ((string) $migration->state === 'transferring'
|| $migration->target_receive_job_id !== null)) {
return;
}
if ($direction === 'source'
&& ((string) $migration->state === 'rollback_running'
|| $migration->rollback_job_id !== null)) {
return;
}
if ((string) $migration->state !== $runningState
&& !((string) $migration->state === 'attention_required'
&& in_array((string) $migration->recovery_action, [
$reconcileAction, $failureAction,
], true))) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 任务与当前迁移阶段不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState(
$migration,
'attention_required',
'operation_effect_unknown',
$reconcileAction
);
return;
}
if ($status === 'failed') {
self::setState(
$migration,
'attention_required',
'migration_vpc_routing_failed',
$failureAction
);
return;
}
if ($status !== 'succeeded') {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 任务终态无效', 409
);
}
if (!CloudVpcPortService::finishMigrationProjectionLocked(
$migration,
$instance,
$direction
)) {
return;
}
if ($direction === 'target') {
$prepare = CloudJob::find(
(int) $migration->source_prepare_job_id
);
if ($prepare === null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 收敛后源准备任务缺失', 409
);
}
$preparePayload = CloudSecrets::decrypt(
(string) $prepare->sealed_payload,
'job-payload'
);
if (!is_array($preparePayload)) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 收敛后源准备载荷无效', 409
);
}
self::queueTransferLocked(
$migration,
$prepare,
$preparePayload
);
return;
}
self::queueRollbackLocked(
$migration,
$instance,
(int) $job->user_id,
'migration_transfer_aborted'
);
}
public static function recover(
CloudInstance $instance,
array $actor,
array $input
): CloudJob {
CloudAccess::manage($actor);
CloudInput::only($input, ['action', 'confirm_recovery']);
$action = $input['action'] ?? null;
if (!is_string($action)
|| !CloudInstanceMigrationWorkflow::isRecoveryAction($action)
|| ($input['confirm_recovery'] ?? false) !== true) {
throw new CloudError('请选择并确认可用的迁移恢复操作');
}
if ($action === 'abort_prepare') {
return self::recoverAbortPrepare($instance);
}
return $instance->getConnection()->transaction(function () use (
$instance,
$actor,
$action
): CloudJob {
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->firstOrFail();
$migration = CloudInstanceMigration::where(
'instance_id', (int) $locked->id
)->where('state', 'attention_required')->orderByDesc('id')
->lockForUpdate()->first();
if ($migration === null
|| !in_array(
$action,
CloudInstanceMigrationWorkflow::recoveryActions(
$migration,
false
),
true
)) {
throw new CloudError('当前迁移没有该恢复出口', 409);
}
$handler = CloudInstanceMigrationWorkflow::recoveryHandler(
$action
);
if ($handler === null || !method_exists(self::class, $handler)) {
throw new \LogicException(
'迁移恢复动作缺少可执行处理方法'
);
}
$migration->retry_generation = (int) $migration->retry_generation + 1;
$migration->updated_at = time();
$migration->save();
return self::{$handler}(
$migration,
$locked,
(int) $actor['user_id']
);
}, 3);
}
private static function recoverAbortPrepare(
CloudInstance $instance
): CloudJob
{
$candidate = CloudInstanceMigration::where(
'instance_id', (int) $instance->id
)->orderByDesc('id')->first();
if ($candidate !== null
&& (string) $candidate->state === 'rollback_running'
&& (string) $candidate->error_code
=== 'migration_prepare_aborted'
&& $candidate->rollback_job_id !== null) {
$existing = CloudJob::find((int) $candidate->rollback_job_id);
if ($existing !== null
&& (string) $existing->kind
=== 'instance_migration_rollback') {
return $existing;
}
}
if ($candidate === null) {
throw new CloudError('当前没有可中止的迁移源准备任务', 409);
}
$evidence = self::abortPrepareEvidence($candidate);
if (!in_array(
'abort_prepare',
CloudInstanceMigrationWorkflow::recoveryActions(
$candidate,
true
),
true
)) {
throw new CloudError('当前迁移没有该恢复出口', 409);
}
$migrationId = (int) $candidate->id;
$prepareJobId = (int) $evidence['job']->id;
$eventId = (string) $evidence['event']->event_id;
$instance->getConnection()->transaction(function () use (
$instance,
$migrationId,
$prepareJobId
): void {
// Keep worker lock order: prepare finalization owns job ->
// migration -> instance. This intent transaction deliberately does
// not lock the job or receipt.
$migration = CloudInstanceMigration::where('id', $migrationId)
->lockForUpdate()->first();
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->first();
if ($migration === null || $locked === null
|| (int) $migration->source_prepare_job_id !== $prepareJobId
|| (int) $migration->instance_id !== (int) $locked->id
|| (string) $migration->state !== 'attention_required'
|| !in_array((string) $migration->recovery_action,
['reconcile_prepare', 'abort_prepare'], true)
|| $migration->target_receive_job_id !== null
|| $migration->rollback_job_id !== null
|| (int) $locked->host_id !== (int) $migration->source_host_id
|| !hash_equals(
strtolower((string) $locked->uuid),
strtolower((string) $migration->source_uuid)
)
|| (int) $locked->reservation_id
!== (int) $migration->source_reservation_id) {
throw new CloudRecoveryInstanceStateConflict(
'迁移源准备状态已经变化,不能中止', 409
);
}
if ((string) $migration->recovery_action
=== 'reconcile_prepare') {
$migration->retry_generation =
(int) $migration->retry_generation + 1;
$migration->recovery_action = 'abort_prepare';
$migration->updated_at = time();
$migration->save();
}
}, 3);
$prepare = CloudJob::find($prepareJobId);
$event = CloudAgentCompletionEvent::find($eventId);
if ($prepare === null || $event === null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移源准备成功回执在中止期间发生变化', 409
);
}
CloudOperationRecoveryService::settleLateReceipt($prepare, $event);
$settled = CloudInstanceMigration::find($migrationId);
$rollback = $settled === null || $settled->rollback_job_id === null
? null : CloudJob::find((int) $settled->rollback_job_id);
if ($rollback === null
|| (string) $rollback->kind !== 'instance_migration_rollback') {
throw new CloudRecoveryInstanceStateConflict(
'迁移中止已进入自动核对,请稍后刷新', 409
);
}
return $rollback;
}
public static function cleanupRetained(
CloudInstance $instance,
array $actor,
array $input
): CloudJob {
CloudAccess::manage($actor);
CloudInput::only($input, ['confirm_source_cleanup']);
if (($input['confirm_source_cleanup'] ?? false) !== true) {
throw new CloudError('请确认删除源宿主机上的旧副本');
}
return $instance->getConnection()->transaction(function () use (
$instance,
$actor
): CloudJob {
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->firstOrFail();
$migration = CloudInstanceMigration::where(
'instance_id', (int) $locked->id
)->where('state', 'source_retained')->where(
'recovery_action', '<>', 'reconcile_cleanup'
)->orderByDesc('id')->lockForUpdate()->first();
if ($migration === null) {
throw new CloudError('当前没有可安全清理的迁移源副本', 409);
}
$migration->retry_generation = (int) $migration->retry_generation + 1;
$migration->cleanup_job_id = null;
$migration->save();
return self::queueCleanupLocked($migration, $locked, $actor);
}, 3);
}
private static function finishPrepareLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
if ((int) $migration->instance_id !== (int) $job->instance_id
|| (int) $migration->source_host_id !== (int) $job->host_id
|| (int) ($payload['maintenance_migration_id'] ?? 0)
!== (int) $migration->id) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源准备任务与迁移账本不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_prepare');
return;
}
if ($status === 'failed') {
if ((bool) ($job->effect_may_have_applied ?? false)) {
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_prepare');
return;
}
self::releasePreparedTargetLocked($migration,
'migration_prepare_failed');
self::setState($migration, 'rolled_back',
'migration_prepare_failed');
$migration->completed_at = time();
$migration->save();
return;
}
if (!is_array($result)
|| !hash_equals((string) $migration->migration_key,
(string) ($result['migration_id'] ?? ''))
|| !is_bool($result['source_was_running'] ?? null)
|| !is_bool($result['source_autostart_enabled'] ?? null)
|| !is_string($result['manifest_digest'] ?? null)
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $result['manifest_digest']) !== 1) {
throw new CloudError('源 Agent 的 PeerMigration 回执无效', 502);
}
$migration->source_was_running = $result['source_was_running'];
$migration->source_autostart_enabled =
$result['source_autostart_enabled'];
$migration->manifest_digest = (string) $result['manifest_digest'];
if ((string) $migration->recovery_action === 'abort_prepare') {
self::finishAbortPrepareLocked($migration, $job, $result);
return;
}
CloudPeerMigrationArtifactService::finishPrepareLocked(
$migration,
$result
);
if ($migration->target_receive_job_id !== null) {
return;
}
if ($migration->vpc_port_id !== null) {
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($instance === null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 路由切换时实例账本缺失', 409
);
}
CloudVpcPortService::beginMigrationToTargetLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_target_routing', '');
return;
}
self::queueTransferLocked($migration, $job, $payload);
}
private static function finishAbortPrepareLocked(
CloudInstanceMigration $migration,
CloudJob $prepareJob,
array $result
): void {
self::assertAbortPrepareResult($migration, $result);
if ((string) $migration->state !== 'attention_required'
|| (string) $migration->recovery_action !== 'abort_prepare'
|| $migration->target_receive_job_id !== null
|| $migration->firewall_job_id !== null
|| $migration->activate_job_id !== null
|| $migration->rollback_job_id !== null
|| $migration->cleanup_job_id !== null
|| $migration->cutover_at !== null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移源准备中止状态已经变化', 409
);
}
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
$reservation = CloudResourceReservation::where(
'id', (int) $migration->target_reservation_id
)->lockForUpdate()->first();
if ($instance === null || $restore === null
|| $reservation === null
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(
strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid)
)
|| (int) $instance->reservation_id
!== (int) $migration->source_reservation_id
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| (int) $restore->reservation_id !== (int) $reservation->id
|| $restore->backup_id !== null
|| $restore->job_id !== null
|| (string) $restore->state !== 'reserved'
|| (int) $reservation->backup_restore_id !== (int) $restore->id
|| (int) $reservation->host_id !== (int) $migration->target_host_id
|| (int) $reservation->resource_pool_id
!== (int) $instance->resource_pool_id
|| (int) $reservation->user_id !== (int) $instance->user_id
|| $reservation->instance_id !== null
|| (string) $reservation->state !== 'held'
|| $reservation->network_attachment_id !== null
|| $reservation->public_ip_pool_id !== null
|| $reservation->public_ip_id !== null
|| !hash_equals(
(string) $restore->operation_id,
(string) $reservation->operation_id
)) {
throw new CloudRecoveryInstanceStateConflict(
'迁移中止前的源实例或目标预留账本已经变化', 409
);
}
CloudPublicNetworkAllocationService::currentClaimSet(
(int) $reservation->id,
'held'
);
$migration->source_was_running = $result['source_was_running'];
$migration->source_autostart_enabled =
$result['source_autostart_enabled'];
$migration->manifest_digest = (string) $result['manifest_digest'];
$migration->updated_at = time();
$migration->save();
self::queueRollbackLocked(
$migration,
$instance,
(int) $prepareJob->user_id,
'migration_prepare_aborted'
);
}
private static function queueTransferLocked(
CloudInstanceMigration $migration,
CloudJob $prepareJob,
array $preparePayload
): void {
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
if ($instance === null || $restore === null
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))
|| (int) $instance->reservation_id
!== (int) $migration->source_reservation_id
|| !hash_equals((string) $migration->source_service_status,
(string) $instance->service_status)
|| !hash_equals((string) $migration->source_network_status,
(string) $instance->network_status)
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| (int) $restore->reservation_id
!== (int) $migration->target_reservation_id
|| (string) $restore->state !== 'reserved') {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源实例或目标预留在停机期间发生变化', 409
);
}
$ticket = $preparePayload['body']['ticket'] ?? null;
if (!is_string($ticket)
|| preg_match('/^[a-f0-9]{64}$/D', $ticket) !== 1) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源任务缺少加密传输票据', 409
);
}
$spec = self::decodeSpec((string) $instance->spec_json);
$resources = self::resources($spec);
$placement = CloudPlacementService::preparedMaintenanceMigrationTarget(
$restore,
(int) $instance->id
);
$reservation = $placement['reservation'];
$publicNetwork = $placement['public_network'];
$targetIdentity = $placement['identity'];
if ((int) $reservation->resource_pool_id
!== (int) $instance->resource_pool_id
|| !hash_equals(
strtolower((string) $restore->new_uuid),
strtolower((string) $migration->target_uuid)
)
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| (int) $reservation->host_id
!== (int) $migration->target_host_id) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标资源池或网络接入身份不一致',
409
);
}
$policy = $spec['platform_network_policy_snapshot'] ?? [];
if (!is_array($policy) || array_is_list($policy)) {
$policy = [];
}
$vpcPort = CloudVpcPortService::migrationTargetMetadataLocked(
$migration,
$instance
);
$body = [
'migration_id' => (string) $migration->migration_key,
'ticket' => $ticket,
'source_endpoints' => self::decodePeerEndpoints(
(string) $migration->source_endpoint
),
'source_identity_sha256' =>
(string) $migration->peer_trust_id,
'source_uuid' => (string) $migration->source_uuid,
'manifest_digest' => (string) $migration->manifest_digest,
'network_suspended' =>
(string) $migration->source_network_status !== 'active',
'debian_12_network_fix' =>
($spec['debian_12_network_fix'] ?? null) === true,
'uuid' => (string) $migration->target_uuid,
'source' => '',
'name' => (string) $instance->name,
'description' => '',
'vcpu' => $resources['cpu_cores'],
'memory_mib' => $resources['memory_mib'],
'disk_gib' => $resources['disk_gib'],
'bandwidth_mbps' => $resources['bandwidth_mbps'],
'public_network' => $publicNetwork->toArray(),
'template_id' => '',
'template_revision' => 0,
'template_generation_token' => '',
'startup_script' => '',
// The target seed preserves credentials and hostname from the
// migrated disk. Injecting panel metadata here would violate the
// Agent's PeerMigration contract.
'hostname' => '',
'guest_ssh_public_key' => '',
'root_password' => '',
'platform_network_policy_snapshot' => $policy,
];
if ($vpcPort !== null) {
$body['vpc_port'] = $vpcPort;
}
$generation = (int) $migration->retry_generation;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
[
'user_id' => (int) $prepareJob->user_id,
'request_idempotency_key' => 'migration-transfer-'
. (string) $migration->migration_key . '-' . $generation,
],
(int) $migration->target_host_id,
'instance_migration_transfer',
'POST',
'/v1/vms/' . rawurlencode((string) $migration->target_uuid)
. '/peer-migration/receive',
$body,
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'reservation_id' => (int) $reservation->id,
'maintenance_migration_id' => (int) $migration->id,
'migration_target' => [
'contract' => 1,
'backup_restore_id' => (int) $restore->id,
'reservation_id' => (int) $reservation->id,
'host_id' => (int) $migration->target_host_id,
'resource_pool_id' =>
(int) $reservation->resource_pool_id,
'operation_id' => (string) $restore->operation_id,
'target_uuid' =>
strtolower((string) $migration->target_uuid),
'target_agent_revision' =>
(int) $placement['target_agent_revision'],
'target_agent_contract_sha256' =>
(string) $placement[
'target_agent_contract_sha256'
],
'target_identity_json' =>
$targetIdentity->identityJson(),
'target_identity_sha256' =>
$targetIdentity->identitySha256(),
'target_claim_set_sha256' =>
$targetIdentity->claimSetSha256(),
'target_public_network_json' =>
$targetIdentity->publicNetworkJson(),
'target_public_network_sha256' =>
$targetIdentity->publicNetworkSha256(),
'vpc_port' => $vpcPort,
],
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'target_host_id' => (int) $migration->target_host_id,
'target_uuid' => (string) $migration->target_uuid,
'manifest_digest' => (string) $migration->manifest_digest,
'target_identity_sha256' =>
$targetIdentity->identitySha256(),
'target_claim_set_sha256' =>
$targetIdentity->claimSetSha256(),
'target_public_network_sha256' =>
$targetIdentity->publicNetworkSha256(),
'target_agent_revision' =>
(int) $placement['target_agent_revision'],
'target_agent_contract_sha256' =>
(string) $placement['target_agent_contract_sha256'],
'vpc_port' => $vpcPort,
],
(string) $restore->operation_id,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/transfer/' . $generation
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标接收任务幂等冲突', 409
);
}
$job->instance_id = (int) $instance->id;
$job->backup_restore_id = (int) $restore->id;
$job->save();
$restore->job_id = (int) $job->id;
$restore->state = 'restoring';
$restore->error_code = '';
$restore->updated_at = time();
$restore->save();
$migration->target_receive_job_id = (int) $job->id;
self::setState($migration, 'transferring', '');
}
private static function finishTransferLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($restore === null || $instance === null
|| (int) $migration->target_receive_job_id !== (int) $job->id
|| (int) $job->backup_restore_id !== (int) $restore->id
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->host_id !== (int) $migration->target_host_id
|| (int) ($payload['maintenance_migration_id'] ?? 0)
!== (int) $migration->id
|| !hash_equals((string) $restore->operation_id,
(string) $job->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标接收任务与迁移账本不一致', 409
);
}
if ($status === 'running') {
return;
}
try {
$requestedNetwork = CloudPublicNetworkSnapshot::fromArray(
$payload['body']['public_network'] ?? null
);
} catch (\InvalidArgumentException) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标接收缺少双栈公网快照', 409
);
}
$now = time();
if ($status === 'uncertain') {
[$target] = self::verifiedTargetBundle(
$migration,
$restore,
$instance,
['held', 'uncertain'],
$requestedNetwork
);
CloudPublicNetworkAllocationService::fenceBundle(
(int) $target->id,
(string) $restore->operation_id
);
$target = CloudResourceReservation::where(
'id', (int) $target->id
)->lockForUpdate()->first();
if ($target === null
|| !in_array((string) $target->state,
['held', 'uncertain'], true)
|| $target->instance_id !== null
|| !hash_equals((string) $restore->operation_id,
(string) $target->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标双栈隔离预留发生变化', 409
);
}
$target->state = 'uncertain';
$target->instance_id = null;
$target->expires_at = null;
$target->updated_at = $now;
$target->save();
$restore->state = 'uncertain';
$restore->error_code = 'operation_effect_unknown';
$restore->updated_at = $now;
$restore->save();
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_transfer');
return;
}
if ($status === 'failed') {
self::restoreTargetReservationForRetryLocked(
$migration, $restore, $instance, $requestedNetwork
);
$restore->state = 'reserved';
$restore->error_code = 'peer_transfer_failed';
$restore->updated_at = $now;
$restore->save();
self::setState($migration, 'attention_required',
'peer_transfer_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'transfer'
));
return;
}
[$target, $targetClaimSet] = self::verifiedTargetBundle(
$migration,
$restore,
$instance,
['held', 'uncertain'],
$requestedNetwork
);
if (!is_array($result)
|| !hash_equals((string) $migration->migration_key,
(string) ($result['migration_id'] ?? ''))
|| !hash_equals((string) $migration->manifest_digest,
(string) ($result['manifest_digest'] ?? ''))
|| !hash_equals(strtolower((string) $migration->target_uuid),
strtolower((string) ($result['uuid'] ?? '')))
|| (string) ($result['status'] ?? '') !== 'shutoff'
|| ($result['staged_powered_off'] ?? null) !== true
|| !self::sameDocument(
$result['vpc_port'] ?? null,
$payload['body']['vpc_port'] ?? null
)) {
throw new CloudError('目标 Agent 的 PeerMigration 回执无效', 502);
}
$rootDiskGib = filter_var(
$payload['body']['disk_gib'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
if ($rootDiskGib === false) {
throw new CloudError('目标 Agent 的系统盘容量回执无效', 502);
}
$sourceHint = CloudResourceReservation::find(
(int) $migration->source_reservation_id
);
if ($sourceHint === null
|| preg_match('/^[a-f0-9]{32}$/D',
(string) $sourceHint->operation_id) !== 1) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源双栈网络操作身份无效', 409
);
}
$handoff = CloudPublicNetworkAllocationService::
handoffMigrationBundles(
(int) $sourceHint->id,
(int) $target->id,
(int) $instance->id,
(string) $sourceHint->operation_id,
(string) $restore->operation_id
);
$sourceClaimSet = $handoff['source'] ?? null;
$committedTargetClaimSet = $handoff['target'] ?? null;
if (!$sourceClaimSet instanceof CloudActivePublicNetworkClaimSet
|| !$committedTargetClaimSet
instanceof CloudActivePublicNetworkClaimSet) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 双栈网络切换回执无效', 409
);
}
if (!hash_equals(
$targetClaimSet->publicNetwork()->fingerprint(),
$committedTargetClaimSet->publicNetwork()->fingerprint()
)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标双栈公网提交身份发生变化', 409
);
}
$target = CloudResourceReservation::where(
'id', (int) $target->id
)->lockForUpdate()->first();
$source = CloudResourceReservation::where(
'id', (int) $migration->source_reservation_id
)->lockForUpdate()->first();
$snapshotDiskGib = CloudPeerMigrationArtifactService::artifactDiskGib(
$migration
);
$expectedTargetClaim = (int) $rootDiskGib + $snapshotDiskGib;
if ($target === null || $source === null
|| (string) $target->state !== 'committed'
|| (int) $target->instance_id !== (int) $instance->id
|| !hash_equals((string) $restore->operation_id,
(string) $target->operation_id)
|| (int) $target->disk_gib !== $expectedTargetClaim
|| (string) $source->state !== 'held'
|| (int) $source->host_id !== (int) $migration->source_host_id
|| (int) $source->resource_pool_id
!== (int) $instance->resource_pool_id
|| (int) $target->resource_pool_id
!== (int) $source->resource_pool_id
|| $source->instance_id !== null
|| (int) $source->disk_gib !== (int) $rootDiskGib
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 切换前资源预留账本不一致', 409
);
}
$targetSpec = self::finalizeTargetSpec(
self::decodeSpec((string) $instance->spec_json),
is_array($payload['body'] ?? null) ? $payload['body'] : [],
$result,
(int) $target->resource_pool_id
);
CloudPeerMigrationArtifactService::finishReceiveLocked(
$migration, $result
);
// Snapshot rows now account for their target volumes. The target
// reservation therefore keeps only the root disk; while the source
// duplicate exists its original reservation accounts for root plus
// every frozen snapshot.
$target->disk_gib = (int) $rootDiskGib;
$source->disk_gib = $expectedTargetClaim;
$source->updated_at = $now;
$source->save();
$target->updated_at = $now;
$target->save();
$spec = $targetSpec;
$spec['managed'] = true;
$spec['migrated_from_host_id'] = (int) $migration->source_host_id;
$spec['migrated_from_uuid'] = (string) $migration->source_uuid;
$spec['maintenance_migration_key'] =
(string) $migration->migration_key;
$spec['traffic_counter_generation'] = [
'host_id' => (int) $migration->target_host_id,
'uuid' => strtolower((string) $migration->target_uuid),
'download_base_bytes' =>
max(0, (int) $instance->lifetime_download_bytes),
'upload_base_bytes' =>
max(0, (int) $instance->lifetime_upload_bytes),
];
CloudVpcPortService::finalizeMigrationCutoverLocked(
$migration,
$instance
);
$instance->host_id = (int) $migration->target_host_id;
$instance->uuid = strtolower((string) $migration->target_uuid);
$instance->resource_pool_id = (int) $target->resource_pool_id;
$instance->reservation_id = (int) $target->id;
$instance->spec_json = json_encode(
$spec,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
$instance->service_status = (string) $migration->source_service_status;
$instance->network_status = (string) $migration->source_network_status;
$instance->traffic_synced_at = 0;
$instance->updated_at = $now;
$instance->save();
CloudInstanceMigrationSettlement::settleCutoverLocked(
$migration,
$sourceClaimSet
);
$restore = CloudBackupRestore::where(
'id', (int) $restore->id
)->lockForUpdate()->first();
if ($restore === null || $restore->reservation_id !== null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标恢复预留未完成结算', 409
);
}
$restore->state = 'ready';
$restore->error_code = '';
$restore->ready_at = $now;
$restore->updated_at = $now;
$restore->save();
$migration->cutover_at = $now;
self::setState($migration, 'cutover', '');
CloudInstanceRuntimeService::seed(
(int) $instance->id,
(int) $instance->host_id,
(string) $instance->uuid
);
$firewall = CloudFirewallService::queueMaintenanceMigrationReapplyLocked(
$instance,
$migration,
(int) $job->user_id,
(int) $migration->retry_generation
);
if ($firewall !== null) {
$migration->firewall_job_id = (int) $firewall->id;
self::setState($migration, 'firewall_running', '');
return;
}
self::advanceAfterTargetReadyLocked(
$migration, $instance, (int) $job->user_id
);
}
private static function restoreTargetReservationForRetryLocked(
CloudInstanceMigration $migration,
CloudBackupRestore $restore,
CloudInstance $instance,
CloudPublicNetworkSnapshot $publicNetwork
): void {
[$reservation] = self::verifiedTargetBundle(
$migration,
$restore,
$instance,
['held', 'uncertain'],
$publicNetwork
);
$reopened = CloudPublicNetworkAllocationService::reopenBundleForRetry(
(int) $reservation->id,
(string) $restore->operation_id,
true
);
if (!hash_equals($publicNetwork->fingerprint(),
$reopened->fingerprint())) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 重试目标双栈公网身份发生变化', 409
);
}
$reservation = CloudResourceReservation::where(
'id', (int) $reservation->id
)->lockForUpdate()->first();
if ($reservation === null || (string) $reservation->state !== 'held'
|| $reservation->instance_id !== null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 重试目标预留没有完整恢复', 409
);
}
}
/**
* @param list<string> $allowedStates
* @return array{0:CloudResourceReservation,
* 1:CloudActivePublicNetworkClaimSet}
*/
private static function verifiedTargetBundle(
CloudInstanceMigration $migration,
CloudBackupRestore $restore,
CloudInstance $instance,
array $allowedStates,
?CloudPublicNetworkSnapshot $expectedNetwork = null
): array {
$reservation = CloudResourceReservation::where(
'id', (int) $migration->target_reservation_id
)->first();
if ($reservation === null
|| !in_array((string) $reservation->state,
$allowedStates, true)
|| (int) $restore->reservation_id !== (int) $reservation->id
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| (int) $reservation->backup_restore_id !== (int) $restore->id
|| $reservation->order_id !== null
|| $reservation->offer_id !== null
|| $reservation->backup_restore_attempt_key !== null
|| $reservation->migration_source_debt_id !== null
|| (int) $reservation->host_id
!== (int) $migration->target_host_id
|| (int) $reservation->resource_pool_id
!== (int) $instance->resource_pool_id
|| (int) $reservation->user_id !== (int) $instance->user_id
|| $reservation->instance_id !== null
|| $reservation->network_attachment_id !== null
|| $reservation->public_ip_pool_id !== null
|| $reservation->public_ip_id !== null
|| !hash_equals((string) $restore->operation_id,
(string) $reservation->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标双栈预留账本不一致', 409
);
}
$claimSet = CloudPublicNetworkAllocationService::currentClaimSet(
(int) $reservation->id,
(string) $reservation->state
);
if ($expectedNetwork !== null && !hash_equals(
$expectedNetwork->fingerprint(),
$claimSet->publicNetwork()->fingerprint()
)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标双栈公网身份不一致', 409
);
}
return [$reservation, $claimSet];
}
/** Release a target held before the source mutation was dispatched. */
private static function releasePreparedTargetLocked(
CloudInstanceMigration $migration,
string $errorCode
): void {
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
if ($restore === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 缺少目标预留目录', 409
);
}
if ((string) $restore->state === 'failed') {
return;
}
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($instance === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 缺少源实例账本', 409
);
}
[$reservation] = self::verifiedTargetBundle(
$migration,
$restore,
$instance,
['held']
);
$releaseReason = 'maintenance_migration_target_not_applied';
CloudPublicNetworkAllocationService::releaseBundle(
(int) $reservation->id,
(string) $restore->operation_id,
$releaseReason,
true
);
$reservation = CloudResourceReservation::where(
'id', (int) $reservation->id
)->lockForUpdate()->first();
if ($reservation === null
|| (string) $reservation->state !== 'held'
|| $reservation->instance_id !== null
|| !hash_equals((string) $restore->operation_id,
(string) $reservation->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标双栈释放预留发生变化', 409
);
}
$now = time();
$reservation->state = 'released';
$reservation->instance_id = null;
$reservation->expires_at = null;
$reservation->released_at = $now;
$reservation->release_reason = $releaseReason;
$reservation->updated_at = $now;
$reservation->save();
$restore->state = 'failed';
$restore->error_code = $errorCode;
$restore->updated_at = $now;
$restore->save();
}
private static function retryTransferLocked(
CloudInstanceMigration $migration,
CloudInstance $instance
): CloudJob {
if ((int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 传输重试时源实例位置已经变化', 409
);
}
$prepare = CloudJob::find((int) $migration->source_prepare_job_id);
if ($prepare === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源准备任务目录缺失', 409
);
}
$payload = CloudSecrets::decrypt(
(string) $prepare->sealed_payload, 'job-payload'
);
if (!is_array($payload)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源准备任务载荷无效', 409
);
}
self::rotateTargetTransferOperationLocked($migration, $instance);
$migration->target_receive_job_id = null;
$migration->updated_at = time();
$migration->save();
self::queueTransferLocked($migration, $prepare, $payload);
$job = CloudJob::find((int) $migration->target_receive_job_id);
if ($job === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 重试任务目录缺失', 409
);
}
return $job;
}
private static function recoverRetryTransferLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
unset($actorUserId);
return self::retryTransferLocked($migration, $instance);
}
private static function recoverRetryVpcTargetLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
unset($actorUserId);
$jobs = CloudVpcPortService::beginMigrationToTargetLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_target_routing', '');
return self::firstVpcRoutingJob($jobs);
}
private static function recoverAbortVpcTargetLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
unset($actorUserId);
$jobs = CloudVpcPortService::beginMigrationToSourceLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_source_routing', '');
return self::firstVpcRoutingJob($jobs);
}
private static function recoverRetryVpcSourceLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
unset($actorUserId);
$jobs = CloudVpcPortService::beginMigrationToSourceLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_source_routing', '');
return self::firstVpcRoutingJob($jobs);
}
private static function recoverAbortTransferLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
if ($migration->vpc_port_id !== null) {
$jobs = CloudVpcPortService::beginMigrationToSourceLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_source_routing', '');
return self::firstVpcRoutingJob($jobs);
}
return self::queueRollbackLocked(
$migration,
$instance,
$actorUserId,
'migration_transfer_aborted'
);
}
/** @param list<CloudJob> $jobs */
private static function firstVpcRoutingJob(array $jobs): CloudJob
{
$job = $jobs[0] ?? null;
if (!$job instanceof CloudJob) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 路由切换没有生成可跟踪任务', 409
);
}
return $job;
}
/**
* A confirmed not-applied transfer may be retried only under a fresh
* Agent operation ID. Rotate the restore, reservation and IP ledgers as
* one locked unit so the prior Journal result cannot be replayed as the
* new attempt.
*/
private static function rotateTargetTransferOperationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance
): void {
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
$reservationHint = CloudResourceReservation::where(
'id', (int) $migration->target_reservation_id
)->first();
if ($restore === null || $reservationHint === null
|| (string) $restore->state !== 'reserved'
|| (int) $restore->reservation_id !== (int) $reservationHint->id
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| (int) $reservationHint->backup_restore_id !== (int) $restore->id
|| (int) $reservationHint->host_id
!== (int) $migration->target_host_id
|| (int) $reservationHint->resource_pool_id
!== (int) $instance->resource_pool_id
|| (int) $reservationHint->user_id !== (int) $instance->user_id
|| $reservationHint->instance_id !== null
|| (string) $reservationHint->state !== 'held'
|| $reservationHint->network_attachment_id !== null
|| $reservationHint->public_ip_pool_id !== null
|| $reservationHint->public_ip_id !== null
|| !hash_equals((string) $restore->operation_id,
(string) $reservationHint->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 传输重试账本不一致', 409
);
}
CloudPublicNetworkAllocationService::currentClaimSet(
(int) $reservationHint->id,
'held'
);
$priorOperationId = (string) $restore->operation_id;
$operationId = bin2hex(random_bytes(16));
CloudPublicNetworkAllocationService::rotateHeldBundleOperation(
(int) $reservationHint->id,
$priorOperationId,
$operationId
);
$now = time();
$restore->operation_id = $operationId;
$restore->job_id = null;
$restore->error_code = '';
$restore->updated_at = $now;
$restore->save();
$reservation = CloudResourceReservation::where(
'id', (int) $reservationHint->id
)->lockForUpdate()->first();
if ($reservation === null
|| !hash_equals($operationId,
(string) $reservation->operation_id)
|| (string) $reservation->state !== 'held'
|| $reservation->instance_id !== null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 传输重试操作没有完整轮换', 409
);
}
}
private static function queueRollbackLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId,
string $errorCode
): CloudJob {
if ((int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))
|| $migration->source_was_running === null
|| $migration->source_autostart_enabled === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源实例回滚账本不完整', 409
);
}
$generation = (int) $migration->retry_generation;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
[
'user_id' => $actorUserId,
'request_idempotency_key' => 'migration-rollback-'
. (string) $migration->migration_key . '-' . $generation,
],
(int) $migration->source_host_id,
'instance_migration_rollback',
'POST',
'/v1/vms/' . rawurlencode((string) $migration->source_uuid)
. '/peer-migration/rollback',
[
'migration_id' => (string) $migration->migration_key,
'source_was_running' => (bool) $migration->source_was_running,
'source_autostart_enabled' =>
(bool) $migration->source_autostart_enabled,
],
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'maintenance_migration_id' => (int) $migration->id,
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'source_host_id' => (int) $migration->source_host_id,
'source_uuid' => (string) $migration->source_uuid,
'source_was_running' =>
(bool) $migration->source_was_running,
'source_autostart_enabled' =>
(bool) $migration->source_autostart_enabled,
],
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/rollback/' . $generation
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源回滚任务幂等冲突', 409
);
}
$job->instance_id = (int) $instance->id;
$job->backup_restore_id = (int) $migration->restore_id;
$job->save();
$migration->rollback_job_id = (int) $job->id;
self::setState($migration, 'rollback_running', $errorCode);
return $job;
}
private static function retryRollbackLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
$migration->rollback_job_id = null;
$migration->updated_at = time();
$migration->save();
return self::queueRollbackLocked(
$migration, $instance, $actorUserId,
'migration_source_rollback_failed'
);
}
private static function recoverRetryRollbackLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
return self::retryRollbackLocked(
$migration,
$instance,
$actorUserId
);
}
private static function finishRollbackLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
unset($payload);
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($instance === null
|| (int) $migration->rollback_job_id !== (int) $job->id
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->host_id !== (int) $migration->source_host_id
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源回滚任务与实例不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_rollback');
return;
}
if ($status === 'failed') {
self::setState($migration, 'attention_required',
'migration_source_rollback_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'rollback'
));
return;
}
if (!is_array($result)
|| !hash_equals(strtolower((string) $migration->source_uuid),
strtolower((string) ($result['uuid'] ?? '')))
|| !is_bool($result['running'] ?? null)
|| !is_bool($result['autostart_enabled'] ?? null)
|| $result['running'] !== (bool) $migration->source_was_running
|| $result['autostart_enabled']
!== (bool) $migration->source_autostart_enabled) {
throw new CloudError(
'源 Agent 未确认实例恢复到迁移前电源和自启动状态', 502
);
}
$abortCode = $migration->target_receive_job_id === null
? 'migration_prepare_aborted'
: 'migration_transfer_aborted';
self::releasePreparedTargetLocked($migration, $abortCode);
self::setState($migration, 'rolled_back', $abortCode);
$migration->completed_at = time();
$migration->save();
}
private static function finishFirewallLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
unset($result, $payload);
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($instance === null
|| (int) $migration->firewall_job_id !== (int) $job->id
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->host_id !== (int) $migration->target_host_id
|| (int) $instance->host_id !== (int) $migration->target_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->target_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 防火墙任务与目标实例不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState($migration, 'attention_required',
'firewall_result_uncertain', 'reconcile_firewall');
return;
}
if ($status === 'failed') {
self::setState($migration, 'attention_required',
'firewall_apply_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'firewall'
));
return;
}
$policy = CloudInstanceFirewall::where(
'instance_id', (int) $instance->id
)->lockForUpdate()->first();
if ($policy === null
|| (int) $policy->host_id !== (int) $instance->host_id
|| (string) $policy->sync_state !== 'active'
|| (int) $policy->desired_revision
!== (int) $policy->applied_revision) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标防火墙没有完成收敛', 409
);
}
self::advanceAfterTargetReadyLocked(
$migration, $instance, (int) $job->user_id
);
}
private static function retryFirewallLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
$migration->firewall_job_id = null;
$migration->updated_at = time();
$migration->save();
$job = CloudFirewallService::queueMaintenanceMigrationReapplyLocked(
$instance,
$migration,
$actorUserId,
(int) $migration->retry_generation
);
if ($job === null) {
self::advanceAfterTargetReadyLocked(
$migration, $instance, $actorUserId
);
$next = $migration->activate_job_id === null
? $migration->cleanup_job_id : $migration->activate_job_id;
$job = $next === null ? null : CloudJob::find((int) $next);
} else {
$migration->firewall_job_id = (int) $job->id;
self::setState($migration, 'firewall_running', '');
}
if ($job === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 防火墙恢复后没有后续任务', 409
);
}
return $job;
}
private static function recoverRetryFirewallLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
return self::retryFirewallLocked(
$migration,
$instance,
$actorUserId
);
}
private static function advanceAfterTargetReadyLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): void {
self::queueActivationLocked($migration, $instance, $actorUserId);
}
private static function queueActivationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
$generation = (int) $migration->retry_generation;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
[
'user_id' => $actorUserId,
'request_idempotency_key' => 'migration-activate-'
. (string) $migration->migration_key . '-' . $generation,
],
(int) $migration->target_host_id,
'instance_migration_activate',
'POST',
'/v1/vms/' . rawurlencode((string) $migration->target_uuid)
. '/peer-migration/activate',
[
'migration_id' => (string) $migration->migration_key,
'target_was_running' =>
(bool) $migration->source_was_running,
'target_autostart_enabled' =>
(bool) $migration->source_autostart_enabled,
],
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'maintenance_migration_id' => (int) $migration->id,
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'target_host_id' => (int) $migration->target_host_id,
'target_uuid' => (string) $migration->target_uuid,
'source_was_running' => (bool) $migration->source_was_running,
'source_autostart_enabled' =>
(bool) $migration->source_autostart_enabled,
],
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/activate/' . $generation
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标激活任务幂等冲突', 409
);
}
$job->instance_id = (int) $instance->id;
$job->backup_restore_id = (int) $migration->restore_id;
$job->save();
$migration->activate_job_id = (int) $job->id;
self::setState($migration, 'activation_running', '');
return $job;
}
private static function retryActivationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
$migration->activate_job_id = null;
$migration->updated_at = time();
$migration->save();
return self::queueActivationLocked(
$migration, $instance, $actorUserId
);
}
private static function recoverRetryActivationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
return self::retryActivationLocked(
$migration,
$instance,
$actorUserId
);
}
private static function finishActivationLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
unset($payload);
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($instance === null
|| (int) $migration->activate_job_id !== (int) $job->id
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->host_id !== (int) $migration->target_host_id
|| (int) $instance->host_id !== (int) $migration->target_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->target_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 激活任务与目标实例不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_activation');
return;
}
if ($status === 'failed') {
self::setState($migration, 'attention_required',
'migration_target_activation_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'activation'
));
return;
}
if (!is_array($result)
|| !hash_equals(strtolower((string) $migration->target_uuid),
strtolower((string) ($result['uuid'] ?? '')))
|| !is_bool($result['running'] ?? null)
|| !is_bool($result['autostart_enabled'] ?? null)
|| $result['running'] !== (bool) $migration->source_was_running
|| $result['autostart_enabled']
!== (bool) $migration->source_autostart_enabled) {
throw new CloudError(
'目标 Agent 未确认实例恢复到迁移前电源和自启动状态', 502
);
}
self::advanceAfterActivationLocked(
$migration, $instance, (int) $job->user_id
);
}
private static function advanceAfterActivationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): void {
if ((string) $migration->cleanup_mode === 'delete') {
self::queueCleanupLocked($migration, $instance, [
'user_id' => $actorUserId,
'request_idempotency_key' => 'migration-cleanup-'
. (string) $migration->migration_key . '-'
. (int) $migration->retry_generation,
]);
return;
}
CloudInstanceMigrationSettlement::markRetainedLocked($migration);
self::setState($migration, 'source_retained', '');
}
private static function queueCleanupLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
array $actor
): CloudJob {
if (!in_array((string) $migration->state,
['cutover', 'activation_running', 'source_retained'], true)) {
throw new CloudError('当前迁移状态不能清理源副本', 409);
}
$generation = (int) $migration->retry_generation;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
$actor,
(int) $migration->source_host_id,
'instance_migration_cleanup',
'POST',
'/v1/vms/' . rawurlencode((string) $migration->source_uuid)
. '/peer-migration/cleanup',
[
'migration_id' => (string) $migration->migration_key,
'remove_managed_disks' => true,
],
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'maintenance_migration_id' => (int) $migration->id,
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'source_host_id' => (int) $migration->source_host_id,
'source_uuid' => (string) $migration->source_uuid,
],
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/source/' . $generation
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源清理任务幂等冲突', 409
);
}
$job->instance_id = (int) $instance->id;
$job->backup_restore_id = (int) $migration->restore_id;
$job->save();
$migration->cleanup_job_id = (int) $job->id;
CloudInstanceMigrationSettlement::bindCleanupLocked($migration, $job);
self::setState($migration, 'cleanup_running', '');
return $job;
}
private static function finishCleanupLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
unset($payload);
if ((int) $migration->cleanup_job_id !== (int) $job->id
|| (int) $job->instance_id !== (int) $migration->instance_id
|| (int) $job->host_id !== (int) $migration->source_host_id) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源清理任务与迁移账本不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
CloudInstanceMigrationSettlement::markCleanupAttentionLocked(
$migration,
$job,
'operation_effect_unknown'
);
self::setState($migration, 'source_retained',
'operation_effect_unknown', 'reconcile_cleanup');
return;
}
if ($status === 'failed') {
CloudInstanceMigrationSettlement::markCleanupFailedLocked(
$migration,
$job,
'migration_cleanup_failed'
);
self::setState($migration, 'source_retained',
'migration_cleanup_failed');
return;
}
if (!is_array($result)
|| ($result['deleted'] ?? null) !== true
|| ($result['managed_artifacts_absent'] ?? null) !== true) {
throw new CloudError(
'源 Agent 未确认实例和托管磁盘均已删除', 502
);
}
CloudInstanceMigrationSettlement::releaseSourceLocked(
$migration,
$job
);
self::setState($migration, 'completed', '');
$migration->completed_at = time();
$migration->save();
}
private static function setState(
CloudInstanceMigration $migration,
string $state,
string $errorCode,
string $recoveryAction = ''
): void {
CloudInstanceMigrationWorkflow::assertState(
$state,
$recoveryAction
);
$migration->state = $state;
$migration->error_code = $errorCode;
$migration->recovery_action = $recoveryAction;
$migration->updated_at = time();
$migration->save();
}
private static function sameDocument($left, $right): bool
{
$canonical = static function ($value) use (&$canonical) {
if (!is_array($value)) {
return $value;
}
if (!array_is_list($value)) {
ksort($value, SORT_STRING);
}
foreach ($value as $key => $child) {
$value[$key] = $canonical($child);
}
return $value;
};
return hash_equals(
hash('sha256', json_encode(
$canonical($left),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)),
hash('sha256', json_encode(
$canonical($right),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
))
);
}
/** @return array{endpoints:list<string>,identity_sha256:string} */
private static function peerCapability(int $hostId, array $actor): array
{
$capabilities = CloudAgentSettingsService::clientForHost(
$hostId, 30.0
)->request('GET', '/v1/capabilities', $actor);
$peer = $capabilities['peer_migration'] ?? null;
$features = $capabilities['features'] ?? null;
$endpoints = is_array($peer) ? ($peer['endpoints'] ?? null) : null;
$identity = is_array($peer)
? ($peer['identity_sha256'] ?? null) : null;
$version = is_array($peer) ? filter_var(
$peer['version'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
) : false;
$chunk = is_array($peer) ? filter_var(
$peer['chunk_size_bytes'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
) : false;
if (!is_array($features)
|| ($features['peer_migration'] ?? null) !== true
|| !is_array($peer)
|| $version !== self::PROTOCOL_VERSION
|| ($peer['pinned_tls'] ?? null) !== true
|| $chunk !== self::CHUNK_SIZE
|| !self::validPeerEndpoints($endpoints)
|| !is_string($identity)
|| preg_match('/^[a-f0-9]{64}$/D', $identity) !== 1) {
throw new CloudError(
'宿主机未就绪 PeerMigration V3、TLS 1.3 身份锁定或固定分块能力',
409
);
}
return [
'endpoints' => array_values($endpoints),
'identity_sha256' => $identity,
];
}
private static function validPeerEndpoints(mixed $values): bool
{
if (!is_array($values) || !array_is_list($values)
|| count($values) < 1 || count($values) > 16) {
return false;
}
$seen = [];
foreach ($values as $value) {
if (!is_string($value) || !self::validPeerEndpoint($value)
|| isset($seen[$value])) {
return false;
}
$seen[$value] = true;
}
return true;
}
private static function validPeerEndpoint(string $value): bool
{
if ($value === '' || strlen($value) > 2048
|| trim($value) !== $value
|| preg_match('/[\\x00\\r\\n\\\\]/', $value) === 1) {
return false;
}
$parts = parse_url($value);
$host = is_array($parts) ? ($parts['host'] ?? null) : null;
if (!is_string($host)) {
return false;
}
$literal = str_starts_with($host, '[') && str_ends_with($host, ']')
? substr($host, 1, -1) : $host;
return is_array($parts)
&& ($parts['scheme'] ?? null) === 'https'
&& filter_var($literal, FILTER_VALIDATE_IP) !== false
&& filter_var($parts['port'] ?? null, FILTER_VALIDATE_INT,
['options' => ['min_range' => 1, 'max_range' => 65535]])
!== false
&& !isset($parts['user'], $parts['pass'], $parts['query'],
$parts['fragment'])
&& (!isset($parts['path']) || $parts['path'] === ''
|| $parts['path'] === '/');
}
/** @param list<string> $endpoints */
private static function encodePeerEndpoints(array $endpoints): string
{
if (!self::validPeerEndpoints($endpoints)) {
throw new CloudError('源宿主机的 PeerMigration 地址无效', 409);
}
$encoded = json_encode(array_values($endpoints),
JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
if (strlen($encoded) > 2048) {
throw new CloudError('源宿主机的 PeerMigration 地址过多', 409);
}
return $encoded;
}
/** @return list<string> */
private static function decodePeerEndpoints(string $encoded): array
{
try {
$endpoints = json_decode($encoded, true, 16,
JSON_THROW_ON_ERROR);
} catch (\Throwable) {
throw new CloudRecoveryPayloadUnavailable(
'PeerMigration 源地址目录无法读取', 409
);
}
if (!self::validPeerEndpoints($endpoints)) {
throw new CloudRecoveryPayloadUnavailable(
'PeerMigration 源地址目录无效', 409
);
}
return array_values($endpoints);
}
private static function catalogAvailable(): bool
{
try {
$schema = (new CloudInstanceMigration())->getConnection()
->getSchemaBuilder();
return $schema->hasTable('cloud_instance_migrations')
&& $schema->hasTable('cloud_peer_migration_artifacts');
} catch (\Throwable $error) {
return false;
}
}
private static function resources(array $spec): array
{
$values = [
'cpu_cores' => $spec['vcpu'] ?? $spec['cpu_cores'] ?? null,
'memory_mib' => $spec['memory_mib'] ?? $spec['ram_mib'] ?? null,
'disk_gib' => $spec['disk_gib'] ?? null,
'bandwidth_mbps' => $spec['bandwidth_mbps'] ?? null,
];
foreach ($values as $field => $value) {
$parsed = filter_var($value, FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]);
if ($parsed === false) {
throw new CloudError('实例缺少可信的资源规格快照', 409);
}
$values[$field] = (int) $parsed;
}
return $values;
}
private static function finalizeTargetSpec(
array $source,
array $body,
array $vm,
int $resourcePoolId
): array {
if ($resourcePoolId < 1) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标资源池快照无效', 409
);
}
$expected = self::targetRestoreSnapshot($body);
try {
$actual = self::targetRestoreSnapshot($vm);
} catch (CloudError $error) {
throw new CloudError(
'目标 Agent 返回的资源或网络快照不一致', 502
);
}
if ($expected !== $actual) {
throw new CloudError(
'目标 Agent 返回的资源或网络快照不一致', 502
);
}
$spec = CloudInstanceMetadata::merge($source, $vm);
foreach ([
'network', 'public_pool_key', 'public_pool_cidr',
'public_ip_cidr', 'public_ip', 'gateway', 'dns_servers',
] as $retired) {
unset($spec[$retired]);
}
return array_replace($spec, [
'vcpu' => $expected['vcpu'],
'cpu_cores' => $expected['vcpu'],
'memory_mib' => $expected['memory_mib'],
'ram_mib' => $expected['memory_mib'],
'disk_gib' => $expected['disk_gib'],
'bandwidth_mbps' => $expected['bandwidth_mbps'],
'public_network' => $expected['public_network'],
'resource_pool_id' => $resourcePoolId,
]);
}
/** @return array<string,mixed> */
private static function targetRestoreSnapshot(array $value): array
{
$integers = [];
foreach ([
'vcpu' => [1, 512],
'memory_mib' => [128, 16 * 1024 * 1024],
'disk_gib' => [1, 4096],
'bandwidth_mbps' => [1, 100000],
] as $field => [$minimum, $maximum]) {
$parsed = filter_var($value[$field] ?? null,
FILTER_VALIDATE_INT, ['options' => [
'min_range' => $minimum, 'max_range' => $maximum,
]]);
if ($parsed === false) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标资源规格快照无效', 409
);
}
$integers[$field] = (int) $parsed;
}
try {
$publicNetwork = CloudPublicNetworkSnapshot::fromArray(
$value['public_network'] ?? null
);
} catch (\InvalidArgumentException) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标网络规格快照无效', 409
);
}
return $integers + [
'public_network' => $publicNetwork->toArray(),
];
}
private static function decodeSpec(string $encoded): array
{
$spec = json_decode($encoded, true);
if (!is_array($spec) || array_is_list($spec)) {
throw new CloudError('实例规格快照损坏', 409);
}
return $spec;
}
private static function uuidV4(): string
{
$bytes = random_bytes(16);
$bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40);
$bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80);
$hex = bin2hex($bytes);
return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-'
. substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-'
. substr($hex, 20, 12);
}
}