You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudInstanceMigrationServi...

2608 lines
105 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudBackupRestore;
use App\Models\CloudInstance;
use App\Models\CloudInstanceFirewall;
use App\Models\CloudInstanceMigration;
use App\Models\CloudJob;
use App\Models\CloudPublicIp;
use App\Models\CloudResourceReservation;
/** Durable cold migration coordinated over the PeerMigration V3 data plane. */
final class CloudInstanceMigrationService
{
private const GIB = 1073741824;
private const PROTOCOL_VERSION = 3;
private const CHUNK_SIZE = 8388608;
private const TICKET_LIFETIME_SECONDS = 604800;
/**
* Queue one plan-bound cleanup for residue left by an interrupted target
* receive. The original receive remains fenced and is never replayed; its
* ordinary automatic recovery will prove absence and repair every PHP
* placement/migration ledger after this successor succeeds.
*/
public static function ensureTargetResidueCleanup(
CloudJob $receiveJob,
array $inspection
): bool {
$cleanup = $inspection['cleanup'] ?? null;
$planId = is_array($cleanup) ? ($cleanup['plan_id'] ?? null) : null;
$steps = is_array($cleanup) ? ($cleanup['steps'] ?? null) : null;
$allowedSteps = [
'delete_managed_volumes', 'delete_seed_volume',
'undefine_network_filter', 'delete_nvram',
'remove_traffic_state',
];
if ((string) $receiveJob->kind !== 'instance_migration_transfer'
|| ($cleanup['available'] ?? false) !== true
|| !is_string($planId)
|| preg_match('/^[a-f0-9]{64}$/D', strtolower($planId)) !== 1
|| !is_array($steps) || !array_is_list($steps) || $steps === []
|| array_diff($steps, $allowedSteps) !== []
|| count(array_unique($steps)) !== count($steps)) {
return false;
}
return $receiveJob->getConnection()->transaction(
static function () use ($receiveJob, $planId, $steps): bool {
$lockedJob = CloudJob::where(
'id', (int) $receiveJob->id
)->lockForUpdate()->first();
$migration = CloudInstanceMigration::where(
'target_receive_job_id', (int) $receiveJob->id
)->lockForUpdate()->first();
$instance = $migration === null ? null : CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
$restore = $migration === null ? null : CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
if ($lockedJob === null || $migration === null
|| $instance === null || $restore === null
|| (string) $lockedJob->kind
!== 'instance_migration_transfer'
|| (string) $lockedJob->phase !== 'terminal'
|| (string) $lockedJob->outcome !== 'failed'
|| (string) $lockedJob->effect_resolution !== 'unknown'
|| (int) $lockedJob->effect_may_have_applied !== 1
|| (int) $lockedJob->safe_to_resubmit !== 0
|| (int) $lockedJob->host_id
!== (int) $migration->target_host_id
|| (int) $lockedJob->instance_id !== (int) $instance->id
|| (int) $lockedJob->backup_restore_id
!== (int) $restore->id
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| !in_array((string) $restore->state,
['restoring', 'uncertain'], true)
|| !hash_equals((string) $lockedJob->operation_id,
(string) $restore->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标残留清理账本不一致', 409
);
}
$priorCleanups = CloudJob::where(
'parent_job_id', (int) $lockedJob->id
)->where('kind', 'instance_migration_target_clean')
->orderBy('id')->lockForUpdate()->get();
foreach ($priorCleanups as $priorCleanup) {
if ((string) $priorCleanup->phase !== 'terminal'
|| (string) $priorCleanup->outcome === 'succeeded'
|| (string) $priorCleanup->effect_resolution
!== 'not_applied') {
return true;
}
}
if (count($priorCleanups) >= 3) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标残留自动清理已达到安全重试上限',
409
);
}
$generation = count($priorCleanups) + 1;
$target = '/v1/vms/'
. rawurlencode((string) $migration->target_uuid)
. '/peer-migration/cleanup-target';
$body = [
'migration_id' => (string) $migration->migration_key,
'prior_operation_id' =>
(string) $lockedJob->operation_id,
'cleanup_plan_id' => strtolower($planId),
];
[$cleanupJob, $created] = CloudJobService::
reserveStandaloneAgentMutation(
[
'user_id' => (int) $lockedJob->user_id,
'request_idempotency_key' =>
'migration-target-clean-'
. (string) $migration->migration_key
. '-' . $generation,
],
(int) $migration->target_host_id,
'instance_migration_target_clean',
'POST',
$target,
$body,
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'maintenance_migration_id' =>
(int) $migration->id,
'prior_job_id' => (int) $lockedJob->id,
'cleanup_plan_id' => strtolower($planId),
'cleanup_steps' => array_values($steps),
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'prior_job_id' => (int) $lockedJob->id,
'prior_operation_id' =>
(string) $lockedJob->operation_id,
'cleanup_plan_id' => strtolower($planId),
'cleanup_steps' => array_values($steps),
],
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id
. '/target-residue-cleanup/' . $generation
);
if (!$created) {
return true;
}
$cleanupJob->parent_job_id = (int) $lockedJob->id;
$cleanupJob->instance_id = (int) $instance->id;
$cleanupJob->backup_restore_id =
(int) $migration->restore_id;
$cleanupJob->save();
self::setState(
$migration,
'attention_required',
'target_residue_cleanup_running',
'reconcile_transfer'
);
return true;
},
3
);
}
public static function start(
CloudInstance $instance,
array $actor,
array $input
): array {
CloudAccess::manage($actor);
CloudInput::only($input, [
'target_host_id', 'cleanup_mode', 'confirm_maintenance_migration',
]);
$targetHostId = filter_var($input['target_host_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$cleanupMode = $input['cleanup_mode'] ?? null;
if ($targetHostId === false
|| !in_array($cleanupMode, ['retain', 'delete'], true)) {
throw new CloudError('请选择目标宿主机和源副本处理方式');
}
if (($input['confirm_maintenance_migration'] ?? false) !== true) {
throw new CloudError('请确认维护迁移会关闭源实例并更换公网 IP');
}
$requestKey = $actor['request_idempotency_key'] ?? null;
if (!is_string($requestKey) || $requestKey === '') {
throw new CloudError('维护迁移必须提供请求幂等标识', 400);
}
$sourceHostId = (int) $instance->host_id;
$target = '/v1/vms/' . rawurlencode((string) $instance->uuid)
. '/peer-migration/prepare';
$canonical = [
'instance_id' => (int) $instance->id,
'target_host_id' => (int) $targetHostId,
'cleanup_mode' => (string) $cleanupMode,
'confirm_maintenance_migration' => true,
];
$fingerprint = CloudJobService::standaloneRequestFingerprint(
$sourceHostId,
'instance_migration_prepare',
'POST',
$target,
$canonical
);
$prior = CloudJob::where('user_id', (int) $actor['user_id'])
->where('request_idempotency_key', $requestKey)->first();
if ($prior !== null) {
if ((string) $prior->kind !== 'instance_migration_prepare'
|| (int) $prior->instance_id !== (int) $instance->id
|| !hash_equals((string) $prior->request_fingerprint,
$fingerprint)
|| !self::ownsPrepareJob((int) $prior->id)) {
throw new CloudError('同一请求标识不能用于不同的任务', 409);
}
return ['job' => $prior];
}
$sourceCapability = self::peerCapability($sourceHostId, $actor);
$targetCapability = self::peerCapability((int) $targetHostId, $actor);
if (hash_equals($sourceCapability['identity_sha256'],
$targetCapability['identity_sha256'])) {
throw new CloudError('两台宿主机不能共用同一个 PeerMigration 自动身份', 409);
}
return $instance->getConnection()->transaction(function () use (
$instance,
$actor,
$targetHostId,
$cleanupMode,
$sourceCapability,
$target,
$canonical,
$fingerprint
): array {
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->first();
if ($locked === null || (string) $locked->status !== 'active') {
throw new CloudError('只有活动实例可以维护迁移', 409);
}
if ((int) $locked->host_id !== (int) $instance->host_id
|| !hash_equals(strtolower((string) $locked->uuid),
strtolower((string) $instance->uuid))) {
throw new CloudError('实例位置已经变化,请刷新后重试', 409);
}
$prior = CloudJob::where('user_id', (int) $actor['user_id'])
->where('request_idempotency_key',
(string) $actor['request_idempotency_key'])
->lockForUpdate()->first();
if ($prior !== null) {
if ((string) $prior->kind !== 'instance_migration_prepare'
|| (int) $prior->instance_id !== (int) $locked->id
|| !hash_equals((string) $prior->request_fingerprint,
$fingerprint)
|| !self::ownsPrepareJob((int) $prior->id)) {
throw new CloudError('同一请求标识不能用于不同的任务', 409);
}
return ['job' => $prior];
}
$spec = self::decodeSpec((string) $locked->spec_json);
if (($spec['managed'] ?? null) !== true) {
throw new CloudError('只有面板托管的实例可以维护迁移', 409);
}
if (CloudJobService::blockingQueryForInstance(
(int) $locked->id
)->lockForUpdate()->exists()) {
throw new CloudError('实例已有未完成任务', 409);
}
$snapshots = CloudPeerMigrationArtifactService::
freezeSnapshotsLocked($locked);
$resources = self::resources($spec);
self::assertStartAvailable(
$locked,
(int) $targetHostId,
$resources,
CloudPeerMigrationArtifactService::additionalDiskGib($snapshots)
);
$vpcPortId = CloudVpcPortService::migrationPortIdLocked(
$locked,
(int) $targetHostId
);
$migrationKey = bin2hex(random_bytes(16));
$targetUuid = self::uuidV4();
$restore = new CloudBackupRestore();
$restore->restore_key = bin2hex(random_bytes(16));
$restore->backup_id = null;
$restore->target_host_id = (int) $targetHostId;
$restore->target_instance_id = null;
$restore->reservation_id = null;
$restore->job_id = null;
$restore->operation_id = bin2hex(random_bytes(16));
$restore->new_uuid = $targetUuid;
$restore->new_name = (string) $locked->name;
$restore->state = 'reserved';
$restore->error_code = '';
$restore->ready_at = null;
$restore->created_at = $restore->updated_at = time();
$restore->save();
$placement = CloudPlacementService::reserveBackupRestore(
$restore,
(int) $locked->user_id,
$resources,
(int) $locked->resource_pool_id,
CloudPeerMigrationArtifactService::additionalDiskGib($snapshots)
);
$reservation = $placement['reservation'];
$reservation->instance_id = (int) $locked->id;
$reservation->updated_at = time();
$reservation->save();
$restore->reservation_id = (int) $reservation->id;
$restore->updated_at = time();
$restore->save();
$migration = new CloudInstanceMigration();
$migration->migration_key = $migrationKey;
$migration->instance_id = (int) $locked->id;
$migration->vpc_port_id = $vpcPortId;
$migration->restore_id = (int) $restore->id;
$migration->source_host_id = (int) $locked->host_id;
$migration->source_uuid = strtolower((string) $locked->uuid);
$migration->source_reservation_id = (int) $locked->reservation_id;
$migration->target_host_id = (int) $targetHostId;
$migration->target_uuid = $targetUuid;
$migration->target_reservation_id = (int) $reservation->id;
$migration->source_prepare_job_id = null;
$migration->target_receive_job_id = null;
$migration->firewall_job_id = null;
$migration->activate_job_id = null;
$migration->rollback_job_id = null;
$migration->cleanup_job_id = null;
$migration->cleanup_mode = (string) $cleanupMode;
$migration->state = 'preparing_source';
$migration->recovery_action = '';
$migration->retry_generation = 0;
$migration->source_endpoint = self::encodePeerEndpoints(
$sourceCapability['endpoints']
);
$migration->peer_trust_id =
$sourceCapability['identity_sha256'];
$migration->manifest_digest = '';
$migration->source_was_running = null;
$migration->source_autostart_enabled = null;
$migration->source_service_status = (string) $locked->service_status;
$migration->source_network_status = (string) $locked->network_status;
$migration->error_code = '';
$migration->cutover_at = null;
$migration->completed_at = null;
$migration->created_at = $migration->updated_at = time();
$migration->save();
CloudPeerMigrationArtifactService::persistLocked(
$migration,
$snapshots,
$resources['disk_gib'] * self::GIB
);
$ticket = bin2hex(random_bytes(32));
$expiresAt = time() + self::TICKET_LIFETIME_SECONDS;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
$actor,
(int) $locked->host_id,
'instance_migration_prepare',
'POST',
$target,
[
'migration_id' => $migrationKey,
'ticket' => $ticket,
'expires_at' => $expiresAt,
'artifacts' => CloudPeerMigrationArtifactService::
sourceRequestArtifacts($migration),
],
[
'owner_id' => (int) $locked->user_id,
'can_manage' => true,
'maintenance_migration_id' => (int) $migration->id,
],
$canonical,
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/prepare'
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'维护迁移源准备任务幂等冲突', 409
);
}
$job->instance_id = (int) $locked->id;
$job->save();
$migration->source_prepare_job_id = (int) $job->id;
$migration->updated_at = time();
$migration->save();
return ['job' => $job];
}, 3);
}
public static function assertStartAvailable(
CloudInstance $instance,
int $targetHostId,
array $resources,
int $snapshotDiskGib = 0,
?int $excludeMigrationId = null
): void {
if (!in_array((string) $instance->service_status,
['active', 'exempt', 'expired_grace', 'suspended_expired'], true)
|| !in_array((string) $instance->network_status,
['active', 'blocked_traffic', 'blocked_expired'], true)) {
throw new CloudError('实例服务或网络状态正在变化,暂不能维护迁移', 409);
}
if ($targetHostId === (int) $instance->host_id) {
throw new CloudError('维护迁移必须选择其他宿主机', 409);
}
if ((int) ($instance->reservation_id ?? 0) < 1
|| (int) ($instance->order_id ?? 0) < 1
|| (int) ($instance->resource_pool_id ?? 0) < 1) {
throw new CloudError('只有具备完整订单和资源预留的实例可以维护迁移', 409);
}
$query = CloudInstanceMigration::where(
'instance_id', (int) $instance->id
)->whereIn('state', array_merge(
CloudInstanceMigrationWorkflow::blockingStates(),
['source_retained']
));
if ($excludeMigrationId !== null) {
$query->where('id', '<>', $excludeMigrationId);
}
if ($query->exists()) {
throw new CloudError('实例已有未完成迁移或待清理源副本', 409);
}
$allowed = array_filter(
CloudPlacementService::maintenanceMigrationTargets(
$instance,
$resources,
$snapshotDiskGib
),
static fn (array $target): bool =>
(int) ($target['id'] ?? 0) === $targetHostId
);
if ($allowed === []) {
throw new CloudError(
'目标宿主机必须位于同一资源池并具备可用容量、br0 和公网 IP',
409
);
}
CloudFirewallService::assertMaintenanceMigrationReady($instance);
}
public static function display(CloudInstanceMigration $migration): array
{
$state = (string) $migration->state;
$action = (string) $migration->recovery_action;
$hasAbortEvidence = self::hasAbortPrepareEvidence($migration);
$recoveryOptions = CloudInstanceMigrationWorkflow::recoveryOptions(
$migration,
$hasAbortEvidence
);
$display = [
'id' => (string) $migration->id,
'state' => $state,
'cleanup_mode' => (string) $migration->cleanup_mode,
'source_host_id' => (string) $migration->source_host_id,
'target_host_id' => (string) $migration->target_host_id,
'source_retained' => $state === 'source_retained',
'can_cleanup' => $state === 'source_retained'
&& $action !== 'reconcile_cleanup',
'recovery_action' => $action === '' ? null : $action,
'can_recover' => $recoveryOptions !== [],
'recovery_actions' => array_column(
$recoveryOptions,
'action'
),
'recovery_options' => $recoveryOptions,
'progress' => CloudInstanceMigrationWorkflow::progress(
$migration
),
'error_code' => (string) $migration->error_code,
'cutover_at' => $migration->cutover_at === null
? null : (int) $migration->cutover_at,
'completed_at' => $migration->completed_at === null
? null : (int) $migration->completed_at,
'created_at' => (int) $migration->created_at,
'updated_at' => (int) $migration->updated_at,
];
foreach (CloudInstanceMigrationWorkflow::stages() as $stage) {
if ($stage['task_key'] === null) {
continue;
}
$field = (string) $stage['job_field'];
$display[(string) $stage['task_key']] =
$migration->{$field} === null
? null : (string) $migration->{$field};
}
return $display;
}
private static function hasAbortPrepareEvidence(
CloudInstanceMigration $migration
): bool {
try {
self::abortPrepareEvidence($migration);
return true;
} catch (\Throwable $error) {
return false;
}
}
/**
* @return array{job:CloudJob,event:CloudAgentCompletionEvent,result:array}
*/
private static function abortPrepareEvidence(
CloudInstanceMigration $migration
): array {
if ((string) $migration->state !== 'attention_required'
|| !in_array((string) $migration->recovery_action,
['reconcile_prepare', 'abort_prepare'], true)
|| $migration->target_receive_job_id !== null
|| $migration->firewall_job_id !== null
|| $migration->activate_job_id !== null
|| $migration->rollback_job_id !== null
|| $migration->cleanup_job_id !== null
|| $migration->cutover_at !== null) {
throw new CloudRecoveryInstanceStateConflict(
'当前迁移不处于可安全中止的源准备阶段', 409
);
}
$job = CloudJob::find((int) $migration->source_prepare_job_id);
if ($job === null
|| (string) $job->kind !== 'instance_migration_prepare'
|| (string) $job->execution_mode !== 'agent'
|| (int) $job->instance_id !== (int) $migration->instance_id
|| (int) $job->host_id !== (int) $migration->source_host_id
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (int) $job->dispatch_attempts !== 1
|| !CloudDispatchLedger::isDispatched($job)
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown') {
throw new CloudRecoveryInstanceStateConflict(
'源准备任务不处于可补偿的效果核对状态', 409
);
}
$event = CloudOperationCompletionService::pendingForJob($job);
if ($event === null
|| !CloudOperationCompletionService::isExactStoredEvent($event)
|| (string) $event->outcome !== 'succeeded'
|| (bool) $event->effect_may_have_applied
|| (bool) $event->safe_to_resubmit
|| ($job->completion_event_id !== null
&& !hash_equals(
(string) $job->completion_event_id,
(string) $event->event_id
))) {
throw new CloudRecoveryInstanceStateConflict(
'源准备任务尚无可安全补偿的精确成功回执', 409
);
}
CloudOperationCompletionService::assertEventIdentityForJob(
$job,
$event
);
$operation = CloudOperationCompletionService::operation($event);
$result = $operation['result'] ?? null;
self::assertAbortPrepareResult($migration, $result);
return ['job' => $job, 'event' => $event, 'result' => $result];
}
private static function assertAbortPrepareResult(
CloudInstanceMigration $migration,
$result
): void {
if (!is_array($result) || array_is_list($result)
|| !hash_equals(
(string) $migration->migration_key,
(string) ($result['migration_id'] ?? '')
)
|| ($result['source_stopped'] ?? null) !== true
|| ($result['source_autostart_disabled'] ?? null) !== true
|| !is_bool($result['source_was_running'] ?? null)
|| !is_bool($result['source_autostart_enabled'] ?? null)
|| !is_string($result['manifest_digest'] ?? null)
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $result['manifest_digest']) !== 1
|| !is_array($result['artifacts'] ?? null)
|| !array_is_list($result['artifacts'])
|| $result['artifacts'] === []) {
throw new CloudRecoveryReceiptInvalid(
'源准备成功回执缺少可安全恢复实例的证据', 409
);
}
}
public static function ownsPrepareJob(int $jobId): bool
{
return self::catalogAvailable()
&& CloudInstanceMigration::where(
'source_prepare_job_id', $jobId
)->exists();
}
public static function hasOpenMigration(int $instanceId): bool
{
return $instanceId > 0
&& self::catalogAvailable()
&& CloudInstanceMigration::where('instance_id', $instanceId)
->whereIn(
'state',
CloudInstanceMigrationWorkflow::blockingStates()
)->exists();
}
public static function assertInstanceMutationAvailable(
CloudInstance $instance,
?string $operationKind = null
): void {
if (self::hasOpenMigration((int) $instance->id)) {
throw new CloudError('实例正在维护迁移,请先等待迁移收敛', 409);
}
if ($operationKind !== null
&& in_array($operationKind, [
'delete', 'delete_force', 'expiry_delete',
], true)
&& self::catalogAvailable()
&& CloudInstanceMigration::where(
'instance_id', (int) $instance->id
)->where('state', 'source_retained')->exists()) {
throw new CloudError(
'迁移源副本尚未释放,请先完成迁移清理后再删除实例',
409
);
}
}
public static function finishJobLocked(
CloudJob $job,
string $status,
$result,
array $payload
): void {
if (!self::catalogAvailable()) {
return;
}
$stage = CloudInstanceMigrationWorkflow::job((string) $job->kind);
if ($stage === null) {
return;
}
$field = (string) $stage['job_field'];
$migration = CloudInstanceMigration::where($field, (int) $job->id)
->lockForUpdate()->first();
if ($migration === null) {
return;
}
$handler = (string) $stage['finish_handler'];
self::{$handler}($migration, $job, $status, $result, $payload);
}
public static function markJobUncertain(CloudJob $job): void
{
if (!self::catalogAvailable()) {
return;
}
$stage = CloudInstanceMigrationWorkflow::job((string) $job->kind);
if ($stage === null) {
return;
}
$field = (string) $stage['job_field'];
$migration = CloudInstanceMigration::where($field, (int) $job->id)
->lockForUpdate()->first();
if ($migration === null) {
return;
}
$action = (string) $stage['reconcile_action'];
if ((string) $job->kind === 'instance_migration_prepare'
&& (string) $migration->recovery_action === 'abort_prepare') {
$action = 'abort_prepare';
}
if ((string) $stage['key'] === 'cleanup') {
CloudInstanceMigrationSettlement::retainSourceLocked($migration);
self::setState($migration, 'source_retained',
'finalization_incident', $action);
return;
}
self::setState($migration, 'attention_required',
'finalization_incident', $action);
}
/**
* Join the multi-host VPC projection back into the migration state
* machine. Every host job carries the same durable migration generation;
* only the last converged result may advance transfer or source rollback.
*/
public static function finishVpcRoutingJobLocked(
CloudJob $job,
string $status,
array $payload
): void {
$migrationId = filter_var(
$payload['maintenance_migration_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$direction = $payload['vpc_migration_direction'] ?? null;
if ($migrationId === false
|| !in_array($direction, ['target', 'source'], true)
|| !in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 宿主机任务缺少完整编排身份', 409
);
}
$reference = CloudInstanceMigration::where(
'id', (int) $migrationId
)->first();
$instance = $reference === null ? null : CloudInstance::where(
'id', (int) $reference->instance_id
)->lockForUpdate()->first();
$migration = $instance === null ? null : CloudInstanceMigration::where(
'id', (int) $migrationId
)->lockForUpdate()->first();
if ($migration === null || $instance === null
|| $migration->vpc_port_id === null
|| (int) $migration->vpc_port_id !== (int) $job->vpc_port_id
|| (int) $migration->instance_id !== (int) $job->instance_id
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(
strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid)
)) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 任务与实例位置账本不一致', 409
);
}
$runningState = $direction === 'target'
? 'vpc_target_routing' : 'vpc_source_routing';
$reconcileAction = $direction === 'target'
? 'reconcile_vpc_target' : 'reconcile_vpc_source';
$failureAction = $direction === 'target'
? 'retry_vpc_target' : 'retry_vpc_source';
if ($direction === 'target'
&& ((string) $migration->state === 'transferring'
|| $migration->target_receive_job_id !== null)) {
return;
}
if ($direction === 'source'
&& ((string) $migration->state === 'rollback_running'
|| $migration->rollback_job_id !== null)) {
return;
}
if ((string) $migration->state !== $runningState
&& !((string) $migration->state === 'attention_required'
&& in_array((string) $migration->recovery_action, [
$reconcileAction, $failureAction,
], true))) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 任务与当前迁移阶段不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState(
$migration,
'attention_required',
'operation_effect_unknown',
$reconcileAction
);
return;
}
if ($status === 'failed') {
self::setState(
$migration,
'attention_required',
'migration_vpc_routing_failed',
$failureAction
);
return;
}
if ($status !== 'succeeded') {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 任务终态无效', 409
);
}
if (!CloudVpcPortService::finishMigrationProjectionLocked(
$migration,
$instance,
$direction
)) {
return;
}
if ($direction === 'target') {
$prepare = CloudJob::find(
(int) $migration->source_prepare_job_id
);
if ($prepare === null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 收敛后源准备任务缺失', 409
);
}
$preparePayload = CloudSecrets::decrypt(
(string) $prepare->sealed_payload,
'job-payload'
);
if (!is_array($preparePayload)) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 收敛后源准备载荷无效', 409
);
}
self::queueTransferLocked(
$migration,
$prepare,
$preparePayload
);
return;
}
self::queueRollbackLocked(
$migration,
$instance,
(int) $job->user_id,
'migration_transfer_aborted'
);
}
public static function recover(
CloudInstance $instance,
array $actor,
array $input
): CloudJob {
CloudAccess::manage($actor);
CloudInput::only($input, ['action', 'confirm_recovery']);
$action = $input['action'] ?? null;
if (!is_string($action)
|| !CloudInstanceMigrationWorkflow::isRecoveryAction($action)
|| ($input['confirm_recovery'] ?? false) !== true) {
throw new CloudError('请选择并确认可用的迁移恢复操作');
}
if ($action === 'abort_prepare') {
return self::recoverAbortPrepare($instance);
}
return $instance->getConnection()->transaction(function () use (
$instance,
$actor,
$action
): CloudJob {
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->firstOrFail();
$migration = CloudInstanceMigration::where(
'instance_id', (int) $locked->id
)->where('state', 'attention_required')->orderByDesc('id')
->lockForUpdate()->first();
if ($migration === null
|| !in_array(
$action,
CloudInstanceMigrationWorkflow::recoveryActions(
$migration,
false
),
true
)) {
throw new CloudError('当前迁移没有该恢复出口', 409);
}
$handler = CloudInstanceMigrationWorkflow::recoveryHandler(
$action
);
if ($handler === null || !method_exists(self::class, $handler)) {
throw new \LogicException(
'迁移恢复动作缺少可执行处理方法'
);
}
$migration->retry_generation = (int) $migration->retry_generation + 1;
$migration->updated_at = time();
$migration->save();
return self::{$handler}(
$migration,
$locked,
(int) $actor['user_id']
);
}, 3);
}
private static function recoverAbortPrepare(
CloudInstance $instance
): CloudJob
{
$candidate = CloudInstanceMigration::where(
'instance_id', (int) $instance->id
)->orderByDesc('id')->first();
if ($candidate !== null
&& (string) $candidate->state === 'rollback_running'
&& (string) $candidate->error_code
=== 'migration_prepare_aborted'
&& $candidate->rollback_job_id !== null) {
$existing = CloudJob::find((int) $candidate->rollback_job_id);
if ($existing !== null
&& (string) $existing->kind
=== 'instance_migration_rollback') {
return $existing;
}
}
if ($candidate === null) {
throw new CloudError('当前没有可中止的迁移源准备任务', 409);
}
$evidence = self::abortPrepareEvidence($candidate);
if (!in_array(
'abort_prepare',
CloudInstanceMigrationWorkflow::recoveryActions(
$candidate,
true
),
true
)) {
throw new CloudError('当前迁移没有该恢复出口', 409);
}
$migrationId = (int) $candidate->id;
$prepareJobId = (int) $evidence['job']->id;
$eventId = (string) $evidence['event']->event_id;
$instance->getConnection()->transaction(function () use (
$instance,
$migrationId,
$prepareJobId
): void {
// Keep worker lock order: prepare finalization owns job ->
// migration -> instance. This intent transaction deliberately does
// not lock the job or receipt.
$migration = CloudInstanceMigration::where('id', $migrationId)
->lockForUpdate()->first();
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->first();
if ($migration === null || $locked === null
|| (int) $migration->source_prepare_job_id !== $prepareJobId
|| (int) $migration->instance_id !== (int) $locked->id
|| (string) $migration->state !== 'attention_required'
|| !in_array((string) $migration->recovery_action,
['reconcile_prepare', 'abort_prepare'], true)
|| $migration->target_receive_job_id !== null
|| $migration->rollback_job_id !== null
|| (int) $locked->host_id !== (int) $migration->source_host_id
|| !hash_equals(
strtolower((string) $locked->uuid),
strtolower((string) $migration->source_uuid)
)
|| (int) $locked->reservation_id
!== (int) $migration->source_reservation_id) {
throw new CloudRecoveryInstanceStateConflict(
'迁移源准备状态已经变化,不能中止', 409
);
}
if ((string) $migration->recovery_action
=== 'reconcile_prepare') {
$migration->retry_generation =
(int) $migration->retry_generation + 1;
$migration->recovery_action = 'abort_prepare';
$migration->updated_at = time();
$migration->save();
}
}, 3);
$prepare = CloudJob::find($prepareJobId);
$event = CloudAgentCompletionEvent::find($eventId);
if ($prepare === null || $event === null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移源准备成功回执在中止期间发生变化', 409
);
}
CloudOperationRecoveryService::settleLateReceipt($prepare, $event);
$settled = CloudInstanceMigration::find($migrationId);
$rollback = $settled === null || $settled->rollback_job_id === null
? null : CloudJob::find((int) $settled->rollback_job_id);
if ($rollback === null
|| (string) $rollback->kind !== 'instance_migration_rollback') {
throw new CloudRecoveryInstanceStateConflict(
'迁移中止已进入自动核对,请稍后刷新', 409
);
}
return $rollback;
}
public static function cleanupRetained(
CloudInstance $instance,
array $actor,
array $input
): CloudJob {
CloudAccess::manage($actor);
CloudInput::only($input, ['confirm_source_cleanup']);
if (($input['confirm_source_cleanup'] ?? false) !== true) {
throw new CloudError('请确认删除源宿主机上的旧副本');
}
return $instance->getConnection()->transaction(function () use (
$instance,
$actor
): CloudJob {
$locked = CloudInstance::where('id', (int) $instance->id)
->lockForUpdate()->firstOrFail();
$migration = CloudInstanceMigration::where(
'instance_id', (int) $locked->id
)->where('state', 'source_retained')->where(
'recovery_action', '<>', 'reconcile_cleanup'
)->orderByDesc('id')->lockForUpdate()->first();
if ($migration === null) {
throw new CloudError('当前没有可安全清理的迁移源副本', 409);
}
$migration->retry_generation = (int) $migration->retry_generation + 1;
$migration->cleanup_job_id = null;
$migration->save();
return self::queueCleanupLocked($migration, $locked, $actor);
}, 3);
}
private static function finishPrepareLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
if ((int) $migration->instance_id !== (int) $job->instance_id
|| (int) $migration->source_host_id !== (int) $job->host_id
|| (int) ($payload['maintenance_migration_id'] ?? 0)
!== (int) $migration->id) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源准备任务与迁移账本不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_prepare');
return;
}
if ($status === 'failed') {
if ((bool) ($job->effect_may_have_applied ?? false)) {
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_prepare');
return;
}
self::releasePreparedTargetLocked($migration,
'migration_prepare_failed');
self::setState($migration, 'rolled_back',
'migration_prepare_failed');
$migration->completed_at = time();
$migration->save();
return;
}
if (!is_array($result)
|| !hash_equals((string) $migration->migration_key,
(string) ($result['migration_id'] ?? ''))
|| !is_bool($result['source_was_running'] ?? null)
|| !is_bool($result['source_autostart_enabled'] ?? null)
|| !is_string($result['manifest_digest'] ?? null)
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $result['manifest_digest']) !== 1) {
throw new CloudError('源 Agent 的 PeerMigration 回执无效', 502);
}
$migration->source_was_running = $result['source_was_running'];
$migration->source_autostart_enabled =
$result['source_autostart_enabled'];
$migration->manifest_digest = (string) $result['manifest_digest'];
if ((string) $migration->recovery_action === 'abort_prepare') {
self::finishAbortPrepareLocked($migration, $job, $result);
return;
}
CloudPeerMigrationArtifactService::finishPrepareLocked(
$migration,
$result
);
if ($migration->target_receive_job_id !== null) {
return;
}
if ($migration->vpc_port_id !== null) {
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($instance === null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 路由切换时实例账本缺失', 409
);
}
CloudVpcPortService::beginMigrationToTargetLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_target_routing', '');
return;
}
self::queueTransferLocked($migration, $job, $payload);
}
private static function finishAbortPrepareLocked(
CloudInstanceMigration $migration,
CloudJob $prepareJob,
array $result
): void {
self::assertAbortPrepareResult($migration, $result);
if ((string) $migration->state !== 'attention_required'
|| (string) $migration->recovery_action !== 'abort_prepare'
|| $migration->target_receive_job_id !== null
|| $migration->firewall_job_id !== null
|| $migration->activate_job_id !== null
|| $migration->rollback_job_id !== null
|| $migration->cleanup_job_id !== null
|| $migration->cutover_at !== null) {
throw new CloudRecoveryInstanceStateConflict(
'迁移源准备中止状态已经变化', 409
);
}
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
$reservation = CloudResourceReservation::where(
'id', (int) $migration->target_reservation_id
)->lockForUpdate()->first();
$ip = $reservation === null ? null : CloudPublicIp::where(
'id', (int) $reservation->public_ip_id
)->lockForUpdate()->first();
if ($instance === null || $restore === null
|| $reservation === null || $ip === null
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(
strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid)
)
|| (int) $instance->reservation_id
!== (int) $migration->source_reservation_id
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| (int) $restore->reservation_id !== (int) $reservation->id
|| $restore->backup_id !== null
|| $restore->job_id !== null
|| (string) $restore->state !== 'reserved'
|| (int) $reservation->backup_restore_id !== (int) $restore->id
|| (int) $reservation->host_id !== (int) $migration->target_host_id
|| (int) $reservation->instance_id !== (int) $instance->id
|| (string) $reservation->state !== 'held'
|| (int) $reservation->public_ip_id !== (int) $ip->id
|| (string) $ip->state !== 'reserved'
|| (int) ($ip->reservation_id ?? 0) !== (int) $reservation->id
|| $ip->instance_id !== null
|| !hash_equals(
(string) $restore->operation_id,
(string) $reservation->operation_id
)
|| !hash_equals(
(string) $reservation->operation_id,
(string) ($ip->operation_id ?? '')
)) {
throw new CloudRecoveryInstanceStateConflict(
'迁移中止前的源实例或目标预留账本已经变化', 409
);
}
$migration->source_was_running = $result['source_was_running'];
$migration->source_autostart_enabled =
$result['source_autostart_enabled'];
$migration->manifest_digest = (string) $result['manifest_digest'];
$migration->updated_at = time();
$migration->save();
self::queueRollbackLocked(
$migration,
$instance,
(int) $prepareJob->user_id,
'migration_prepare_aborted'
);
}
private static function queueTransferLocked(
CloudInstanceMigration $migration,
CloudJob $prepareJob,
array $preparePayload
): void {
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
if ($instance === null || $restore === null
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))
|| (int) $instance->reservation_id
!== (int) $migration->source_reservation_id
|| !hash_equals((string) $migration->source_service_status,
(string) $instance->service_status)
|| !hash_equals((string) $migration->source_network_status,
(string) $instance->network_status)
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| (int) $restore->reservation_id
!== (int) $migration->target_reservation_id
|| (string) $restore->state !== 'reserved') {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源实例或目标预留在停机期间发生变化', 409
);
}
$ticket = $preparePayload['body']['ticket'] ?? null;
if (!is_string($ticket)
|| preg_match('/^[a-f0-9]{64}$/D', $ticket) !== 1) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源任务缺少加密传输票据', 409
);
}
$spec = self::decodeSpec((string) $instance->spec_json);
$resources = self::resources($spec);
$placement = CloudPlacementService::preparedBackupRestorePlacement(
$restore,
(int) $instance->id
);
$reservation = $placement['reservation'];
$network = $placement['network'];
$policy = $spec['platform_network_policy_snapshot'] ?? [];
if (!is_array($policy) || array_is_list($policy)) {
$policy = [];
}
$vpcPort = CloudVpcPortService::migrationTargetMetadataLocked(
$migration,
$instance
);
$body = [
'migration_id' => (string) $migration->migration_key,
'ticket' => $ticket,
'source_endpoints' => self::decodePeerEndpoints(
(string) $migration->source_endpoint
),
'source_identity_sha256' =>
(string) $migration->peer_trust_id,
'source_uuid' => (string) $migration->source_uuid,
'manifest_digest' => (string) $migration->manifest_digest,
'network_suspended' =>
(string) $migration->source_network_status !== 'active',
'debian_12_network_fix' =>
($spec['debian_12_network_fix'] ?? null) === true,
'uuid' => (string) $migration->target_uuid,
'source' => '',
'name' => (string) $instance->name,
'description' => '',
'vcpu' => $resources['cpu_cores'],
'memory_mib' => $resources['memory_mib'],
'disk_gib' => $resources['disk_gib'],
'bandwidth_mbps' => $resources['bandwidth_mbps'],
'network' => 'bridge:br0',
'template_id' => '',
'template_revision' => 0,
'template_generation_token' => '',
'startup_script' => '',
'hostname' => CloudInput::hostname((string) (
$spec['guest_hostname'] ?? $instance->name
)),
'guest_ssh_public_key' => '',
'root_password' => '',
'public_pool_key' => (string) $network['public_pool_key'],
'public_pool_cidr' => (string) $network['public_pool_cidr'],
'public_ip_cidr' => (string) $network['public_ip_cidr'],
'gateway' => (string) $network['gateway'],
'dns_servers' => array_values($network['dns_servers']),
'platform_network_policy_snapshot' => $policy,
];
if ($vpcPort !== null) {
$body['vpc_port'] = $vpcPort;
}
$generation = (int) $migration->retry_generation;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
[
'user_id' => (int) $prepareJob->user_id,
'request_idempotency_key' => 'migration-transfer-'
. (string) $migration->migration_key . '-' . $generation,
],
(int) $migration->target_host_id,
'instance_migration_transfer',
'POST',
'/v1/vms/' . rawurlencode((string) $migration->target_uuid)
. '/peer-migration/receive',
$body,
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'reservation_id' => (int) $reservation->id,
'maintenance_migration_id' => (int) $migration->id,
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'target_host_id' => (int) $migration->target_host_id,
'target_uuid' => (string) $migration->target_uuid,
'manifest_digest' => (string) $migration->manifest_digest,
],
(string) $restore->operation_id,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/transfer/' . $generation
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标接收任务幂等冲突', 409
);
}
$job->instance_id = (int) $instance->id;
$job->backup_restore_id = (int) $restore->id;
$job->save();
$restore->job_id = (int) $job->id;
$restore->state = 'restoring';
$restore->error_code = '';
$restore->updated_at = time();
$restore->save();
$migration->target_receive_job_id = (int) $job->id;
self::setState($migration, 'transferring', '');
}
private static function finishTransferLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($restore === null || $instance === null
|| (int) $migration->target_receive_job_id !== (int) $job->id
|| (int) $job->backup_restore_id !== (int) $restore->id
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->host_id !== (int) $migration->target_host_id
|| (int) ($payload['maintenance_migration_id'] ?? 0)
!== (int) $migration->id
|| !hash_equals((string) $restore->operation_id,
(string) $job->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标接收任务与迁移账本不一致', 409
);
}
if ($status === 'running') {
return;
}
$requestedIp = $payload['body']['public_ip_cidr'] ?? null;
if (!is_string($requestedIp)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标接收缺少公网 IP 快照', 409
);
}
$now = time();
if ($status === 'uncertain') {
CloudPlacementService::finishBackupRestore(
$restore, (int) $instance->id, $requestedIp, 'uncertain'
);
$restore->state = 'uncertain';
$restore->error_code = 'operation_effect_unknown';
$restore->updated_at = $now;
$restore->save();
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_transfer');
return;
}
if ($status === 'failed') {
self::restoreTargetReservationForRetryLocked(
$migration, $restore, $instance, $requestedIp
);
$restore->state = 'reserved';
$restore->error_code = 'peer_transfer_failed';
$restore->updated_at = $now;
$restore->save();
self::setState($migration, 'attention_required',
'peer_transfer_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'transfer'
));
return;
}
if (!is_array($result)
|| !hash_equals((string) $migration->migration_key,
(string) ($result['migration_id'] ?? ''))
|| !hash_equals((string) $migration->manifest_digest,
(string) ($result['manifest_digest'] ?? ''))
|| !hash_equals(strtolower((string) $migration->target_uuid),
strtolower((string) ($result['uuid'] ?? '')))
|| (string) ($result['status'] ?? '') !== 'shutoff'
|| !self::sameDocument(
$result['vpc_port'] ?? null,
$payload['body']['vpc_port'] ?? null
)) {
throw new CloudError('目标 Agent 的 PeerMigration 回执无效', 502);
}
$rootDiskGib = filter_var(
$payload['body']['disk_gib'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$target = CloudResourceReservation::where(
'id', (int) $migration->target_reservation_id
)->lockForUpdate()->first();
$source = CloudResourceReservation::where(
'id', (int) $migration->source_reservation_id
)->lockForUpdate()->first();
$snapshotDiskGib = CloudPeerMigrationArtifactService::artifactDiskGib(
$migration
);
$expectedTargetClaim = $rootDiskGib === false ? 0
: (int) $rootDiskGib + $snapshotDiskGib;
if ($rootDiskGib === false || $target === null || $source === null
|| !in_array((string) $target->state,
['held', 'uncertain'], true)
|| (int) $target->disk_gib !== $expectedTargetClaim
|| (string) $source->state !== 'committed'
|| (int) $source->host_id !== (int) $migration->source_host_id
|| (int) $source->instance_id !== (int) $instance->id
|| (int) $source->disk_gib !== (int) $rootDiskGib
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 切换前资源预留账本不一致', 409
);
}
$targetSpec = self::finalizeTargetSpec(
self::decodeSpec((string) $instance->spec_json),
is_array($payload['body'] ?? null) ? $payload['body'] : [],
$result,
(int) $target->resource_pool_id
);
CloudPeerMigrationArtifactService::finishReceiveLocked(
$migration, $result
);
// Snapshot rows now account for their target volumes. The target
// reservation therefore keeps only the root disk; while the source
// duplicate exists its original reservation accounts for root plus
// every frozen snapshot.
$target->disk_gib = (int) $rootDiskGib;
$target->updated_at = $now;
$target->save();
$source->disk_gib = $expectedTargetClaim;
$source->updated_at = $now;
$source->save();
CloudPlacementService::finishBackupRestore(
$restore, (int) $instance->id, $requestedIp, 'succeeded'
);
$target = CloudResourceReservation::where(
'id', (int) $migration->target_reservation_id
)->lockForUpdate()->first();
if ($target === null || (string) $target->state !== 'committed') {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标资源没有完成提交', 409
);
}
$spec = $targetSpec;
$spec['managed'] = true;
$spec['migrated_from_host_id'] = (int) $migration->source_host_id;
$spec['migrated_from_uuid'] = (string) $migration->source_uuid;
$spec['maintenance_migration_key'] =
(string) $migration->migration_key;
$spec['traffic_counter_generation'] = [
'host_id' => (int) $migration->target_host_id,
'uuid' => strtolower((string) $migration->target_uuid),
'download_base_bytes' =>
max(0, (int) $instance->lifetime_download_bytes),
'upload_base_bytes' =>
max(0, (int) $instance->lifetime_upload_bytes),
];
CloudVpcPortService::finalizeMigrationCutoverLocked(
$migration,
$instance
);
$instance->host_id = (int) $migration->target_host_id;
$instance->uuid = strtolower((string) $migration->target_uuid);
$instance->resource_pool_id = (int) $target->resource_pool_id;
$instance->reservation_id = (int) $target->id;
$instance->spec_json = json_encode(
$spec,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
$instance->service_status = (string) $migration->source_service_status;
$instance->network_status = (string) $migration->source_network_status;
$instance->traffic_synced_at = 0;
$instance->updated_at = $now;
$instance->save();
$restore->state = 'ready';
$restore->error_code = '';
$restore->ready_at = $now;
$restore->updated_at = $now;
$restore->save();
$migration->cutover_at = $now;
self::setState($migration, 'cutover', '');
CloudInstanceRuntimeService::seed(
(int) $instance->id,
(int) $instance->host_id,
(string) $instance->uuid
);
$firewall = CloudFirewallService::queueMaintenanceMigrationReapplyLocked(
$instance,
$migration,
(int) $job->user_id,
(int) $migration->retry_generation
);
if ($firewall !== null) {
$migration->firewall_job_id = (int) $firewall->id;
self::setState($migration, 'firewall_running', '');
return;
}
self::advanceAfterTargetReadyLocked(
$migration, $instance, (int) $job->user_id
);
}
private static function restoreTargetReservationForRetryLocked(
CloudInstanceMigration $migration,
CloudBackupRestore $restore,
CloudInstance $instance,
string $publicIpCidr
): void {
$reservation = CloudResourceReservation::where(
'id', (int) $migration->target_reservation_id
)->lockForUpdate()->first();
$ip = $reservation === null ? null : CloudPublicIp::where(
'id', (int) $reservation->public_ip_id
)->lockForUpdate()->first();
if ($reservation === null || $ip === null
|| (int) $reservation->backup_restore_id !== (int) $restore->id
|| (int) $reservation->instance_id !== (int) $instance->id
|| !hash_equals((string) $reservation->operation_id,
(string) $restore->operation_id)
|| !hash_equals((string) $ip->address . '/'
. (int) $ip->prefix_length, $publicIpCidr)
|| (int) ($ip->reservation_id ?? 0) !== (int) $reservation->id
|| !hash_equals((string) $reservation->operation_id,
(string) ($ip->operation_id ?? ''))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 重试目标预留账本不一致', 409
);
}
if ((string) $reservation->state === 'held'
&& (string) $ip->state === 'reserved') {
return;
}
if ((string) $reservation->state !== 'uncertain'
|| (string) $ip->state !== 'uncertain') {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标预留不能恢复为可重试状态', 409
);
}
$reservation->state = 'held';
$reservation->updated_at = time();
$reservation->save();
$ip->state = 'reserved';
$ip->instance_id = null;
$ip->updated_at = time();
$ip->save();
}
/** Release a target held before the source mutation was dispatched. */
private static function releasePreparedTargetLocked(
CloudInstanceMigration $migration,
string $errorCode
): void {
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
if ($restore === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 缺少目标预留目录', 409
);
}
if ((string) $restore->state === 'failed') {
return;
}
$placement = CloudPlacementService::preparedBackupRestorePlacement(
$restore, (int) $migration->instance_id
);
CloudPlacementService::finishBackupRestore(
$restore,
(int) $migration->instance_id,
(string) $placement['network']['public_ip_cidr'],
'failed'
);
$restore->state = 'failed';
$restore->error_code = $errorCode;
$restore->updated_at = time();
$restore->save();
}
private static function retryTransferLocked(
CloudInstanceMigration $migration,
CloudInstance $instance
): CloudJob {
if ((int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 传输重试时源实例位置已经变化', 409
);
}
$prepare = CloudJob::find((int) $migration->source_prepare_job_id);
if ($prepare === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源准备任务目录缺失', 409
);
}
$payload = CloudSecrets::decrypt(
(string) $prepare->sealed_payload, 'job-payload'
);
if (!is_array($payload)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源准备任务载荷无效', 409
);
}
self::rotateTargetTransferOperationLocked($migration, $instance);
$migration->target_receive_job_id = null;
$migration->updated_at = time();
$migration->save();
self::queueTransferLocked($migration, $prepare, $payload);
$job = CloudJob::find((int) $migration->target_receive_job_id);
if ($job === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 重试任务目录缺失', 409
);
}
return $job;
}
private static function recoverRetryTransferLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
unset($actorUserId);
return self::retryTransferLocked($migration, $instance);
}
private static function recoverRetryVpcTargetLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
unset($actorUserId);
$jobs = CloudVpcPortService::beginMigrationToTargetLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_target_routing', '');
return self::firstVpcRoutingJob($jobs);
}
private static function recoverAbortVpcTargetLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
unset($actorUserId);
$jobs = CloudVpcPortService::beginMigrationToSourceLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_source_routing', '');
return self::firstVpcRoutingJob($jobs);
}
private static function recoverRetryVpcSourceLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
unset($actorUserId);
$jobs = CloudVpcPortService::beginMigrationToSourceLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_source_routing', '');
return self::firstVpcRoutingJob($jobs);
}
private static function recoverAbortTransferLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
if ($migration->vpc_port_id !== null) {
$jobs = CloudVpcPortService::beginMigrationToSourceLocked(
$migration,
$instance
);
self::setState($migration, 'vpc_source_routing', '');
return self::firstVpcRoutingJob($jobs);
}
return self::queueRollbackLocked(
$migration,
$instance,
$actorUserId,
'migration_transfer_aborted'
);
}
/** @param list<CloudJob> $jobs */
private static function firstVpcRoutingJob(array $jobs): CloudJob
{
$job = $jobs[0] ?? null;
if (!$job instanceof CloudJob) {
throw new CloudRecoveryInstanceStateConflict(
'迁移 VPC 路由切换没有生成可跟踪任务', 409
);
}
return $job;
}
/**
* A confirmed not-applied transfer may be retried only under a fresh
* Agent operation ID. Rotate the restore, reservation and IP ledgers as
* one locked unit so the prior Journal result cannot be replayed as the
* new attempt.
*/
private static function rotateTargetTransferOperationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance
): void {
$restore = CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
$reservation = CloudResourceReservation::where(
'id', (int) $migration->target_reservation_id
)->lockForUpdate()->first();
$ip = $reservation === null ? null : CloudPublicIp::where(
'id', (int) $reservation->public_ip_id
)->lockForUpdate()->first();
if ($restore === null || $reservation === null || $ip === null
|| (string) $restore->state !== 'reserved'
|| (int) $restore->reservation_id !== (int) $reservation->id
|| (int) $reservation->backup_restore_id !== (int) $restore->id
|| (int) $reservation->instance_id !== (int) $instance->id
|| (string) $reservation->state !== 'held'
|| (int) ($ip->reservation_id ?? 0) !== (int) $reservation->id
|| (string) $ip->state !== 'reserved'
|| !hash_equals((string) $restore->operation_id,
(string) $reservation->operation_id)
|| !hash_equals((string) $restore->operation_id,
(string) ($ip->operation_id ?? ''))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 传输重试账本不一致', 409
);
}
$operationId = bin2hex(random_bytes(16));
$now = time();
$restore->operation_id = $operationId;
$restore->job_id = null;
$restore->error_code = '';
$restore->updated_at = $now;
$restore->save();
$reservation->operation_id = $operationId;
$reservation->updated_at = $now;
$reservation->save();
$ip->operation_id = $operationId;
$ip->updated_at = $now;
$ip->save();
}
private static function queueRollbackLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId,
string $errorCode
): CloudJob {
if ((int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))
|| $migration->source_was_running === null
|| $migration->source_autostart_enabled === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源实例回滚账本不完整', 409
);
}
$generation = (int) $migration->retry_generation;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
[
'user_id' => $actorUserId,
'request_idempotency_key' => 'migration-rollback-'
. (string) $migration->migration_key . '-' . $generation,
],
(int) $migration->source_host_id,
'instance_migration_rollback',
'POST',
'/v1/vms/' . rawurlencode((string) $migration->source_uuid)
. '/peer-migration/rollback',
[
'migration_id' => (string) $migration->migration_key,
'source_was_running' => (bool) $migration->source_was_running,
'source_autostart_enabled' =>
(bool) $migration->source_autostart_enabled,
],
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'maintenance_migration_id' => (int) $migration->id,
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'source_host_id' => (int) $migration->source_host_id,
'source_uuid' => (string) $migration->source_uuid,
'source_was_running' =>
(bool) $migration->source_was_running,
'source_autostart_enabled' =>
(bool) $migration->source_autostart_enabled,
],
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/rollback/' . $generation
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源回滚任务幂等冲突', 409
);
}
$job->instance_id = (int) $instance->id;
$job->backup_restore_id = (int) $migration->restore_id;
$job->save();
$migration->rollback_job_id = (int) $job->id;
self::setState($migration, 'rollback_running', $errorCode);
return $job;
}
private static function retryRollbackLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
$migration->rollback_job_id = null;
$migration->updated_at = time();
$migration->save();
return self::queueRollbackLocked(
$migration, $instance, $actorUserId,
'migration_source_rollback_failed'
);
}
private static function recoverRetryRollbackLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
return self::retryRollbackLocked(
$migration,
$instance,
$actorUserId
);
}
private static function finishRollbackLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
unset($payload);
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($instance === null
|| (int) $migration->rollback_job_id !== (int) $job->id
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->host_id !== (int) $migration->source_host_id
|| (int) $instance->host_id !== (int) $migration->source_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->source_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源回滚任务与实例不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_rollback');
return;
}
if ($status === 'failed') {
self::setState($migration, 'attention_required',
'migration_source_rollback_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'rollback'
));
return;
}
if (!is_array($result)
|| !hash_equals(strtolower((string) $migration->source_uuid),
strtolower((string) ($result['uuid'] ?? '')))
|| !is_bool($result['running'] ?? null)
|| !is_bool($result['autostart_enabled'] ?? null)
|| $result['running'] !== (bool) $migration->source_was_running
|| $result['autostart_enabled']
!== (bool) $migration->source_autostart_enabled) {
throw new CloudError(
'源 Agent 未确认实例恢复到迁移前电源和自启动状态', 502
);
}
$abortCode = $migration->target_receive_job_id === null
? 'migration_prepare_aborted'
: 'migration_transfer_aborted';
self::releasePreparedTargetLocked($migration, $abortCode);
self::setState($migration, 'rolled_back', $abortCode);
$migration->completed_at = time();
$migration->save();
}
private static function finishFirewallLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
unset($result, $payload);
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($instance === null
|| (int) $migration->firewall_job_id !== (int) $job->id
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->host_id !== (int) $migration->target_host_id
|| (int) $instance->host_id !== (int) $migration->target_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->target_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 防火墙任务与目标实例不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState($migration, 'attention_required',
'firewall_result_uncertain', 'reconcile_firewall');
return;
}
if ($status === 'failed') {
self::setState($migration, 'attention_required',
'firewall_apply_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'firewall'
));
return;
}
$policy = CloudInstanceFirewall::where(
'instance_id', (int) $instance->id
)->lockForUpdate()->first();
if ($policy === null
|| (int) $policy->host_id !== (int) $instance->host_id
|| (string) $policy->sync_state !== 'active'
|| (int) $policy->desired_revision
!== (int) $policy->applied_revision) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标防火墙没有完成收敛', 409
);
}
self::advanceAfterTargetReadyLocked(
$migration, $instance, (int) $job->user_id
);
}
private static function retryFirewallLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
$migration->firewall_job_id = null;
$migration->updated_at = time();
$migration->save();
$job = CloudFirewallService::queueMaintenanceMigrationReapplyLocked(
$instance,
$migration,
$actorUserId,
(int) $migration->retry_generation
);
if ($job === null) {
self::advanceAfterTargetReadyLocked(
$migration, $instance, $actorUserId
);
$next = $migration->activate_job_id === null
? $migration->cleanup_job_id : $migration->activate_job_id;
$job = $next === null ? null : CloudJob::find((int) $next);
} else {
$migration->firewall_job_id = (int) $job->id;
self::setState($migration, 'firewall_running', '');
}
if ($job === null) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 防火墙恢复后没有后续任务', 409
);
}
return $job;
}
private static function recoverRetryFirewallLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
return self::retryFirewallLocked(
$migration,
$instance,
$actorUserId
);
}
private static function advanceAfterTargetReadyLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): void {
self::queueActivationLocked($migration, $instance, $actorUserId);
}
private static function queueActivationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
$generation = (int) $migration->retry_generation;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
[
'user_id' => $actorUserId,
'request_idempotency_key' => 'migration-activate-'
. (string) $migration->migration_key . '-' . $generation,
],
(int) $migration->target_host_id,
'instance_migration_activate',
'POST',
'/v1/vms/' . rawurlencode((string) $migration->target_uuid)
. '/peer-migration/activate',
[
'migration_id' => (string) $migration->migration_key,
'target_was_running' =>
(bool) $migration->source_was_running,
'target_autostart_enabled' =>
(bool) $migration->source_autostart_enabled,
],
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'maintenance_migration_id' => (int) $migration->id,
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'target_host_id' => (int) $migration->target_host_id,
'target_uuid' => (string) $migration->target_uuid,
'source_was_running' => (bool) $migration->source_was_running,
'source_autostart_enabled' =>
(bool) $migration->source_autostart_enabled,
],
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/activate/' . $generation
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标激活任务幂等冲突', 409
);
}
$job->instance_id = (int) $instance->id;
$job->backup_restore_id = (int) $migration->restore_id;
$job->save();
$migration->activate_job_id = (int) $job->id;
self::setState($migration, 'activation_running', '');
return $job;
}
private static function retryActivationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
$migration->activate_job_id = null;
$migration->updated_at = time();
$migration->save();
return self::queueActivationLocked(
$migration, $instance, $actorUserId
);
}
private static function recoverRetryActivationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): CloudJob {
return self::retryActivationLocked(
$migration,
$instance,
$actorUserId
);
}
private static function finishActivationLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
unset($payload);
$instance = CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
if ($instance === null
|| (int) $migration->activate_job_id !== (int) $job->id
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->host_id !== (int) $migration->target_host_id
|| (int) $instance->host_id !== (int) $migration->target_host_id
|| !hash_equals(strtolower((string) $instance->uuid),
strtolower((string) $migration->target_uuid))) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 激活任务与目标实例不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
self::setState($migration, 'attention_required',
'operation_effect_unknown', 'reconcile_activation');
return;
}
if ($status === 'failed') {
self::setState($migration, 'attention_required',
'migration_target_activation_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'activation'
));
return;
}
if (!is_array($result)
|| !hash_equals(strtolower((string) $migration->target_uuid),
strtolower((string) ($result['uuid'] ?? '')))
|| !is_bool($result['running'] ?? null)
|| !is_bool($result['autostart_enabled'] ?? null)
|| $result['running'] !== (bool) $migration->source_was_running
|| $result['autostart_enabled']
!== (bool) $migration->source_autostart_enabled) {
throw new CloudError(
'目标 Agent 未确认实例恢复到迁移前电源和自启动状态', 502
);
}
self::advanceAfterActivationLocked(
$migration, $instance, (int) $job->user_id
);
}
private static function advanceAfterActivationLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
int $actorUserId
): void {
if ((string) $migration->cleanup_mode === 'delete') {
self::queueCleanupLocked($migration, $instance, [
'user_id' => $actorUserId,
'request_idempotency_key' => 'migration-cleanup-'
. (string) $migration->migration_key . '-'
. (int) $migration->retry_generation,
]);
return;
}
CloudInstanceMigrationSettlement::retainSourceLocked($migration);
self::setState($migration, 'source_retained', '');
}
private static function queueCleanupLocked(
CloudInstanceMigration $migration,
CloudInstance $instance,
array $actor
): CloudJob {
if (!in_array((string) $migration->state,
['cutover', 'activation_running', 'source_retained'], true)) {
throw new CloudError('当前迁移状态不能清理源副本', 409);
}
$generation = (int) $migration->retry_generation;
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
$actor,
(int) $migration->source_host_id,
'instance_migration_cleanup',
'POST',
'/v1/vms/' . rawurlencode((string) $migration->source_uuid)
. '/peer-migration/cleanup',
[
'migration_id' => (string) $migration->migration_key,
'remove_managed_disks' => true,
],
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'maintenance_migration_id' => (int) $migration->id,
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'source_host_id' => (int) $migration->source_host_id,
'source_uuid' => (string) $migration->source_uuid,
],
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id . '/source/' . $generation
);
if (!$created) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源清理任务幂等冲突', 409
);
}
$job->instance_id = (int) $instance->id;
$job->backup_restore_id = (int) $migration->restore_id;
$job->save();
$migration->cleanup_job_id = (int) $job->id;
self::setState($migration, 'cleanup_running', '');
return $job;
}
private static function finishCleanupLocked(
CloudInstanceMigration $migration,
CloudJob $job,
string $status,
$result,
array $payload
): void {
unset($payload);
if ((int) $migration->cleanup_job_id !== (int) $job->id
|| (int) $job->instance_id !== (int) $migration->instance_id
|| (int) $job->host_id !== (int) $migration->source_host_id) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 源清理任务与迁移账本不一致', 409
);
}
if ($status === 'running') {
return;
}
if ($status === 'uncertain') {
CloudInstanceMigrationSettlement::retainSourceLocked($migration);
self::setState($migration, 'source_retained',
'operation_effect_unknown', 'reconcile_cleanup');
return;
}
if ($status === 'failed') {
CloudInstanceMigrationSettlement::retainSourceLocked($migration);
self::setState($migration, 'source_retained',
'migration_cleanup_failed');
return;
}
if (!is_array($result)
|| ($result['deleted'] ?? null) !== true
|| ($result['managed_artifacts_absent'] ?? null) !== true) {
throw new CloudError(
'源 Agent 未确认实例和托管磁盘均已删除', 502
);
}
if (CloudInstanceMigrationSettlement::isSettled($migration)) {
CloudInstanceMigrationSettlement::releaseRetainedSourceLocked(
$migration
);
} else {
CloudInstanceMigrationSettlement::finalizeSourceDeletionLocked(
$migration
);
}
self::setState($migration, 'completed', '');
$migration->completed_at = time();
$migration->save();
}
private static function setState(
CloudInstanceMigration $migration,
string $state,
string $errorCode,
string $recoveryAction = ''
): void {
CloudInstanceMigrationWorkflow::assertState(
$state,
$recoveryAction
);
$migration->state = $state;
$migration->error_code = $errorCode;
$migration->recovery_action = $recoveryAction;
$migration->updated_at = time();
$migration->save();
}
private static function sameDocument($left, $right): bool
{
$canonical = static function ($value) use (&$canonical) {
if (!is_array($value)) {
return $value;
}
if (!array_is_list($value)) {
ksort($value, SORT_STRING);
}
foreach ($value as $key => $child) {
$value[$key] = $canonical($child);
}
return $value;
};
return hash_equals(
hash('sha256', json_encode(
$canonical($left),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)),
hash('sha256', json_encode(
$canonical($right),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
))
);
}
/** @return array{endpoints:list<string>,identity_sha256:string} */
private static function peerCapability(int $hostId, array $actor): array
{
$capabilities = CloudAgentSettingsService::clientForHost(
$hostId, 30.0
)->request('GET', '/v1/capabilities', $actor);
$peer = $capabilities['peer_migration'] ?? null;
$features = $capabilities['features'] ?? null;
$endpoints = is_array($peer) ? ($peer['endpoints'] ?? null) : null;
$identity = is_array($peer)
? ($peer['identity_sha256'] ?? null) : null;
$version = is_array($peer) ? filter_var(
$peer['version'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
) : false;
$chunk = is_array($peer) ? filter_var(
$peer['chunk_size_bytes'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
) : false;
if (!is_array($features)
|| ($features['peer_migration'] ?? null) !== true
|| !is_array($peer)
|| $version !== self::PROTOCOL_VERSION
|| ($peer['pinned_tls'] ?? null) !== true
|| $chunk !== self::CHUNK_SIZE
|| !self::validPeerEndpoints($endpoints)
|| !is_string($identity)
|| preg_match('/^[a-f0-9]{64}$/D', $identity) !== 1) {
throw new CloudError(
'宿主机未就绪 PeerMigration V3、TLS 1.3 身份锁定或固定分块能力',
409
);
}
return [
'endpoints' => array_values($endpoints),
'identity_sha256' => $identity,
];
}
private static function validPeerEndpoints(mixed $values): bool
{
if (!is_array($values) || !array_is_list($values)
|| count($values) < 1 || count($values) > 16) {
return false;
}
$seen = [];
foreach ($values as $value) {
if (!is_string($value) || !self::validPeerEndpoint($value)
|| isset($seen[$value])) {
return false;
}
$seen[$value] = true;
}
return true;
}
private static function validPeerEndpoint(string $value): bool
{
if ($value === '' || strlen($value) > 2048
|| trim($value) !== $value
|| preg_match('/[\\x00\\r\\n\\\\]/', $value) === 1) {
return false;
}
$parts = parse_url($value);
$host = is_array($parts) ? ($parts['host'] ?? null) : null;
if (!is_string($host)) {
return false;
}
$literal = str_starts_with($host, '[') && str_ends_with($host, ']')
? substr($host, 1, -1) : $host;
return is_array($parts)
&& ($parts['scheme'] ?? null) === 'https'
&& filter_var($literal, FILTER_VALIDATE_IP) !== false
&& filter_var($parts['port'] ?? null, FILTER_VALIDATE_INT,
['options' => ['min_range' => 1, 'max_range' => 65535]])
!== false
&& !isset($parts['user'], $parts['pass'], $parts['query'],
$parts['fragment'])
&& (!isset($parts['path']) || $parts['path'] === ''
|| $parts['path'] === '/');
}
/** @param list<string> $endpoints */
private static function encodePeerEndpoints(array $endpoints): string
{
if (!self::validPeerEndpoints($endpoints)) {
throw new CloudError('源宿主机的 PeerMigration 地址无效', 409);
}
$encoded = json_encode(array_values($endpoints),
JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
if (strlen($encoded) > 2048) {
throw new CloudError('源宿主机的 PeerMigration 地址过多', 409);
}
return $encoded;
}
/** @return list<string> */
private static function decodePeerEndpoints(string $encoded): array
{
try {
$endpoints = json_decode($encoded, true, 16,
JSON_THROW_ON_ERROR);
} catch (\Throwable) {
throw new CloudRecoveryPayloadUnavailable(
'PeerMigration 源地址目录无法读取', 409
);
}
if (!self::validPeerEndpoints($endpoints)) {
throw new CloudRecoveryPayloadUnavailable(
'PeerMigration 源地址目录无效', 409
);
}
return array_values($endpoints);
}
private static function catalogAvailable(): bool
{
try {
$schema = (new CloudInstanceMigration())->getConnection()
->getSchemaBuilder();
return $schema->hasTable('cloud_instance_migrations')
&& $schema->hasTable('cloud_peer_migration_artifacts');
} catch (\Throwable $error) {
return false;
}
}
private static function resources(array $spec): array
{
$values = [
'cpu_cores' => $spec['vcpu'] ?? $spec['cpu_cores'] ?? null,
'memory_mib' => $spec['memory_mib'] ?? $spec['ram_mib'] ?? null,
'disk_gib' => $spec['disk_gib'] ?? null,
'bandwidth_mbps' => $spec['bandwidth_mbps'] ?? null,
];
foreach ($values as $field => $value) {
$parsed = filter_var($value, FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]);
if ($parsed === false) {
throw new CloudError('实例缺少可信的资源规格快照', 409);
}
$values[$field] = (int) $parsed;
}
return $values;
}
private static function finalizeTargetSpec(
array $source,
array $body,
array $vm,
int $resourcePoolId
): array {
if ($resourcePoolId < 1) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标资源池快照无效', 409
);
}
$expected = self::targetRestoreSnapshot($body);
try {
$actual = self::targetRestoreSnapshot($vm);
} catch (CloudError $error) {
throw new CloudError(
'目标 Agent 返回的资源或网络快照不一致', 502
);
}
if ($expected !== $actual) {
throw new CloudError(
'目标 Agent 返回的资源或网络快照不一致', 502
);
}
$publicIp = explode('/', $expected['public_ip_cidr'], 2)[0];
$spec = CloudInstanceMetadata::merge($source, $vm);
return array_replace($spec, [
'vcpu' => $expected['vcpu'],
'cpu_cores' => $expected['vcpu'],
'memory_mib' => $expected['memory_mib'],
'ram_mib' => $expected['memory_mib'],
'disk_gib' => $expected['disk_gib'],
'bandwidth_mbps' => $expected['bandwidth_mbps'],
'network' => $expected['network'],
'public_pool_key' => $expected['public_pool_key'],
'public_pool_cidr' => $expected['public_pool_cidr'],
'public_ip_cidr' => $expected['public_ip_cidr'],
'public_ip' => $publicIp,
'gateway' => $expected['gateway'],
'dns_servers' => $expected['dns_servers'],
'resource_pool_id' => $resourcePoolId,
]);
}
/** @return array<string,mixed> */
private static function targetRestoreSnapshot(array $value): array
{
$integers = [];
foreach ([
'vcpu' => [1, 512],
'memory_mib' => [128, 16 * 1024 * 1024],
'disk_gib' => [1, 4096],
'bandwidth_mbps' => [1, 100000],
] as $field => [$minimum, $maximum]) {
$parsed = filter_var($value[$field] ?? null,
FILTER_VALIDATE_INT, ['options' => [
'min_range' => $minimum, 'max_range' => $maximum,
]]);
if ($parsed === false) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标资源规格快照无效', 409
);
}
$integers[$field] = (int) $parsed;
}
$network = $value['network'] ?? null;
$poolKey = $value['public_pool_key'] ?? null;
$poolCidr = $value['public_pool_cidr'] ?? null;
$publicIpCidr = $value['public_ip_cidr'] ?? null;
$gateway = $value['gateway'] ?? null;
$dns = $value['dns_servers'] ?? null;
if ($network !== 'bridge:br0'
|| !is_string($poolKey)
|| preg_match('/^[a-f0-9]{32}$/D', $poolKey) !== 1
|| !is_string($poolCidr)
|| preg_match('/^[0-9.]+\/(?:[1-9]|[12][0-9]|30)$/D',
$poolCidr) !== 1
|| !is_string($publicIpCidr)
|| !is_string($gateway)
|| filter_var($gateway, FILTER_VALIDATE_IP,
FILTER_FLAG_IPV4) === false
|| !is_array($dns) || !array_is_list($dns)
|| $dns === [] || count($dns) > 4) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标网络规格快照无效', 409
);
}
try {
$publicIpCidr = CloudInput::publicIpCidr($publicIpCidr);
} catch (CloudError $error) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标网络规格快照无效', 409
);
}
$poolParts = explode('/', $poolCidr, 2);
if (filter_var($poolParts[0], FILTER_VALIDATE_IP,
FILTER_FLAG_IPV4) === false
|| count(array_unique($dns, SORT_STRING)) !== count($dns)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标网络规格快照无效', 409
);
}
foreach ($dns as $server) {
if (!is_string($server)
|| filter_var($server, FILTER_VALIDATE_IP,
FILTER_FLAG_IPV4) === false) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标网络规格快照无效', 409
);
}
}
return $integers + [
'network' => $network,
'public_pool_key' => $poolKey,
'public_pool_cidr' => $poolCidr,
'public_ip_cidr' => $publicIpCidr,
'gateway' => $gateway,
'dns_servers' => array_values($dns),
];
}
private static function decodeSpec(string $encoded): array
{
$spec = json_decode($encoded, true);
if (!is_array($spec) || array_is_list($spec)) {
throw new CloudError('实例规格快照损坏', 409);
}
return $spec;
}
private static function uuidV4(): string
{
$bytes = random_bytes(16);
$bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40);
$bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80);
$hex = bin2hex($bytes);
return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-'
. substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-'
. substr($hex, 20, 12);
}
}