You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2844 lines
114 KiB
PHP
2844 lines
114 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudAgentCompletionEvent;
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceFirewall;
|
|
use App\Models\CloudInstanceMigration;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudResourceReservation;
|
|
|
|
/** Durable cold migration coordinated over the PeerMigration V3 data plane. */
|
|
final class CloudInstanceMigrationService
|
|
{
|
|
private const GIB = 1073741824;
|
|
private const PROTOCOL_VERSION = 3;
|
|
private const CHUNK_SIZE = 8388608;
|
|
private const TICKET_LIFETIME_SECONDS = 604800;
|
|
/**
|
|
* Queue one plan-bound cleanup for residue left by an interrupted target
|
|
* receive. The original receive remains fenced and is never replayed; its
|
|
* ordinary automatic recovery will prove absence and repair every PHP
|
|
* placement/migration ledger after this successor succeeds.
|
|
*/
|
|
public static function ensureTargetResidueCleanup(
|
|
CloudJob $receiveJob,
|
|
array $inspection
|
|
): bool {
|
|
$cleanup = $inspection['cleanup'] ?? null;
|
|
$planId = is_array($cleanup) ? ($cleanup['plan_id'] ?? null) : null;
|
|
$steps = is_array($cleanup) ? ($cleanup['steps'] ?? null) : null;
|
|
$allowedSteps = [
|
|
'delete_managed_volumes', 'delete_seed_volume',
|
|
'undefine_network_filter', 'delete_nvram',
|
|
'remove_traffic_state',
|
|
];
|
|
if ((string) $receiveJob->kind !== 'instance_migration_transfer'
|
|
|| ($cleanup['available'] ?? false) !== true
|
|
|| !is_string($planId)
|
|
|| preg_match('/^[a-f0-9]{64}$/D', strtolower($planId)) !== 1
|
|
|| !is_array($steps) || !array_is_list($steps) || $steps === []
|
|
|| array_diff($steps, $allowedSteps) !== []
|
|
|| count(array_unique($steps)) !== count($steps)) {
|
|
return false;
|
|
}
|
|
return $receiveJob->getConnection()->transaction(
|
|
static function () use ($receiveJob, $planId, $steps): bool {
|
|
$lockedJob = CloudJob::where(
|
|
'id', (int) $receiveJob->id
|
|
)->lockForUpdate()->first();
|
|
$migration = CloudInstanceMigration::where(
|
|
'target_receive_job_id', (int) $receiveJob->id
|
|
)->lockForUpdate()->first();
|
|
$instance = $migration === null ? null : CloudInstance::where(
|
|
'id', (int) $migration->instance_id
|
|
)->lockForUpdate()->first();
|
|
$restore = $migration === null ? null : CloudBackupRestore::where(
|
|
'id', (int) $migration->restore_id
|
|
)->lockForUpdate()->first();
|
|
if ($lockedJob === null || $migration === null
|
|
|| $instance === null || $restore === null
|
|
|| (string) $lockedJob->kind
|
|
!== 'instance_migration_transfer'
|
|
|| (string) $lockedJob->phase !== 'terminal'
|
|
|| (string) $lockedJob->outcome !== 'failed'
|
|
|| (string) $lockedJob->effect_resolution !== 'unknown'
|
|
|| (int) $lockedJob->effect_may_have_applied !== 1
|
|
|| (int) $lockedJob->safe_to_resubmit !== 0
|
|
|| (int) $lockedJob->host_id
|
|
!== (int) $migration->target_host_id
|
|
|| (int) $lockedJob->instance_id !== (int) $instance->id
|
|
|| (int) $lockedJob->backup_restore_id
|
|
!== (int) $restore->id
|
|
|| (int) $restore->target_host_id
|
|
!== (int) $migration->target_host_id
|
|
|| !in_array((string) $restore->state,
|
|
['restoring', 'uncertain'], true)
|
|
|| !hash_equals((string) $lockedJob->operation_id,
|
|
(string) $restore->operation_id)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标残留清理账本不一致', 409
|
|
);
|
|
}
|
|
$priorCleanups = CloudJob::where(
|
|
'parent_job_id', (int) $lockedJob->id
|
|
)->where('kind', 'instance_migration_target_clean')
|
|
->orderBy('id')->lockForUpdate()->get();
|
|
foreach ($priorCleanups as $priorCleanup) {
|
|
if ((string) $priorCleanup->phase !== 'terminal'
|
|
|| (string) $priorCleanup->outcome === 'succeeded'
|
|
|| (string) $priorCleanup->effect_resolution
|
|
!== 'not_applied') {
|
|
return true;
|
|
}
|
|
}
|
|
if (count($priorCleanups) >= 3) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标残留自动清理已达到安全重试上限',
|
|
409
|
|
);
|
|
}
|
|
$generation = count($priorCleanups) + 1;
|
|
$target = '/v1/vms/'
|
|
. rawurlencode((string) $migration->target_uuid)
|
|
. '/peer-migration/cleanup-target';
|
|
$body = [
|
|
'migration_id' => (string) $migration->migration_key,
|
|
'prior_operation_id' =>
|
|
(string) $lockedJob->operation_id,
|
|
'cleanup_plan_id' => strtolower($planId),
|
|
];
|
|
[$cleanupJob, $created] = CloudJobService::
|
|
reserveStandaloneAgentMutation(
|
|
[
|
|
'user_id' => (int) $lockedJob->user_id,
|
|
'request_idempotency_key' =>
|
|
'migration-target-clean-'
|
|
. (string) $migration->migration_key
|
|
. '-' . $generation,
|
|
],
|
|
(int) $migration->target_host_id,
|
|
'instance_migration_target_clean',
|
|
'POST',
|
|
$target,
|
|
$body,
|
|
[
|
|
'owner_id' => (int) $instance->user_id,
|
|
'can_manage' => true,
|
|
'maintenance_migration_id' =>
|
|
(int) $migration->id,
|
|
'prior_job_id' => (int) $lockedJob->id,
|
|
'cleanup_plan_id' => strtolower($planId),
|
|
'cleanup_steps' => array_values($steps),
|
|
],
|
|
[
|
|
'migration_id' => (int) $migration->id,
|
|
'generation' => $generation,
|
|
'prior_job_id' => (int) $lockedJob->id,
|
|
'prior_operation_id' =>
|
|
(string) $lockedJob->operation_id,
|
|
'cleanup_plan_id' => strtolower($planId),
|
|
'cleanup_steps' => array_values($steps),
|
|
],
|
|
null,
|
|
'/v1/instance-maintenance-migrations/'
|
|
. (int) $migration->id
|
|
. '/target-residue-cleanup/' . $generation
|
|
);
|
|
if (!$created) {
|
|
return true;
|
|
}
|
|
$cleanupJob->parent_job_id = (int) $lockedJob->id;
|
|
$cleanupJob->instance_id = (int) $instance->id;
|
|
$cleanupJob->backup_restore_id =
|
|
(int) $migration->restore_id;
|
|
$cleanupJob->save();
|
|
self::setState(
|
|
$migration,
|
|
'attention_required',
|
|
'target_residue_cleanup_running',
|
|
'reconcile_transfer'
|
|
);
|
|
return true;
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
public static function start(
|
|
CloudInstance $instance,
|
|
array $actor,
|
|
array $input
|
|
): array {
|
|
CloudAccess::manage($actor);
|
|
CloudInput::only($input, [
|
|
'target_host_id', 'target_resource_pool_id', 'cleanup_mode',
|
|
'confirm_maintenance_migration',
|
|
]);
|
|
$targetHostId = filter_var($input['target_host_id'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$targetResourcePoolId = filter_var(
|
|
$input['target_resource_pool_id'] ?? null,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
);
|
|
$cleanupMode = $input['cleanup_mode'] ?? null;
|
|
if ($targetHostId === false || $targetResourcePoolId === false
|
|
|| !in_array($cleanupMode, ['retain', 'delete'], true)) {
|
|
throw new CloudError(
|
|
'请选择目标宿主机、资源池和源副本处理方式'
|
|
);
|
|
}
|
|
if (($input['confirm_maintenance_migration'] ?? false) !== true) {
|
|
throw new CloudError(
|
|
'请确认维护迁移会关闭源实例并更换公网 IPv4 和 IPv6'
|
|
);
|
|
}
|
|
$requestKey = $actor['request_idempotency_key'] ?? null;
|
|
if (!is_string($requestKey) || $requestKey === '') {
|
|
throw new CloudError('维护迁移必须提供请求幂等标识', 400);
|
|
}
|
|
$sourceHostId = (int) $instance->host_id;
|
|
$target = '/v1/vms/' . rawurlencode((string) $instance->uuid)
|
|
. '/peer-migration/prepare';
|
|
$canonical = [
|
|
'instance_id' => (int) $instance->id,
|
|
'target_host_id' => (int) $targetHostId,
|
|
'target_resource_pool_id' => (int) $targetResourcePoolId,
|
|
'cleanup_mode' => (string) $cleanupMode,
|
|
'confirm_maintenance_migration' => true,
|
|
];
|
|
$fingerprint = CloudJobService::standaloneRequestFingerprint(
|
|
$sourceHostId,
|
|
'instance_migration_prepare',
|
|
'POST',
|
|
$target,
|
|
$canonical
|
|
);
|
|
$prior = CloudJob::where('user_id', (int) $actor['user_id'])
|
|
->where('request_idempotency_key', $requestKey)->first();
|
|
if ($prior !== null) {
|
|
if ((string) $prior->kind !== 'instance_migration_prepare'
|
|
|| (int) $prior->instance_id !== (int) $instance->id
|
|
|| !hash_equals((string) $prior->request_fingerprint,
|
|
$fingerprint)
|
|
|| !self::ownsPrepareJob((int) $prior->id)) {
|
|
throw new CloudError('同一请求标识不能用于不同的任务', 409);
|
|
}
|
|
return ['job' => $prior];
|
|
}
|
|
|
|
$sourceCapability = self::peerCapability($sourceHostId, $actor);
|
|
$targetCapability = self::peerCapability((int) $targetHostId, $actor);
|
|
if (hash_equals($sourceCapability['identity_sha256'],
|
|
$targetCapability['identity_sha256'])) {
|
|
throw new CloudError('两台宿主机不能共用同一个 PeerMigration 自动身份', 409);
|
|
}
|
|
|
|
return $instance->getConnection()->transaction(function () use (
|
|
$instance,
|
|
$actor,
|
|
$targetHostId,
|
|
$targetResourcePoolId,
|
|
$cleanupMode,
|
|
$sourceCapability,
|
|
$target,
|
|
$canonical,
|
|
$fingerprint
|
|
): array {
|
|
$locked = CloudInstance::where('id', (int) $instance->id)
|
|
->lockForUpdate()->first();
|
|
if ($locked === null || (string) $locked->status !== 'active') {
|
|
throw new CloudError('只有活动实例可以维护迁移', 409);
|
|
}
|
|
if ((int) $locked->host_id !== (int) $instance->host_id
|
|
|| !hash_equals(strtolower((string) $locked->uuid),
|
|
strtolower((string) $instance->uuid))) {
|
|
throw new CloudError('实例位置已经变化,请刷新后重试', 409);
|
|
}
|
|
$prior = CloudJob::where('user_id', (int) $actor['user_id'])
|
|
->where('request_idempotency_key',
|
|
(string) $actor['request_idempotency_key'])
|
|
->lockForUpdate()->first();
|
|
if ($prior !== null) {
|
|
if ((string) $prior->kind !== 'instance_migration_prepare'
|
|
|| (int) $prior->instance_id !== (int) $locked->id
|
|
|| !hash_equals((string) $prior->request_fingerprint,
|
|
$fingerprint)
|
|
|| !self::ownsPrepareJob((int) $prior->id)) {
|
|
throw new CloudError('同一请求标识不能用于不同的任务', 409);
|
|
}
|
|
return ['job' => $prior];
|
|
}
|
|
$spec = self::decodeSpec((string) $locked->spec_json);
|
|
if (($spec['managed'] ?? null) !== true) {
|
|
throw new CloudError('只有面板托管的实例可以维护迁移', 409);
|
|
}
|
|
if (CloudJobService::blockingQueryForInstance(
|
|
(int) $locked->id
|
|
)->lockForUpdate()->exists()) {
|
|
throw new CloudError('实例已有未完成任务', 409);
|
|
}
|
|
$snapshots = CloudPeerMigrationArtifactService::
|
|
freezeSnapshotsLocked($locked);
|
|
$resources = self::resources($spec);
|
|
self::assertStartAvailable(
|
|
$locked,
|
|
(int) $targetHostId,
|
|
(int) $targetResourcePoolId,
|
|
$resources,
|
|
CloudPeerMigrationArtifactService::additionalDiskGib($snapshots)
|
|
);
|
|
$vpcPortId = CloudVpcPortService::migrationPortIdLocked(
|
|
$locked,
|
|
(int) $targetHostId
|
|
);
|
|
|
|
$migrationKey = bin2hex(random_bytes(16));
|
|
$targetUuid = self::uuidV4();
|
|
$restore = new CloudBackupRestore();
|
|
$restore->restore_key = bin2hex(random_bytes(16));
|
|
$restore->backup_id = null;
|
|
$restore->target_host_id = (int) $targetHostId;
|
|
$restore->target_instance_id = null;
|
|
$restore->reservation_id = null;
|
|
$restore->job_id = null;
|
|
$restore->operation_id = bin2hex(random_bytes(16));
|
|
$restore->new_uuid = $targetUuid;
|
|
$restore->new_name = (string) $locked->name;
|
|
$restore->state = 'reserved';
|
|
$restore->error_code = '';
|
|
$restore->ready_at = null;
|
|
$restore->created_at = $restore->updated_at = time();
|
|
$restore->save();
|
|
$placement = CloudPlacementService::
|
|
reserveMaintenanceMigrationTarget(
|
|
$restore,
|
|
(int) $locked->user_id,
|
|
$resources,
|
|
(int) $targetResourcePoolId,
|
|
CloudPeerMigrationArtifactService::additionalDiskGib($snapshots)
|
|
);
|
|
$sourceReservation = CloudResourceReservation::where(
|
|
'id',
|
|
(int) $locked->reservation_id
|
|
)->lockForUpdate()->first();
|
|
if ($sourceReservation === null
|
|
|| (string) $sourceReservation->state !== 'committed'
|
|
|| (int) $sourceReservation->host_id !== (int) $locked->host_id
|
|
|| (int) $sourceReservation->instance_id !== (int) $locked->id
|
|
|| (int) $sourceReservation->resource_pool_id
|
|
!== (int) $locked->resource_pool_id
|
|
|| (int) ($sourceReservation->order_id ?? 0)
|
|
!== (int) $locked->order_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源资源预留与实例所属资源池不一致',
|
|
409
|
|
);
|
|
}
|
|
CloudPublicNetworkAllocationService::currentClaimSet(
|
|
(int) $sourceReservation->id,
|
|
'committed',
|
|
(int) $locked->id
|
|
);
|
|
$reservation = $placement['reservation'];
|
|
$restore->reservation_id = (int) $reservation->id;
|
|
$restore->updated_at = time();
|
|
$restore->save();
|
|
|
|
$migration = new CloudInstanceMigration();
|
|
$migration->migration_key = $migrationKey;
|
|
$migration->instance_id = (int) $locked->id;
|
|
$migration->vpc_port_id = $vpcPortId;
|
|
$migration->restore_id = (int) $restore->id;
|
|
$migration->source_host_id = (int) $locked->host_id;
|
|
$migration->source_uuid = strtolower((string) $locked->uuid);
|
|
$migration->source_reservation_id = (int) $locked->reservation_id;
|
|
$migration->target_host_id = (int) $targetHostId;
|
|
$migration->target_uuid = $targetUuid;
|
|
$migration->target_reservation_id = (int) $reservation->id;
|
|
$migration->source_prepare_job_id = null;
|
|
$migration->target_receive_job_id = null;
|
|
$migration->firewall_job_id = null;
|
|
$migration->activate_job_id = null;
|
|
$migration->rollback_job_id = null;
|
|
$migration->cleanup_job_id = null;
|
|
$migration->cleanup_mode = (string) $cleanupMode;
|
|
$migration->state = 'preparing_source';
|
|
$migration->recovery_action = '';
|
|
$migration->retry_generation = 0;
|
|
$migration->source_endpoint = self::encodePeerEndpoints(
|
|
$sourceCapability['endpoints']
|
|
);
|
|
$migration->peer_trust_id =
|
|
$sourceCapability['identity_sha256'];
|
|
$migration->manifest_digest = '';
|
|
$migration->source_was_running = null;
|
|
$migration->source_autostart_enabled = null;
|
|
$migration->source_service_status = (string) $locked->service_status;
|
|
$migration->source_network_status = (string) $locked->network_status;
|
|
$migration->error_code = '';
|
|
$migration->cutover_at = null;
|
|
$migration->completed_at = null;
|
|
$migration->created_at = $migration->updated_at = time();
|
|
$migration->save();
|
|
|
|
CloudPeerMigrationArtifactService::persistLocked(
|
|
$migration,
|
|
$snapshots,
|
|
$resources['disk_gib'] * self::GIB
|
|
);
|
|
$ticket = bin2hex(random_bytes(32));
|
|
$expiresAt = time() + self::TICKET_LIFETIME_SECONDS;
|
|
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
|
|
$actor,
|
|
(int) $locked->host_id,
|
|
'instance_migration_prepare',
|
|
'POST',
|
|
$target,
|
|
[
|
|
'migration_id' => $migrationKey,
|
|
'ticket' => $ticket,
|
|
'expires_at' => $expiresAt,
|
|
'artifacts' => CloudPeerMigrationArtifactService::
|
|
sourceRequestArtifacts($migration),
|
|
],
|
|
[
|
|
'owner_id' => (int) $locked->user_id,
|
|
'can_manage' => true,
|
|
'maintenance_migration_id' => (int) $migration->id,
|
|
],
|
|
$canonical,
|
|
null,
|
|
'/v1/instance-maintenance-migrations/'
|
|
. (int) $migration->id . '/prepare'
|
|
);
|
|
if (!$created) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'维护迁移源准备任务幂等冲突', 409
|
|
);
|
|
}
|
|
$job->instance_id = (int) $locked->id;
|
|
$job->save();
|
|
$migration->source_prepare_job_id = (int) $job->id;
|
|
$migration->updated_at = time();
|
|
$migration->save();
|
|
return ['job' => $job];
|
|
}, 3);
|
|
}
|
|
|
|
public static function assertStartAvailable(
|
|
CloudInstance $instance,
|
|
int $targetHostId,
|
|
int $targetResourcePoolId,
|
|
array $resources,
|
|
int $snapshotDiskGib = 0,
|
|
?int $excludeMigrationId = null
|
|
): void {
|
|
if (!in_array((string) $instance->service_status,
|
|
['active', 'exempt', 'expired_grace', 'suspended_expired'], true)
|
|
|| !in_array((string) $instance->network_status,
|
|
['active', 'blocked_traffic', 'blocked_expired'], true)) {
|
|
throw new CloudError('实例服务或网络状态正在变化,暂不能维护迁移', 409);
|
|
}
|
|
if ($targetHostId === (int) $instance->host_id) {
|
|
throw new CloudError('维护迁移必须选择其他宿主机', 409);
|
|
}
|
|
if ((int) ($instance->reservation_id ?? 0) < 1
|
|
|| (int) ($instance->order_id ?? 0) < 1
|
|
|| (int) ($instance->resource_pool_id ?? 0) < 1) {
|
|
throw new CloudError('只有具备完整订单和资源预留的实例可以维护迁移', 409);
|
|
}
|
|
if ($targetResourcePoolId !== (int) $instance->resource_pool_id) {
|
|
throw new CloudError('维护迁移必须保持原资源池身份', 409);
|
|
}
|
|
$query = CloudInstanceMigration::where(
|
|
'instance_id', (int) $instance->id
|
|
)->whereIn('state', array_merge(
|
|
CloudInstanceMigrationWorkflow::blockingStates(),
|
|
['source_retained']
|
|
));
|
|
if ($excludeMigrationId !== null) {
|
|
$query->where('id', '<>', $excludeMigrationId);
|
|
}
|
|
if ($query->exists()) {
|
|
throw new CloudError('实例已有未完成迁移或待清理源副本', 409);
|
|
}
|
|
$allowed = array_filter(
|
|
CloudPlacementService::maintenanceMigrationTargets(
|
|
$instance,
|
|
$resources,
|
|
$snapshotDiskGib
|
|
),
|
|
static fn (array $target): bool =>
|
|
(int) ($target['id'] ?? 0) === $targetHostId
|
|
&& (int) ($target['resource_pool_id'] ?? 0)
|
|
=== $targetResourcePoolId
|
|
);
|
|
if ($allowed === []) {
|
|
throw new CloudError(
|
|
'目标宿主机必须具备原资源池的活动成员关系、可用容量、已验证网络接入和公网 IP',
|
|
409
|
|
);
|
|
}
|
|
CloudFirewallService::assertMaintenanceMigrationReady($instance);
|
|
}
|
|
|
|
public static function display(CloudInstanceMigration $migration): array
|
|
{
|
|
$state = (string) $migration->state;
|
|
$action = (string) $migration->recovery_action;
|
|
$targetReservation = CloudResourceReservation::find(
|
|
(int) $migration->target_reservation_id
|
|
);
|
|
$hasAbortEvidence = self::hasAbortPrepareEvidence($migration);
|
|
$recoveryOptions = CloudInstanceMigrationWorkflow::recoveryOptions(
|
|
$migration,
|
|
$hasAbortEvidence
|
|
);
|
|
$display = [
|
|
'id' => (string) $migration->id,
|
|
'state' => $state,
|
|
'cleanup_mode' => (string) $migration->cleanup_mode,
|
|
'source_host_id' => (string) $migration->source_host_id,
|
|
'target_host_id' => (string) $migration->target_host_id,
|
|
'target_resource_pool_id' => $targetReservation === null
|
|
? null : (string) $targetReservation->resource_pool_id,
|
|
'source_retained' => $state === 'source_retained',
|
|
'can_cleanup' => $state === 'source_retained'
|
|
&& $action !== 'reconcile_cleanup',
|
|
'recovery_action' => $action === '' ? null : $action,
|
|
'can_recover' => $recoveryOptions !== [],
|
|
'recovery_actions' => array_column(
|
|
$recoveryOptions,
|
|
'action'
|
|
),
|
|
'recovery_options' => $recoveryOptions,
|
|
'progress' => CloudInstanceMigrationWorkflow::progress(
|
|
$migration
|
|
),
|
|
'error_code' => (string) $migration->error_code,
|
|
'cutover_at' => $migration->cutover_at === null
|
|
? null : (int) $migration->cutover_at,
|
|
'completed_at' => $migration->completed_at === null
|
|
? null : (int) $migration->completed_at,
|
|
'created_at' => (int) $migration->created_at,
|
|
'updated_at' => (int) $migration->updated_at,
|
|
];
|
|
foreach (CloudInstanceMigrationWorkflow::stages() as $stage) {
|
|
if ($stage['task_key'] === null) {
|
|
continue;
|
|
}
|
|
$field = (string) $stage['job_field'];
|
|
$display[(string) $stage['task_key']] =
|
|
$migration->{$field} === null
|
|
? null : (string) $migration->{$field};
|
|
}
|
|
return $display;
|
|
}
|
|
|
|
private static function hasAbortPrepareEvidence(
|
|
CloudInstanceMigration $migration
|
|
): bool {
|
|
try {
|
|
self::abortPrepareEvidence($migration);
|
|
return true;
|
|
} catch (\Throwable $error) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @return array{job:CloudJob,event:CloudAgentCompletionEvent,result:array}
|
|
*/
|
|
private static function abortPrepareEvidence(
|
|
CloudInstanceMigration $migration
|
|
): array {
|
|
if ((string) $migration->state !== 'attention_required'
|
|
|| !in_array((string) $migration->recovery_action,
|
|
['reconcile_prepare', 'abort_prepare'], true)
|
|
|| $migration->target_receive_job_id !== null
|
|
|| $migration->firewall_job_id !== null
|
|
|| $migration->activate_job_id !== null
|
|
|| $migration->rollback_job_id !== null
|
|
|| $migration->cleanup_job_id !== null
|
|
|| $migration->cutover_at !== null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'当前迁移不处于可安全中止的源准备阶段', 409
|
|
);
|
|
}
|
|
$job = CloudJob::find((int) $migration->source_prepare_job_id);
|
|
if ($job === null
|
|
|| (string) $job->kind !== 'instance_migration_prepare'
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (int) $job->instance_id !== (int) $migration->instance_id
|
|
|| (int) $job->host_id !== (int) $migration->source_host_id
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| (int) $job->dispatch_attempts !== 1
|
|
|| !CloudDispatchLedger::isDispatched($job)
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown') {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'源准备任务不处于可补偿的效果核对状态', 409
|
|
);
|
|
}
|
|
$event = CloudOperationCompletionService::pendingForJob($job);
|
|
if ($event === null
|
|
|| !CloudOperationCompletionService::isExactStoredEvent($event)
|
|
|| (string) $event->outcome !== 'succeeded'
|
|
|| (bool) $event->effect_may_have_applied
|
|
|| (bool) $event->safe_to_resubmit
|
|
|| ($job->completion_event_id !== null
|
|
&& !hash_equals(
|
|
(string) $job->completion_event_id,
|
|
(string) $event->event_id
|
|
))) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'源准备任务尚无可安全补偿的精确成功回执', 409
|
|
);
|
|
}
|
|
CloudOperationCompletionService::assertEventIdentityForJob(
|
|
$job,
|
|
$event
|
|
);
|
|
$operation = CloudOperationCompletionService::operation($event);
|
|
$result = $operation['result'] ?? null;
|
|
self::assertAbortPrepareResult($migration, $result);
|
|
return ['job' => $job, 'event' => $event, 'result' => $result];
|
|
}
|
|
|
|
private static function assertAbortPrepareResult(
|
|
CloudInstanceMigration $migration,
|
|
$result
|
|
): void {
|
|
if (!is_array($result) || array_is_list($result)
|
|
|| !hash_equals(
|
|
(string) $migration->migration_key,
|
|
(string) ($result['migration_id'] ?? '')
|
|
)
|
|
|| ($result['source_stopped'] ?? null) !== true
|
|
|| ($result['source_autostart_disabled'] ?? null) !== true
|
|
|| !is_bool($result['source_was_running'] ?? null)
|
|
|| !is_bool($result['source_autostart_enabled'] ?? null)
|
|
|| !is_string($result['manifest_digest'] ?? null)
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $result['manifest_digest']) !== 1
|
|
|| !is_array($result['artifacts'] ?? null)
|
|
|| !array_is_list($result['artifacts'])
|
|
|| $result['artifacts'] === []) {
|
|
throw new CloudRecoveryReceiptInvalid(
|
|
'源准备成功回执缺少可安全恢复实例的证据', 409
|
|
);
|
|
}
|
|
}
|
|
|
|
public static function ownsPrepareJob(int $jobId): bool
|
|
{
|
|
return self::catalogAvailable()
|
|
&& CloudInstanceMigration::where(
|
|
'source_prepare_job_id', $jobId
|
|
)->exists();
|
|
}
|
|
|
|
public static function hasOpenMigration(int $instanceId): bool
|
|
{
|
|
return $instanceId > 0
|
|
&& self::catalogAvailable()
|
|
&& CloudInstanceMigration::where('instance_id', $instanceId)
|
|
->whereIn(
|
|
'state',
|
|
CloudInstanceMigrationWorkflow::blockingStates()
|
|
)->exists();
|
|
}
|
|
|
|
public static function assertInstanceMutationAvailable(
|
|
CloudInstance $instance,
|
|
?string $operationKind = null
|
|
): void {
|
|
if (self::hasOpenMigration((int) $instance->id)) {
|
|
throw new CloudError('实例正在维护迁移,请先等待迁移收敛', 409);
|
|
}
|
|
if ($operationKind !== null
|
|
&& in_array($operationKind, [
|
|
'delete', 'delete_force', 'expiry_delete',
|
|
], true)
|
|
&& self::catalogAvailable()
|
|
&& CloudInstanceMigration::where(
|
|
'instance_id', (int) $instance->id
|
|
)->where('state', 'source_retained')->exists()) {
|
|
throw new CloudError(
|
|
'迁移源副本尚未释放,请先完成迁移清理后再删除实例',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
|
|
public static function finishJobLocked(
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $payload
|
|
): void {
|
|
if (!self::catalogAvailable()) {
|
|
return;
|
|
}
|
|
$stage = CloudInstanceMigrationWorkflow::job((string) $job->kind);
|
|
if ($stage === null) {
|
|
return;
|
|
}
|
|
$field = (string) $stage['job_field'];
|
|
$migration = CloudInstanceMigration::where($field, (int) $job->id)
|
|
->lockForUpdate()->first();
|
|
if ($migration === null) {
|
|
return;
|
|
}
|
|
$handler = (string) $stage['finish_handler'];
|
|
self::{$handler}($migration, $job, $status, $result, $payload);
|
|
}
|
|
|
|
public static function markJobUncertain(CloudJob $job): void
|
|
{
|
|
if (!self::catalogAvailable()) {
|
|
return;
|
|
}
|
|
$stage = CloudInstanceMigrationWorkflow::job((string) $job->kind);
|
|
if ($stage === null) {
|
|
return;
|
|
}
|
|
$field = (string) $stage['job_field'];
|
|
$migration = CloudInstanceMigration::where($field, (int) $job->id)
|
|
->lockForUpdate()->first();
|
|
if ($migration === null) {
|
|
return;
|
|
}
|
|
$action = (string) $stage['reconcile_action'];
|
|
if ((string) $job->kind === 'instance_migration_prepare'
|
|
&& (string) $migration->recovery_action === 'abort_prepare') {
|
|
$action = 'abort_prepare';
|
|
}
|
|
if ((string) $stage['key'] === 'cleanup') {
|
|
CloudInstanceMigrationSettlement::markCleanupAttentionLocked(
|
|
$migration,
|
|
$job,
|
|
'finalization_incident'
|
|
);
|
|
self::setState($migration, 'source_retained',
|
|
'finalization_incident', $action);
|
|
return;
|
|
}
|
|
self::setState($migration, 'attention_required',
|
|
'finalization_incident', $action);
|
|
}
|
|
|
|
/**
|
|
* Join the multi-host VPC projection back into the migration state
|
|
* machine. Every host job carries the same durable migration generation;
|
|
* only the last converged result may advance transfer or source rollback.
|
|
*/
|
|
public static function finishVpcRoutingJobLocked(
|
|
CloudJob $job,
|
|
string $status,
|
|
array $payload
|
|
): void {
|
|
$migrationId = filter_var(
|
|
$payload['maintenance_migration_id'] ?? null,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
);
|
|
$direction = $payload['vpc_migration_direction'] ?? null;
|
|
if ($migrationId === false
|
|
|| !in_array($direction, ['target', 'source'], true)
|
|
|| !in_array((string) $job->kind,
|
|
['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 宿主机任务缺少完整编排身份', 409
|
|
);
|
|
}
|
|
$reference = CloudInstanceMigration::where(
|
|
'id', (int) $migrationId
|
|
)->first();
|
|
$instance = $reference === null ? null : CloudInstance::where(
|
|
'id', (int) $reference->instance_id
|
|
)->lockForUpdate()->first();
|
|
$migration = $instance === null ? null : CloudInstanceMigration::where(
|
|
'id', (int) $migrationId
|
|
)->lockForUpdate()->first();
|
|
if ($migration === null || $instance === null
|
|
|| $migration->vpc_port_id === null
|
|
|| (int) $migration->vpc_port_id !== (int) $job->vpc_port_id
|
|
|| (int) $migration->instance_id !== (int) $job->instance_id
|
|
|| (int) $instance->host_id !== (int) $migration->source_host_id
|
|
|| !hash_equals(
|
|
strtolower((string) $instance->uuid),
|
|
strtolower((string) $migration->source_uuid)
|
|
)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 任务与实例位置账本不一致', 409
|
|
);
|
|
}
|
|
$runningState = $direction === 'target'
|
|
? 'vpc_target_routing' : 'vpc_source_routing';
|
|
$reconcileAction = $direction === 'target'
|
|
? 'reconcile_vpc_target' : 'reconcile_vpc_source';
|
|
$failureAction = $direction === 'target'
|
|
? 'retry_vpc_target' : 'retry_vpc_source';
|
|
if ($direction === 'target'
|
|
&& ((string) $migration->state === 'transferring'
|
|
|| $migration->target_receive_job_id !== null)) {
|
|
return;
|
|
}
|
|
if ($direction === 'source'
|
|
&& ((string) $migration->state === 'rollback_running'
|
|
|| $migration->rollback_job_id !== null)) {
|
|
return;
|
|
}
|
|
if ((string) $migration->state !== $runningState
|
|
&& !((string) $migration->state === 'attention_required'
|
|
&& in_array((string) $migration->recovery_action, [
|
|
$reconcileAction, $failureAction,
|
|
], true))) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 任务与当前迁移阶段不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
if ($status === 'uncertain') {
|
|
self::setState(
|
|
$migration,
|
|
'attention_required',
|
|
'operation_effect_unknown',
|
|
$reconcileAction
|
|
);
|
|
return;
|
|
}
|
|
if ($status === 'failed') {
|
|
self::setState(
|
|
$migration,
|
|
'attention_required',
|
|
'migration_vpc_routing_failed',
|
|
$failureAction
|
|
);
|
|
return;
|
|
}
|
|
if ($status !== 'succeeded') {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 任务终态无效', 409
|
|
);
|
|
}
|
|
if (!CloudVpcPortService::finishMigrationProjectionLocked(
|
|
$migration,
|
|
$instance,
|
|
$direction
|
|
)) {
|
|
return;
|
|
}
|
|
if ($direction === 'target') {
|
|
$prepare = CloudJob::find(
|
|
(int) $migration->source_prepare_job_id
|
|
);
|
|
if ($prepare === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 收敛后源准备任务缺失', 409
|
|
);
|
|
}
|
|
$preparePayload = CloudSecrets::decrypt(
|
|
(string) $prepare->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
if (!is_array($preparePayload)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 收敛后源准备载荷无效', 409
|
|
);
|
|
}
|
|
self::queueTransferLocked(
|
|
$migration,
|
|
$prepare,
|
|
$preparePayload
|
|
);
|
|
return;
|
|
}
|
|
self::queueRollbackLocked(
|
|
$migration,
|
|
$instance,
|
|
(int) $job->user_id,
|
|
'migration_transfer_aborted'
|
|
);
|
|
}
|
|
|
|
public static function recover(
|
|
CloudInstance $instance,
|
|
array $actor,
|
|
array $input
|
|
): CloudJob {
|
|
CloudAccess::manage($actor);
|
|
CloudInput::only($input, ['action', 'confirm_recovery']);
|
|
$action = $input['action'] ?? null;
|
|
if (!is_string($action)
|
|
|| !CloudInstanceMigrationWorkflow::isRecoveryAction($action)
|
|
|| ($input['confirm_recovery'] ?? false) !== true) {
|
|
throw new CloudError('请选择并确认可用的迁移恢复操作');
|
|
}
|
|
if ($action === 'abort_prepare') {
|
|
return self::recoverAbortPrepare($instance);
|
|
}
|
|
return $instance->getConnection()->transaction(function () use (
|
|
$instance,
|
|
$actor,
|
|
$action
|
|
): CloudJob {
|
|
$locked = CloudInstance::where('id', (int) $instance->id)
|
|
->lockForUpdate()->firstOrFail();
|
|
$migration = CloudInstanceMigration::where(
|
|
'instance_id', (int) $locked->id
|
|
)->where('state', 'attention_required')->orderByDesc('id')
|
|
->lockForUpdate()->first();
|
|
if ($migration === null
|
|
|| !in_array(
|
|
$action,
|
|
CloudInstanceMigrationWorkflow::recoveryActions(
|
|
$migration,
|
|
false
|
|
),
|
|
true
|
|
)) {
|
|
throw new CloudError('当前迁移没有该恢复出口', 409);
|
|
}
|
|
$handler = CloudInstanceMigrationWorkflow::recoveryHandler(
|
|
$action
|
|
);
|
|
if ($handler === null || !method_exists(self::class, $handler)) {
|
|
throw new \LogicException(
|
|
'迁移恢复动作缺少可执行处理方法'
|
|
);
|
|
}
|
|
$migration->retry_generation = (int) $migration->retry_generation + 1;
|
|
$migration->updated_at = time();
|
|
$migration->save();
|
|
return self::{$handler}(
|
|
$migration,
|
|
$locked,
|
|
(int) $actor['user_id']
|
|
);
|
|
}, 3);
|
|
}
|
|
|
|
private static function recoverAbortPrepare(
|
|
CloudInstance $instance
|
|
): CloudJob
|
|
{
|
|
$candidate = CloudInstanceMigration::where(
|
|
'instance_id', (int) $instance->id
|
|
)->orderByDesc('id')->first();
|
|
if ($candidate !== null
|
|
&& (string) $candidate->state === 'rollback_running'
|
|
&& (string) $candidate->error_code
|
|
=== 'migration_prepare_aborted'
|
|
&& $candidate->rollback_job_id !== null) {
|
|
$existing = CloudJob::find((int) $candidate->rollback_job_id);
|
|
if ($existing !== null
|
|
&& (string) $existing->kind
|
|
=== 'instance_migration_rollback') {
|
|
return $existing;
|
|
}
|
|
}
|
|
if ($candidate === null) {
|
|
throw new CloudError('当前没有可中止的迁移源准备任务', 409);
|
|
}
|
|
$evidence = self::abortPrepareEvidence($candidate);
|
|
if (!in_array(
|
|
'abort_prepare',
|
|
CloudInstanceMigrationWorkflow::recoveryActions(
|
|
$candidate,
|
|
true
|
|
),
|
|
true
|
|
)) {
|
|
throw new CloudError('当前迁移没有该恢复出口', 409);
|
|
}
|
|
$migrationId = (int) $candidate->id;
|
|
$prepareJobId = (int) $evidence['job']->id;
|
|
$eventId = (string) $evidence['event']->event_id;
|
|
|
|
$instance->getConnection()->transaction(function () use (
|
|
$instance,
|
|
$migrationId,
|
|
$prepareJobId
|
|
): void {
|
|
// Keep worker lock order: prepare finalization owns job ->
|
|
// migration -> instance. This intent transaction deliberately does
|
|
// not lock the job or receipt.
|
|
$migration = CloudInstanceMigration::where('id', $migrationId)
|
|
->lockForUpdate()->first();
|
|
$locked = CloudInstance::where('id', (int) $instance->id)
|
|
->lockForUpdate()->first();
|
|
if ($migration === null || $locked === null
|
|
|| (int) $migration->source_prepare_job_id !== $prepareJobId
|
|
|| (int) $migration->instance_id !== (int) $locked->id
|
|
|| (string) $migration->state !== 'attention_required'
|
|
|| !in_array((string) $migration->recovery_action,
|
|
['reconcile_prepare', 'abort_prepare'], true)
|
|
|| $migration->target_receive_job_id !== null
|
|
|| $migration->rollback_job_id !== null
|
|
|| (int) $locked->host_id !== (int) $migration->source_host_id
|
|
|| !hash_equals(
|
|
strtolower((string) $locked->uuid),
|
|
strtolower((string) $migration->source_uuid)
|
|
)
|
|
|| (int) $locked->reservation_id
|
|
!== (int) $migration->source_reservation_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移源准备状态已经变化,不能中止', 409
|
|
);
|
|
}
|
|
if ((string) $migration->recovery_action
|
|
=== 'reconcile_prepare') {
|
|
$migration->retry_generation =
|
|
(int) $migration->retry_generation + 1;
|
|
$migration->recovery_action = 'abort_prepare';
|
|
$migration->updated_at = time();
|
|
$migration->save();
|
|
}
|
|
}, 3);
|
|
|
|
$prepare = CloudJob::find($prepareJobId);
|
|
$event = CloudAgentCompletionEvent::find($eventId);
|
|
if ($prepare === null || $event === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移源准备成功回执在中止期间发生变化', 409
|
|
);
|
|
}
|
|
CloudOperationRecoveryService::settleLateReceipt($prepare, $event);
|
|
$settled = CloudInstanceMigration::find($migrationId);
|
|
$rollback = $settled === null || $settled->rollback_job_id === null
|
|
? null : CloudJob::find((int) $settled->rollback_job_id);
|
|
if ($rollback === null
|
|
|| (string) $rollback->kind !== 'instance_migration_rollback') {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移中止已进入自动核对,请稍后刷新', 409
|
|
);
|
|
}
|
|
return $rollback;
|
|
}
|
|
|
|
public static function cleanupRetained(
|
|
CloudInstance $instance,
|
|
array $actor,
|
|
array $input
|
|
): CloudJob {
|
|
CloudAccess::manage($actor);
|
|
CloudInput::only($input, ['confirm_source_cleanup']);
|
|
if (($input['confirm_source_cleanup'] ?? false) !== true) {
|
|
throw new CloudError('请确认删除源宿主机上的旧副本');
|
|
}
|
|
return $instance->getConnection()->transaction(function () use (
|
|
$instance,
|
|
$actor
|
|
): CloudJob {
|
|
$locked = CloudInstance::where('id', (int) $instance->id)
|
|
->lockForUpdate()->firstOrFail();
|
|
$migration = CloudInstanceMigration::where(
|
|
'instance_id', (int) $locked->id
|
|
)->where('state', 'source_retained')->where(
|
|
'recovery_action', '<>', 'reconcile_cleanup'
|
|
)->orderByDesc('id')->lockForUpdate()->first();
|
|
if ($migration === null) {
|
|
throw new CloudError('当前没有可安全清理的迁移源副本', 409);
|
|
}
|
|
$migration->retry_generation = (int) $migration->retry_generation + 1;
|
|
$migration->cleanup_job_id = null;
|
|
$migration->save();
|
|
return self::queueCleanupLocked($migration, $locked, $actor);
|
|
}, 3);
|
|
}
|
|
|
|
private static function finishPrepareLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $payload
|
|
): void {
|
|
if ((int) $migration->instance_id !== (int) $job->instance_id
|
|
|| (int) $migration->source_host_id !== (int) $job->host_id
|
|
|| (int) ($payload['maintenance_migration_id'] ?? 0)
|
|
!== (int) $migration->id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源准备任务与迁移账本不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
if ($status === 'uncertain') {
|
|
self::setState($migration, 'attention_required',
|
|
'operation_effect_unknown', 'reconcile_prepare');
|
|
return;
|
|
}
|
|
if ($status === 'failed') {
|
|
if ((bool) ($job->effect_may_have_applied ?? false)) {
|
|
self::setState($migration, 'attention_required',
|
|
'operation_effect_unknown', 'reconcile_prepare');
|
|
return;
|
|
}
|
|
self::releasePreparedTargetLocked($migration,
|
|
'migration_prepare_failed');
|
|
self::setState($migration, 'rolled_back',
|
|
'migration_prepare_failed');
|
|
$migration->completed_at = time();
|
|
$migration->save();
|
|
return;
|
|
}
|
|
if (!is_array($result)
|
|
|| !hash_equals((string) $migration->migration_key,
|
|
(string) ($result['migration_id'] ?? ''))
|
|
|| !is_bool($result['source_was_running'] ?? null)
|
|
|| !is_bool($result['source_autostart_enabled'] ?? null)
|
|
|| !is_string($result['manifest_digest'] ?? null)
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $result['manifest_digest']) !== 1) {
|
|
throw new CloudError('源 Agent 的 PeerMigration 回执无效', 502);
|
|
}
|
|
$migration->source_was_running = $result['source_was_running'];
|
|
$migration->source_autostart_enabled =
|
|
$result['source_autostart_enabled'];
|
|
$migration->manifest_digest = (string) $result['manifest_digest'];
|
|
if ((string) $migration->recovery_action === 'abort_prepare') {
|
|
self::finishAbortPrepareLocked($migration, $job, $result);
|
|
return;
|
|
}
|
|
CloudPeerMigrationArtifactService::finishPrepareLocked(
|
|
$migration,
|
|
$result
|
|
);
|
|
if ($migration->target_receive_job_id !== null) {
|
|
return;
|
|
}
|
|
if ($migration->vpc_port_id !== null) {
|
|
$instance = CloudInstance::where(
|
|
'id', (int) $migration->instance_id
|
|
)->lockForUpdate()->first();
|
|
if ($instance === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 路由切换时实例账本缺失', 409
|
|
);
|
|
}
|
|
CloudVpcPortService::beginMigrationToTargetLocked(
|
|
$migration,
|
|
$instance
|
|
);
|
|
self::setState($migration, 'vpc_target_routing', '');
|
|
return;
|
|
}
|
|
self::queueTransferLocked($migration, $job, $payload);
|
|
}
|
|
|
|
private static function finishAbortPrepareLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudJob $prepareJob,
|
|
array $result
|
|
): void {
|
|
self::assertAbortPrepareResult($migration, $result);
|
|
if ((string) $migration->state !== 'attention_required'
|
|
|| (string) $migration->recovery_action !== 'abort_prepare'
|
|
|| $migration->target_receive_job_id !== null
|
|
|| $migration->firewall_job_id !== null
|
|
|| $migration->activate_job_id !== null
|
|
|| $migration->rollback_job_id !== null
|
|
|| $migration->cleanup_job_id !== null
|
|
|| $migration->cutover_at !== null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移源准备中止状态已经变化', 409
|
|
);
|
|
}
|
|
$instance = CloudInstance::where(
|
|
'id', (int) $migration->instance_id
|
|
)->lockForUpdate()->first();
|
|
$restore = CloudBackupRestore::where(
|
|
'id', (int) $migration->restore_id
|
|
)->lockForUpdate()->first();
|
|
$reservation = CloudResourceReservation::where(
|
|
'id', (int) $migration->target_reservation_id
|
|
)->lockForUpdate()->first();
|
|
if ($instance === null || $restore === null
|
|
|| $reservation === null
|
|
|| (int) $instance->host_id !== (int) $migration->source_host_id
|
|
|| !hash_equals(
|
|
strtolower((string) $instance->uuid),
|
|
strtolower((string) $migration->source_uuid)
|
|
)
|
|
|| (int) $instance->reservation_id
|
|
!== (int) $migration->source_reservation_id
|
|
|| (int) $restore->target_host_id
|
|
!== (int) $migration->target_host_id
|
|
|| (int) $restore->reservation_id !== (int) $reservation->id
|
|
|| $restore->backup_id !== null
|
|
|| $restore->job_id !== null
|
|
|| (string) $restore->state !== 'reserved'
|
|
|| (int) $reservation->backup_restore_id !== (int) $restore->id
|
|
|| (int) $reservation->host_id !== (int) $migration->target_host_id
|
|
|| (int) $reservation->resource_pool_id
|
|
!== (int) $instance->resource_pool_id
|
|
|| (int) $reservation->user_id !== (int) $instance->user_id
|
|
|| $reservation->instance_id !== null
|
|
|| (string) $reservation->state !== 'held'
|
|
|| $reservation->network_attachment_id !== null
|
|
|| $reservation->public_ip_pool_id !== null
|
|
|| $reservation->public_ip_id !== null
|
|
|| !hash_equals(
|
|
(string) $restore->operation_id,
|
|
(string) $reservation->operation_id
|
|
)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移中止前的源实例或目标预留账本已经变化', 409
|
|
);
|
|
}
|
|
CloudPublicNetworkAllocationService::currentClaimSet(
|
|
(int) $reservation->id,
|
|
'held'
|
|
);
|
|
$migration->source_was_running = $result['source_was_running'];
|
|
$migration->source_autostart_enabled =
|
|
$result['source_autostart_enabled'];
|
|
$migration->manifest_digest = (string) $result['manifest_digest'];
|
|
$migration->updated_at = time();
|
|
$migration->save();
|
|
self::queueRollbackLocked(
|
|
$migration,
|
|
$instance,
|
|
(int) $prepareJob->user_id,
|
|
'migration_prepare_aborted'
|
|
);
|
|
}
|
|
|
|
private static function queueTransferLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudJob $prepareJob,
|
|
array $preparePayload
|
|
): void {
|
|
$instance = CloudInstance::where(
|
|
'id', (int) $migration->instance_id
|
|
)->lockForUpdate()->first();
|
|
$restore = CloudBackupRestore::where(
|
|
'id', (int) $migration->restore_id
|
|
)->lockForUpdate()->first();
|
|
if ($instance === null || $restore === null
|
|
|| (int) $instance->host_id !== (int) $migration->source_host_id
|
|
|| !hash_equals(strtolower((string) $instance->uuid),
|
|
strtolower((string) $migration->source_uuid))
|
|
|| (int) $instance->reservation_id
|
|
!== (int) $migration->source_reservation_id
|
|
|| !hash_equals((string) $migration->source_service_status,
|
|
(string) $instance->service_status)
|
|
|| !hash_equals((string) $migration->source_network_status,
|
|
(string) $instance->network_status)
|
|
|| (int) $restore->target_host_id
|
|
!== (int) $migration->target_host_id
|
|
|| (int) $restore->reservation_id
|
|
!== (int) $migration->target_reservation_id
|
|
|| (string) $restore->state !== 'reserved') {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源实例或目标预留在停机期间发生变化', 409
|
|
);
|
|
}
|
|
$ticket = $preparePayload['body']['ticket'] ?? null;
|
|
if (!is_string($ticket)
|
|
|| preg_match('/^[a-f0-9]{64}$/D', $ticket) !== 1) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源任务缺少加密传输票据', 409
|
|
);
|
|
}
|
|
$spec = self::decodeSpec((string) $instance->spec_json);
|
|
$resources = self::resources($spec);
|
|
$placement = CloudPlacementService::preparedMaintenanceMigrationTarget(
|
|
$restore,
|
|
(int) $instance->id
|
|
);
|
|
$reservation = $placement['reservation'];
|
|
$publicNetwork = $placement['public_network'];
|
|
$targetIdentity = $placement['identity'];
|
|
if ((int) $reservation->resource_pool_id
|
|
!== (int) $instance->resource_pool_id
|
|
|| !hash_equals(
|
|
strtolower((string) $restore->new_uuid),
|
|
strtolower((string) $migration->target_uuid)
|
|
)
|
|
|| (int) $restore->target_host_id
|
|
!== (int) $migration->target_host_id
|
|
|| (int) $reservation->host_id
|
|
!== (int) $migration->target_host_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标资源池或网络接入身份不一致',
|
|
409
|
|
);
|
|
}
|
|
$policy = $spec['platform_network_policy_snapshot'] ?? [];
|
|
if (!is_array($policy) || array_is_list($policy)) {
|
|
$policy = [];
|
|
}
|
|
$vpcPort = CloudVpcPortService::migrationTargetMetadataLocked(
|
|
$migration,
|
|
$instance
|
|
);
|
|
$body = [
|
|
'migration_id' => (string) $migration->migration_key,
|
|
'ticket' => $ticket,
|
|
'source_endpoints' => self::decodePeerEndpoints(
|
|
(string) $migration->source_endpoint
|
|
),
|
|
'source_identity_sha256' =>
|
|
(string) $migration->peer_trust_id,
|
|
'source_uuid' => (string) $migration->source_uuid,
|
|
'manifest_digest' => (string) $migration->manifest_digest,
|
|
'network_suspended' =>
|
|
(string) $migration->source_network_status !== 'active',
|
|
'debian_12_network_fix' =>
|
|
($spec['debian_12_network_fix'] ?? null) === true,
|
|
'uuid' => (string) $migration->target_uuid,
|
|
'source' => '',
|
|
'name' => (string) $instance->name,
|
|
'description' => '',
|
|
'vcpu' => $resources['cpu_cores'],
|
|
'memory_mib' => $resources['memory_mib'],
|
|
'disk_gib' => $resources['disk_gib'],
|
|
'bandwidth_mbps' => $resources['bandwidth_mbps'],
|
|
'public_network' => $publicNetwork->toArray(),
|
|
'template_id' => '',
|
|
'template_revision' => 0,
|
|
'template_generation_token' => '',
|
|
'startup_script' => '',
|
|
// The target seed preserves credentials and hostname from the
|
|
// migrated disk. Injecting panel metadata here would violate the
|
|
// Agent's PeerMigration contract.
|
|
'hostname' => '',
|
|
'guest_ssh_public_key' => '',
|
|
'root_password' => '',
|
|
'platform_network_policy_snapshot' => $policy,
|
|
];
|
|
if ($vpcPort !== null) {
|
|
$body['vpc_port'] = $vpcPort;
|
|
}
|
|
$generation = (int) $migration->retry_generation;
|
|
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
|
|
[
|
|
'user_id' => (int) $prepareJob->user_id,
|
|
'request_idempotency_key' => 'migration-transfer-'
|
|
. (string) $migration->migration_key . '-' . $generation,
|
|
],
|
|
(int) $migration->target_host_id,
|
|
'instance_migration_transfer',
|
|
'POST',
|
|
'/v1/vms/' . rawurlencode((string) $migration->target_uuid)
|
|
. '/peer-migration/receive',
|
|
$body,
|
|
[
|
|
'owner_id' => (int) $instance->user_id,
|
|
'can_manage' => true,
|
|
'reservation_id' => (int) $reservation->id,
|
|
'maintenance_migration_id' => (int) $migration->id,
|
|
'migration_target' => [
|
|
'contract' => 1,
|
|
'backup_restore_id' => (int) $restore->id,
|
|
'reservation_id' => (int) $reservation->id,
|
|
'host_id' => (int) $migration->target_host_id,
|
|
'resource_pool_id' =>
|
|
(int) $reservation->resource_pool_id,
|
|
'operation_id' => (string) $restore->operation_id,
|
|
'target_uuid' =>
|
|
strtolower((string) $migration->target_uuid),
|
|
'target_agent_revision' =>
|
|
(int) $placement['target_agent_revision'],
|
|
'target_agent_contract_sha256' =>
|
|
(string) $placement[
|
|
'target_agent_contract_sha256'
|
|
],
|
|
'target_identity_json' =>
|
|
$targetIdentity->identityJson(),
|
|
'target_identity_sha256' =>
|
|
$targetIdentity->identitySha256(),
|
|
'target_claim_set_sha256' =>
|
|
$targetIdentity->claimSetSha256(),
|
|
'target_public_network_json' =>
|
|
$targetIdentity->publicNetworkJson(),
|
|
'target_public_network_sha256' =>
|
|
$targetIdentity->publicNetworkSha256(),
|
|
'vpc_port' => $vpcPort,
|
|
],
|
|
],
|
|
[
|
|
'migration_id' => (int) $migration->id,
|
|
'generation' => $generation,
|
|
'target_host_id' => (int) $migration->target_host_id,
|
|
'target_uuid' => (string) $migration->target_uuid,
|
|
'manifest_digest' => (string) $migration->manifest_digest,
|
|
'target_identity_sha256' =>
|
|
$targetIdentity->identitySha256(),
|
|
'target_claim_set_sha256' =>
|
|
$targetIdentity->claimSetSha256(),
|
|
'target_public_network_sha256' =>
|
|
$targetIdentity->publicNetworkSha256(),
|
|
'target_agent_revision' =>
|
|
(int) $placement['target_agent_revision'],
|
|
'target_agent_contract_sha256' =>
|
|
(string) $placement['target_agent_contract_sha256'],
|
|
'vpc_port' => $vpcPort,
|
|
],
|
|
(string) $restore->operation_id,
|
|
'/v1/instance-maintenance-migrations/'
|
|
. (int) $migration->id . '/transfer/' . $generation
|
|
);
|
|
if (!$created) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标接收任务幂等冲突', 409
|
|
);
|
|
}
|
|
$job->instance_id = (int) $instance->id;
|
|
$job->backup_restore_id = (int) $restore->id;
|
|
$job->save();
|
|
$restore->job_id = (int) $job->id;
|
|
$restore->state = 'restoring';
|
|
$restore->error_code = '';
|
|
$restore->updated_at = time();
|
|
$restore->save();
|
|
$migration->target_receive_job_id = (int) $job->id;
|
|
self::setState($migration, 'transferring', '');
|
|
}
|
|
|
|
private static function finishTransferLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $payload
|
|
): void {
|
|
$restore = CloudBackupRestore::where(
|
|
'id', (int) $migration->restore_id
|
|
)->lockForUpdate()->first();
|
|
$instance = CloudInstance::where(
|
|
'id', (int) $migration->instance_id
|
|
)->lockForUpdate()->first();
|
|
if ($restore === null || $instance === null
|
|
|| (int) $migration->target_receive_job_id !== (int) $job->id
|
|
|| (int) $job->backup_restore_id !== (int) $restore->id
|
|
|| (int) $job->instance_id !== (int) $instance->id
|
|
|| (int) $job->host_id !== (int) $migration->target_host_id
|
|
|| (int) ($payload['maintenance_migration_id'] ?? 0)
|
|
!== (int) $migration->id
|
|
|| !hash_equals((string) $restore->operation_id,
|
|
(string) $job->operation_id)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标接收任务与迁移账本不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
try {
|
|
$requestedNetwork = CloudPublicNetworkSnapshot::fromArray(
|
|
$payload['body']['public_network'] ?? null
|
|
);
|
|
} catch (\InvalidArgumentException) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标接收缺少双栈公网快照', 409
|
|
);
|
|
}
|
|
$now = time();
|
|
if ($status === 'uncertain') {
|
|
[$target] = self::verifiedTargetBundle(
|
|
$migration,
|
|
$restore,
|
|
$instance,
|
|
['held', 'uncertain'],
|
|
$requestedNetwork
|
|
);
|
|
CloudPublicNetworkAllocationService::fenceBundle(
|
|
(int) $target->id,
|
|
(string) $restore->operation_id
|
|
);
|
|
$target = CloudResourceReservation::where(
|
|
'id', (int) $target->id
|
|
)->lockForUpdate()->first();
|
|
if ($target === null
|
|
|| !in_array((string) $target->state,
|
|
['held', 'uncertain'], true)
|
|
|| $target->instance_id !== null
|
|
|| !hash_equals((string) $restore->operation_id,
|
|
(string) $target->operation_id)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标双栈隔离预留发生变化', 409
|
|
);
|
|
}
|
|
$target->state = 'uncertain';
|
|
$target->instance_id = null;
|
|
$target->expires_at = null;
|
|
$target->updated_at = $now;
|
|
$target->save();
|
|
$restore->state = 'uncertain';
|
|
$restore->error_code = 'operation_effect_unknown';
|
|
$restore->updated_at = $now;
|
|
$restore->save();
|
|
self::setState($migration, 'attention_required',
|
|
'operation_effect_unknown', 'reconcile_transfer');
|
|
return;
|
|
}
|
|
if ($status === 'failed') {
|
|
self::restoreTargetReservationForRetryLocked(
|
|
$migration, $restore, $instance, $requestedNetwork
|
|
);
|
|
$restore->state = 'reserved';
|
|
$restore->error_code = 'peer_transfer_failed';
|
|
$restore->updated_at = $now;
|
|
$restore->save();
|
|
self::setState($migration, 'attention_required',
|
|
'peer_transfer_failed',
|
|
(string) CloudInstanceMigrationWorkflow::failureAction(
|
|
'transfer'
|
|
));
|
|
return;
|
|
}
|
|
[$target, $targetClaimSet] = self::verifiedTargetBundle(
|
|
$migration,
|
|
$restore,
|
|
$instance,
|
|
['held', 'uncertain'],
|
|
$requestedNetwork
|
|
);
|
|
if (!is_array($result)
|
|
|| !hash_equals((string) $migration->migration_key,
|
|
(string) ($result['migration_id'] ?? ''))
|
|
|| !hash_equals((string) $migration->manifest_digest,
|
|
(string) ($result['manifest_digest'] ?? ''))
|
|
|| !hash_equals(strtolower((string) $migration->target_uuid),
|
|
strtolower((string) ($result['uuid'] ?? '')))
|
|
|| (string) ($result['status'] ?? '') !== 'shutoff'
|
|
|| ($result['staged_powered_off'] ?? null) !== true
|
|
|| !self::sameDocument(
|
|
$result['vpc_port'] ?? null,
|
|
$payload['body']['vpc_port'] ?? null
|
|
)) {
|
|
throw new CloudError('目标 Agent 的 PeerMigration 回执无效', 502);
|
|
}
|
|
$rootDiskGib = filter_var(
|
|
$payload['body']['disk_gib'] ?? null,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
);
|
|
if ($rootDiskGib === false) {
|
|
throw new CloudError('目标 Agent 的系统盘容量回执无效', 502);
|
|
}
|
|
$sourceHint = CloudResourceReservation::find(
|
|
(int) $migration->source_reservation_id
|
|
);
|
|
if ($sourceHint === null
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) $sourceHint->operation_id) !== 1) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源双栈网络操作身份无效', 409
|
|
);
|
|
}
|
|
$handoff = CloudPublicNetworkAllocationService::
|
|
handoffMigrationBundles(
|
|
(int) $sourceHint->id,
|
|
(int) $target->id,
|
|
(int) $instance->id,
|
|
(string) $sourceHint->operation_id,
|
|
(string) $restore->operation_id
|
|
);
|
|
$sourceClaimSet = $handoff['source'] ?? null;
|
|
$committedTargetClaimSet = $handoff['target'] ?? null;
|
|
if (!$sourceClaimSet instanceof CloudActivePublicNetworkClaimSet
|
|
|| !$committedTargetClaimSet
|
|
instanceof CloudActivePublicNetworkClaimSet) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 双栈网络切换回执无效', 409
|
|
);
|
|
}
|
|
if (!hash_equals(
|
|
$targetClaimSet->publicNetwork()->fingerprint(),
|
|
$committedTargetClaimSet->publicNetwork()->fingerprint()
|
|
)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标双栈公网提交身份发生变化', 409
|
|
);
|
|
}
|
|
$target = CloudResourceReservation::where(
|
|
'id', (int) $target->id
|
|
)->lockForUpdate()->first();
|
|
$source = CloudResourceReservation::where(
|
|
'id', (int) $migration->source_reservation_id
|
|
)->lockForUpdate()->first();
|
|
$snapshotDiskGib = CloudPeerMigrationArtifactService::artifactDiskGib(
|
|
$migration
|
|
);
|
|
$expectedTargetClaim = (int) $rootDiskGib + $snapshotDiskGib;
|
|
if ($target === null || $source === null
|
|
|| (string) $target->state !== 'committed'
|
|
|| (int) $target->instance_id !== (int) $instance->id
|
|
|| !hash_equals((string) $restore->operation_id,
|
|
(string) $target->operation_id)
|
|
|| (int) $target->disk_gib !== $expectedTargetClaim
|
|
|| (string) $source->state !== 'held'
|
|
|| (int) $source->host_id !== (int) $migration->source_host_id
|
|
|| (int) $source->resource_pool_id
|
|
!== (int) $instance->resource_pool_id
|
|
|| (int) $target->resource_pool_id
|
|
!== (int) $source->resource_pool_id
|
|
|| $source->instance_id !== null
|
|
|| (int) $source->disk_gib !== (int) $rootDiskGib
|
|
|| (int) $instance->host_id !== (int) $migration->source_host_id
|
|
|| !hash_equals(strtolower((string) $instance->uuid),
|
|
strtolower((string) $migration->source_uuid))) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 切换前资源预留账本不一致', 409
|
|
);
|
|
}
|
|
$targetSpec = self::finalizeTargetSpec(
|
|
self::decodeSpec((string) $instance->spec_json),
|
|
is_array($payload['body'] ?? null) ? $payload['body'] : [],
|
|
$result,
|
|
(int) $target->resource_pool_id
|
|
);
|
|
CloudPeerMigrationArtifactService::finishReceiveLocked(
|
|
$migration, $result
|
|
);
|
|
// Snapshot rows now account for their target volumes. The target
|
|
// reservation therefore keeps only the root disk; while the source
|
|
// duplicate exists its original reservation accounts for root plus
|
|
// every frozen snapshot.
|
|
$target->disk_gib = (int) $rootDiskGib;
|
|
$source->disk_gib = $expectedTargetClaim;
|
|
$source->updated_at = $now;
|
|
$source->save();
|
|
$target->updated_at = $now;
|
|
$target->save();
|
|
|
|
$spec = $targetSpec;
|
|
$spec['managed'] = true;
|
|
$spec['migrated_from_host_id'] = (int) $migration->source_host_id;
|
|
$spec['migrated_from_uuid'] = (string) $migration->source_uuid;
|
|
$spec['maintenance_migration_key'] =
|
|
(string) $migration->migration_key;
|
|
$spec['traffic_counter_generation'] = [
|
|
'host_id' => (int) $migration->target_host_id,
|
|
'uuid' => strtolower((string) $migration->target_uuid),
|
|
'download_base_bytes' =>
|
|
max(0, (int) $instance->lifetime_download_bytes),
|
|
'upload_base_bytes' =>
|
|
max(0, (int) $instance->lifetime_upload_bytes),
|
|
];
|
|
CloudVpcPortService::finalizeMigrationCutoverLocked(
|
|
$migration,
|
|
$instance
|
|
);
|
|
$instance->host_id = (int) $migration->target_host_id;
|
|
$instance->uuid = strtolower((string) $migration->target_uuid);
|
|
$instance->resource_pool_id = (int) $target->resource_pool_id;
|
|
$instance->reservation_id = (int) $target->id;
|
|
$instance->spec_json = json_encode(
|
|
$spec,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
$instance->service_status = (string) $migration->source_service_status;
|
|
$instance->network_status = (string) $migration->source_network_status;
|
|
$instance->traffic_synced_at = 0;
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
CloudInstanceMigrationSettlement::settleCutoverLocked(
|
|
$migration,
|
|
$sourceClaimSet
|
|
);
|
|
$restore = CloudBackupRestore::where(
|
|
'id', (int) $restore->id
|
|
)->lockForUpdate()->first();
|
|
if ($restore === null || $restore->reservation_id !== null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标恢复预留未完成结算', 409
|
|
);
|
|
}
|
|
$restore->state = 'ready';
|
|
$restore->error_code = '';
|
|
$restore->ready_at = $now;
|
|
$restore->updated_at = $now;
|
|
$restore->save();
|
|
$migration->cutover_at = $now;
|
|
self::setState($migration, 'cutover', '');
|
|
CloudInstanceRuntimeService::seed(
|
|
(int) $instance->id,
|
|
(int) $instance->host_id,
|
|
(string) $instance->uuid
|
|
);
|
|
$firewall = CloudFirewallService::queueMaintenanceMigrationReapplyLocked(
|
|
$instance,
|
|
$migration,
|
|
(int) $job->user_id,
|
|
(int) $migration->retry_generation
|
|
);
|
|
if ($firewall !== null) {
|
|
$migration->firewall_job_id = (int) $firewall->id;
|
|
self::setState($migration, 'firewall_running', '');
|
|
return;
|
|
}
|
|
self::advanceAfterTargetReadyLocked(
|
|
$migration, $instance, (int) $job->user_id
|
|
);
|
|
}
|
|
|
|
private static function restoreTargetReservationForRetryLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudBackupRestore $restore,
|
|
CloudInstance $instance,
|
|
CloudPublicNetworkSnapshot $publicNetwork
|
|
): void {
|
|
[$reservation] = self::verifiedTargetBundle(
|
|
$migration,
|
|
$restore,
|
|
$instance,
|
|
['held', 'uncertain'],
|
|
$publicNetwork
|
|
);
|
|
$reopened = CloudPublicNetworkAllocationService::reopenBundleForRetry(
|
|
(int) $reservation->id,
|
|
(string) $restore->operation_id,
|
|
true
|
|
);
|
|
if (!hash_equals($publicNetwork->fingerprint(),
|
|
$reopened->fingerprint())) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 重试目标双栈公网身份发生变化', 409
|
|
);
|
|
}
|
|
$reservation = CloudResourceReservation::where(
|
|
'id', (int) $reservation->id
|
|
)->lockForUpdate()->first();
|
|
if ($reservation === null || (string) $reservation->state !== 'held'
|
|
|| $reservation->instance_id !== null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 重试目标预留没有完整恢复', 409
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param list<string> $allowedStates
|
|
* @return array{0:CloudResourceReservation,
|
|
* 1:CloudActivePublicNetworkClaimSet}
|
|
*/
|
|
private static function verifiedTargetBundle(
|
|
CloudInstanceMigration $migration,
|
|
CloudBackupRestore $restore,
|
|
CloudInstance $instance,
|
|
array $allowedStates,
|
|
?CloudPublicNetworkSnapshot $expectedNetwork = null
|
|
): array {
|
|
$reservation = CloudResourceReservation::where(
|
|
'id', (int) $migration->target_reservation_id
|
|
)->first();
|
|
if ($reservation === null
|
|
|| !in_array((string) $reservation->state,
|
|
$allowedStates, true)
|
|
|| (int) $restore->reservation_id !== (int) $reservation->id
|
|
|| (int) $restore->target_host_id
|
|
!== (int) $migration->target_host_id
|
|
|| (int) $reservation->backup_restore_id !== (int) $restore->id
|
|
|| $reservation->order_id !== null
|
|
|| $reservation->offer_id !== null
|
|
|| $reservation->backup_restore_attempt_key !== null
|
|
|| $reservation->migration_source_debt_id !== null
|
|
|| (int) $reservation->host_id
|
|
!== (int) $migration->target_host_id
|
|
|| (int) $reservation->resource_pool_id
|
|
!== (int) $instance->resource_pool_id
|
|
|| (int) $reservation->user_id !== (int) $instance->user_id
|
|
|| $reservation->instance_id !== null
|
|
|| $reservation->network_attachment_id !== null
|
|
|| $reservation->public_ip_pool_id !== null
|
|
|| $reservation->public_ip_id !== null
|
|
|| !hash_equals((string) $restore->operation_id,
|
|
(string) $reservation->operation_id)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标双栈预留账本不一致', 409
|
|
);
|
|
}
|
|
$claimSet = CloudPublicNetworkAllocationService::currentClaimSet(
|
|
(int) $reservation->id,
|
|
(string) $reservation->state
|
|
);
|
|
if ($expectedNetwork !== null && !hash_equals(
|
|
$expectedNetwork->fingerprint(),
|
|
$claimSet->publicNetwork()->fingerprint()
|
|
)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标双栈公网身份不一致', 409
|
|
);
|
|
}
|
|
return [$reservation, $claimSet];
|
|
}
|
|
|
|
/** Release a target held before the source mutation was dispatched. */
|
|
private static function releasePreparedTargetLocked(
|
|
CloudInstanceMigration $migration,
|
|
string $errorCode
|
|
): void {
|
|
$restore = CloudBackupRestore::where(
|
|
'id', (int) $migration->restore_id
|
|
)->lockForUpdate()->first();
|
|
if ($restore === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 缺少目标预留目录', 409
|
|
);
|
|
}
|
|
if ((string) $restore->state === 'failed') {
|
|
return;
|
|
}
|
|
$instance = CloudInstance::where(
|
|
'id', (int) $migration->instance_id
|
|
)->lockForUpdate()->first();
|
|
if ($instance === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 缺少源实例账本', 409
|
|
);
|
|
}
|
|
[$reservation] = self::verifiedTargetBundle(
|
|
$migration,
|
|
$restore,
|
|
$instance,
|
|
['held']
|
|
);
|
|
$releaseReason = 'maintenance_migration_target_not_applied';
|
|
CloudPublicNetworkAllocationService::releaseBundle(
|
|
(int) $reservation->id,
|
|
(string) $restore->operation_id,
|
|
$releaseReason,
|
|
true
|
|
);
|
|
$reservation = CloudResourceReservation::where(
|
|
'id', (int) $reservation->id
|
|
)->lockForUpdate()->first();
|
|
if ($reservation === null
|
|
|| (string) $reservation->state !== 'held'
|
|
|| $reservation->instance_id !== null
|
|
|| !hash_equals((string) $restore->operation_id,
|
|
(string) $reservation->operation_id)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标双栈释放预留发生变化', 409
|
|
);
|
|
}
|
|
$now = time();
|
|
$reservation->state = 'released';
|
|
$reservation->instance_id = null;
|
|
$reservation->expires_at = null;
|
|
$reservation->released_at = $now;
|
|
$reservation->release_reason = $releaseReason;
|
|
$reservation->updated_at = $now;
|
|
$reservation->save();
|
|
$restore->state = 'failed';
|
|
$restore->error_code = $errorCode;
|
|
$restore->updated_at = $now;
|
|
$restore->save();
|
|
}
|
|
|
|
private static function retryTransferLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance
|
|
): CloudJob {
|
|
if ((int) $instance->host_id !== (int) $migration->source_host_id
|
|
|| !hash_equals(strtolower((string) $instance->uuid),
|
|
strtolower((string) $migration->source_uuid))) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 传输重试时源实例位置已经变化', 409
|
|
);
|
|
}
|
|
$prepare = CloudJob::find((int) $migration->source_prepare_job_id);
|
|
if ($prepare === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源准备任务目录缺失', 409
|
|
);
|
|
}
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $prepare->sealed_payload, 'job-payload'
|
|
);
|
|
if (!is_array($payload)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源准备任务载荷无效', 409
|
|
);
|
|
}
|
|
self::rotateTargetTransferOperationLocked($migration, $instance);
|
|
$migration->target_receive_job_id = null;
|
|
$migration->updated_at = time();
|
|
$migration->save();
|
|
self::queueTransferLocked($migration, $prepare, $payload);
|
|
$job = CloudJob::find((int) $migration->target_receive_job_id);
|
|
if ($job === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 重试任务目录缺失', 409
|
|
);
|
|
}
|
|
return $job;
|
|
}
|
|
|
|
private static function recoverRetryTransferLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
unset($actorUserId);
|
|
return self::retryTransferLocked($migration, $instance);
|
|
}
|
|
|
|
private static function recoverRetryVpcTargetLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
unset($actorUserId);
|
|
$jobs = CloudVpcPortService::beginMigrationToTargetLocked(
|
|
$migration,
|
|
$instance
|
|
);
|
|
self::setState($migration, 'vpc_target_routing', '');
|
|
return self::firstVpcRoutingJob($jobs);
|
|
}
|
|
|
|
private static function recoverAbortVpcTargetLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
unset($actorUserId);
|
|
$jobs = CloudVpcPortService::beginMigrationToSourceLocked(
|
|
$migration,
|
|
$instance
|
|
);
|
|
self::setState($migration, 'vpc_source_routing', '');
|
|
return self::firstVpcRoutingJob($jobs);
|
|
}
|
|
|
|
private static function recoverRetryVpcSourceLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
unset($actorUserId);
|
|
$jobs = CloudVpcPortService::beginMigrationToSourceLocked(
|
|
$migration,
|
|
$instance
|
|
);
|
|
self::setState($migration, 'vpc_source_routing', '');
|
|
return self::firstVpcRoutingJob($jobs);
|
|
}
|
|
|
|
private static function recoverAbortTransferLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
if ($migration->vpc_port_id !== null) {
|
|
$jobs = CloudVpcPortService::beginMigrationToSourceLocked(
|
|
$migration,
|
|
$instance
|
|
);
|
|
self::setState($migration, 'vpc_source_routing', '');
|
|
return self::firstVpcRoutingJob($jobs);
|
|
}
|
|
return self::queueRollbackLocked(
|
|
$migration,
|
|
$instance,
|
|
$actorUserId,
|
|
'migration_transfer_aborted'
|
|
);
|
|
}
|
|
|
|
/** @param list<CloudJob> $jobs */
|
|
private static function firstVpcRoutingJob(array $jobs): CloudJob
|
|
{
|
|
$job = $jobs[0] ?? null;
|
|
if (!$job instanceof CloudJob) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'迁移 VPC 路由切换没有生成可跟踪任务', 409
|
|
);
|
|
}
|
|
return $job;
|
|
}
|
|
|
|
/**
|
|
* A confirmed not-applied transfer may be retried only under a fresh
|
|
* Agent operation ID. Rotate the restore, reservation and IP ledgers as
|
|
* one locked unit so the prior Journal result cannot be replayed as the
|
|
* new attempt.
|
|
*/
|
|
private static function rotateTargetTransferOperationLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance
|
|
): void {
|
|
$restore = CloudBackupRestore::where(
|
|
'id', (int) $migration->restore_id
|
|
)->lockForUpdate()->first();
|
|
$reservationHint = CloudResourceReservation::where(
|
|
'id', (int) $migration->target_reservation_id
|
|
)->first();
|
|
if ($restore === null || $reservationHint === null
|
|
|| (string) $restore->state !== 'reserved'
|
|
|| (int) $restore->reservation_id !== (int) $reservationHint->id
|
|
|| (int) $restore->target_host_id
|
|
!== (int) $migration->target_host_id
|
|
|| (int) $reservationHint->backup_restore_id !== (int) $restore->id
|
|
|| (int) $reservationHint->host_id
|
|
!== (int) $migration->target_host_id
|
|
|| (int) $reservationHint->resource_pool_id
|
|
!== (int) $instance->resource_pool_id
|
|
|| (int) $reservationHint->user_id !== (int) $instance->user_id
|
|
|| $reservationHint->instance_id !== null
|
|
|| (string) $reservationHint->state !== 'held'
|
|
|| $reservationHint->network_attachment_id !== null
|
|
|| $reservationHint->public_ip_pool_id !== null
|
|
|| $reservationHint->public_ip_id !== null
|
|
|| !hash_equals((string) $restore->operation_id,
|
|
(string) $reservationHint->operation_id)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 传输重试账本不一致', 409
|
|
);
|
|
}
|
|
CloudPublicNetworkAllocationService::currentClaimSet(
|
|
(int) $reservationHint->id,
|
|
'held'
|
|
);
|
|
$priorOperationId = (string) $restore->operation_id;
|
|
$operationId = bin2hex(random_bytes(16));
|
|
CloudPublicNetworkAllocationService::rotateHeldBundleOperation(
|
|
(int) $reservationHint->id,
|
|
$priorOperationId,
|
|
$operationId
|
|
);
|
|
$now = time();
|
|
$restore->operation_id = $operationId;
|
|
$restore->job_id = null;
|
|
$restore->error_code = '';
|
|
$restore->updated_at = $now;
|
|
$restore->save();
|
|
$reservation = CloudResourceReservation::where(
|
|
'id', (int) $reservationHint->id
|
|
)->lockForUpdate()->first();
|
|
if ($reservation === null
|
|
|| !hash_equals($operationId,
|
|
(string) $reservation->operation_id)
|
|
|| (string) $reservation->state !== 'held'
|
|
|| $reservation->instance_id !== null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 传输重试操作没有完整轮换', 409
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function queueRollbackLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId,
|
|
string $errorCode
|
|
): CloudJob {
|
|
if ((int) $instance->host_id !== (int) $migration->source_host_id
|
|
|| !hash_equals(strtolower((string) $instance->uuid),
|
|
strtolower((string) $migration->source_uuid))
|
|
|| $migration->source_was_running === null
|
|
|| $migration->source_autostart_enabled === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源实例回滚账本不完整', 409
|
|
);
|
|
}
|
|
$generation = (int) $migration->retry_generation;
|
|
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
|
|
[
|
|
'user_id' => $actorUserId,
|
|
'request_idempotency_key' => 'migration-rollback-'
|
|
. (string) $migration->migration_key . '-' . $generation,
|
|
],
|
|
(int) $migration->source_host_id,
|
|
'instance_migration_rollback',
|
|
'POST',
|
|
'/v1/vms/' . rawurlencode((string) $migration->source_uuid)
|
|
. '/peer-migration/rollback',
|
|
[
|
|
'migration_id' => (string) $migration->migration_key,
|
|
'source_was_running' => (bool) $migration->source_was_running,
|
|
'source_autostart_enabled' =>
|
|
(bool) $migration->source_autostart_enabled,
|
|
],
|
|
[
|
|
'owner_id' => (int) $instance->user_id,
|
|
'can_manage' => true,
|
|
'maintenance_migration_id' => (int) $migration->id,
|
|
],
|
|
[
|
|
'migration_id' => (int) $migration->id,
|
|
'generation' => $generation,
|
|
'source_host_id' => (int) $migration->source_host_id,
|
|
'source_uuid' => (string) $migration->source_uuid,
|
|
'source_was_running' =>
|
|
(bool) $migration->source_was_running,
|
|
'source_autostart_enabled' =>
|
|
(bool) $migration->source_autostart_enabled,
|
|
],
|
|
null,
|
|
'/v1/instance-maintenance-migrations/'
|
|
. (int) $migration->id . '/rollback/' . $generation
|
|
);
|
|
if (!$created) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源回滚任务幂等冲突', 409
|
|
);
|
|
}
|
|
$job->instance_id = (int) $instance->id;
|
|
$job->backup_restore_id = (int) $migration->restore_id;
|
|
$job->save();
|
|
$migration->rollback_job_id = (int) $job->id;
|
|
self::setState($migration, 'rollback_running', $errorCode);
|
|
return $job;
|
|
}
|
|
|
|
private static function retryRollbackLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
$migration->rollback_job_id = null;
|
|
$migration->updated_at = time();
|
|
$migration->save();
|
|
return self::queueRollbackLocked(
|
|
$migration, $instance, $actorUserId,
|
|
'migration_source_rollback_failed'
|
|
);
|
|
}
|
|
|
|
private static function recoverRetryRollbackLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
return self::retryRollbackLocked(
|
|
$migration,
|
|
$instance,
|
|
$actorUserId
|
|
);
|
|
}
|
|
|
|
private static function finishRollbackLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $payload
|
|
): void {
|
|
unset($payload);
|
|
$instance = CloudInstance::where(
|
|
'id', (int) $migration->instance_id
|
|
)->lockForUpdate()->first();
|
|
if ($instance === null
|
|
|| (int) $migration->rollback_job_id !== (int) $job->id
|
|
|| (int) $job->instance_id !== (int) $instance->id
|
|
|| (int) $job->host_id !== (int) $migration->source_host_id
|
|
|| (int) $instance->host_id !== (int) $migration->source_host_id
|
|
|| !hash_equals(strtolower((string) $instance->uuid),
|
|
strtolower((string) $migration->source_uuid))) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源回滚任务与实例不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
if ($status === 'uncertain') {
|
|
self::setState($migration, 'attention_required',
|
|
'operation_effect_unknown', 'reconcile_rollback');
|
|
return;
|
|
}
|
|
if ($status === 'failed') {
|
|
self::setState($migration, 'attention_required',
|
|
'migration_source_rollback_failed',
|
|
(string) CloudInstanceMigrationWorkflow::failureAction(
|
|
'rollback'
|
|
));
|
|
return;
|
|
}
|
|
if (!is_array($result)
|
|
|| !hash_equals(strtolower((string) $migration->source_uuid),
|
|
strtolower((string) ($result['uuid'] ?? '')))
|
|
|| !is_bool($result['running'] ?? null)
|
|
|| !is_bool($result['autostart_enabled'] ?? null)
|
|
|| $result['running'] !== (bool) $migration->source_was_running
|
|
|| $result['autostart_enabled']
|
|
!== (bool) $migration->source_autostart_enabled) {
|
|
throw new CloudError(
|
|
'源 Agent 未确认实例恢复到迁移前电源和自启动状态', 502
|
|
);
|
|
}
|
|
$abortCode = $migration->target_receive_job_id === null
|
|
? 'migration_prepare_aborted'
|
|
: 'migration_transfer_aborted';
|
|
self::releasePreparedTargetLocked($migration, $abortCode);
|
|
self::setState($migration, 'rolled_back', $abortCode);
|
|
$migration->completed_at = time();
|
|
$migration->save();
|
|
}
|
|
|
|
private static function finishFirewallLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $payload
|
|
): void {
|
|
unset($result, $payload);
|
|
$instance = CloudInstance::where(
|
|
'id', (int) $migration->instance_id
|
|
)->lockForUpdate()->first();
|
|
if ($instance === null
|
|
|| (int) $migration->firewall_job_id !== (int) $job->id
|
|
|| (int) $job->instance_id !== (int) $instance->id
|
|
|| (int) $job->host_id !== (int) $migration->target_host_id
|
|
|| (int) $instance->host_id !== (int) $migration->target_host_id
|
|
|| !hash_equals(strtolower((string) $instance->uuid),
|
|
strtolower((string) $migration->target_uuid))) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 防火墙任务与目标实例不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
if ($status === 'uncertain') {
|
|
self::setState($migration, 'attention_required',
|
|
'firewall_result_uncertain', 'reconcile_firewall');
|
|
return;
|
|
}
|
|
if ($status === 'failed') {
|
|
self::setState($migration, 'attention_required',
|
|
'firewall_apply_failed',
|
|
(string) CloudInstanceMigrationWorkflow::failureAction(
|
|
'firewall'
|
|
));
|
|
return;
|
|
}
|
|
$policy = CloudInstanceFirewall::where(
|
|
'instance_id', (int) $instance->id
|
|
)->lockForUpdate()->first();
|
|
if ($policy === null
|
|
|| (int) $policy->host_id !== (int) $instance->host_id
|
|
|| (string) $policy->sync_state !== 'active'
|
|
|| (int) $policy->desired_revision
|
|
!== (int) $policy->applied_revision) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标防火墙没有完成收敛', 409
|
|
);
|
|
}
|
|
self::advanceAfterTargetReadyLocked(
|
|
$migration, $instance, (int) $job->user_id
|
|
);
|
|
}
|
|
|
|
private static function retryFirewallLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
$migration->firewall_job_id = null;
|
|
$migration->updated_at = time();
|
|
$migration->save();
|
|
$job = CloudFirewallService::queueMaintenanceMigrationReapplyLocked(
|
|
$instance,
|
|
$migration,
|
|
$actorUserId,
|
|
(int) $migration->retry_generation
|
|
);
|
|
if ($job === null) {
|
|
self::advanceAfterTargetReadyLocked(
|
|
$migration, $instance, $actorUserId
|
|
);
|
|
$next = $migration->activate_job_id === null
|
|
? $migration->cleanup_job_id : $migration->activate_job_id;
|
|
$job = $next === null ? null : CloudJob::find((int) $next);
|
|
} else {
|
|
$migration->firewall_job_id = (int) $job->id;
|
|
self::setState($migration, 'firewall_running', '');
|
|
}
|
|
if ($job === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 防火墙恢复后没有后续任务', 409
|
|
);
|
|
}
|
|
return $job;
|
|
}
|
|
|
|
private static function recoverRetryFirewallLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
return self::retryFirewallLocked(
|
|
$migration,
|
|
$instance,
|
|
$actorUserId
|
|
);
|
|
}
|
|
|
|
private static function advanceAfterTargetReadyLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): void {
|
|
self::queueActivationLocked($migration, $instance, $actorUserId);
|
|
}
|
|
|
|
private static function queueActivationLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
$generation = (int) $migration->retry_generation;
|
|
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
|
|
[
|
|
'user_id' => $actorUserId,
|
|
'request_idempotency_key' => 'migration-activate-'
|
|
. (string) $migration->migration_key . '-' . $generation,
|
|
],
|
|
(int) $migration->target_host_id,
|
|
'instance_migration_activate',
|
|
'POST',
|
|
'/v1/vms/' . rawurlencode((string) $migration->target_uuid)
|
|
. '/peer-migration/activate',
|
|
[
|
|
'migration_id' => (string) $migration->migration_key,
|
|
'target_was_running' =>
|
|
(bool) $migration->source_was_running,
|
|
'target_autostart_enabled' =>
|
|
(bool) $migration->source_autostart_enabled,
|
|
],
|
|
[
|
|
'owner_id' => (int) $instance->user_id,
|
|
'can_manage' => true,
|
|
'maintenance_migration_id' => (int) $migration->id,
|
|
],
|
|
[
|
|
'migration_id' => (int) $migration->id,
|
|
'generation' => $generation,
|
|
'target_host_id' => (int) $migration->target_host_id,
|
|
'target_uuid' => (string) $migration->target_uuid,
|
|
'source_was_running' => (bool) $migration->source_was_running,
|
|
'source_autostart_enabled' =>
|
|
(bool) $migration->source_autostart_enabled,
|
|
],
|
|
null,
|
|
'/v1/instance-maintenance-migrations/'
|
|
. (int) $migration->id . '/activate/' . $generation
|
|
);
|
|
if (!$created) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标激活任务幂等冲突', 409
|
|
);
|
|
}
|
|
$job->instance_id = (int) $instance->id;
|
|
$job->backup_restore_id = (int) $migration->restore_id;
|
|
$job->save();
|
|
$migration->activate_job_id = (int) $job->id;
|
|
self::setState($migration, 'activation_running', '');
|
|
return $job;
|
|
}
|
|
|
|
private static function retryActivationLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
$migration->activate_job_id = null;
|
|
$migration->updated_at = time();
|
|
$migration->save();
|
|
return self::queueActivationLocked(
|
|
$migration, $instance, $actorUserId
|
|
);
|
|
}
|
|
|
|
private static function recoverRetryActivationLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): CloudJob {
|
|
return self::retryActivationLocked(
|
|
$migration,
|
|
$instance,
|
|
$actorUserId
|
|
);
|
|
}
|
|
|
|
private static function finishActivationLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $payload
|
|
): void {
|
|
unset($payload);
|
|
$instance = CloudInstance::where(
|
|
'id', (int) $migration->instance_id
|
|
)->lockForUpdate()->first();
|
|
if ($instance === null
|
|
|| (int) $migration->activate_job_id !== (int) $job->id
|
|
|| (int) $job->instance_id !== (int) $instance->id
|
|
|| (int) $job->host_id !== (int) $migration->target_host_id
|
|
|| (int) $instance->host_id !== (int) $migration->target_host_id
|
|
|| !hash_equals(strtolower((string) $instance->uuid),
|
|
strtolower((string) $migration->target_uuid))) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 激活任务与目标实例不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
if ($status === 'uncertain') {
|
|
self::setState($migration, 'attention_required',
|
|
'operation_effect_unknown', 'reconcile_activation');
|
|
return;
|
|
}
|
|
if ($status === 'failed') {
|
|
self::setState($migration, 'attention_required',
|
|
'migration_target_activation_failed',
|
|
(string) CloudInstanceMigrationWorkflow::failureAction(
|
|
'activation'
|
|
));
|
|
return;
|
|
}
|
|
if (!is_array($result)
|
|
|| !hash_equals(strtolower((string) $migration->target_uuid),
|
|
strtolower((string) ($result['uuid'] ?? '')))
|
|
|| !is_bool($result['running'] ?? null)
|
|
|| !is_bool($result['autostart_enabled'] ?? null)
|
|
|| $result['running'] !== (bool) $migration->source_was_running
|
|
|| $result['autostart_enabled']
|
|
!== (bool) $migration->source_autostart_enabled) {
|
|
throw new CloudError(
|
|
'目标 Agent 未确认实例恢复到迁移前电源和自启动状态', 502
|
|
);
|
|
}
|
|
self::advanceAfterActivationLocked(
|
|
$migration, $instance, (int) $job->user_id
|
|
);
|
|
}
|
|
|
|
private static function advanceAfterActivationLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
int $actorUserId
|
|
): void {
|
|
if ((string) $migration->cleanup_mode === 'delete') {
|
|
self::queueCleanupLocked($migration, $instance, [
|
|
'user_id' => $actorUserId,
|
|
'request_idempotency_key' => 'migration-cleanup-'
|
|
. (string) $migration->migration_key . '-'
|
|
. (int) $migration->retry_generation,
|
|
]);
|
|
return;
|
|
}
|
|
CloudInstanceMigrationSettlement::markRetainedLocked($migration);
|
|
self::setState($migration, 'source_retained', '');
|
|
}
|
|
|
|
private static function queueCleanupLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudInstance $instance,
|
|
array $actor
|
|
): CloudJob {
|
|
if (!in_array((string) $migration->state,
|
|
['cutover', 'activation_running', 'source_retained'], true)) {
|
|
throw new CloudError('当前迁移状态不能清理源副本', 409);
|
|
}
|
|
$generation = (int) $migration->retry_generation;
|
|
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
|
|
$actor,
|
|
(int) $migration->source_host_id,
|
|
'instance_migration_cleanup',
|
|
'POST',
|
|
'/v1/vms/' . rawurlencode((string) $migration->source_uuid)
|
|
. '/peer-migration/cleanup',
|
|
[
|
|
'migration_id' => (string) $migration->migration_key,
|
|
'remove_managed_disks' => true,
|
|
],
|
|
[
|
|
'owner_id' => (int) $instance->user_id,
|
|
'can_manage' => true,
|
|
'maintenance_migration_id' => (int) $migration->id,
|
|
],
|
|
[
|
|
'migration_id' => (int) $migration->id,
|
|
'generation' => $generation,
|
|
'source_host_id' => (int) $migration->source_host_id,
|
|
'source_uuid' => (string) $migration->source_uuid,
|
|
],
|
|
null,
|
|
'/v1/instance-maintenance-migrations/'
|
|
. (int) $migration->id . '/source/' . $generation
|
|
);
|
|
if (!$created) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源清理任务幂等冲突', 409
|
|
);
|
|
}
|
|
$job->instance_id = (int) $instance->id;
|
|
$job->backup_restore_id = (int) $migration->restore_id;
|
|
$job->save();
|
|
$migration->cleanup_job_id = (int) $job->id;
|
|
CloudInstanceMigrationSettlement::bindCleanupLocked($migration, $job);
|
|
self::setState($migration, 'cleanup_running', '');
|
|
return $job;
|
|
}
|
|
|
|
private static function finishCleanupLocked(
|
|
CloudInstanceMigration $migration,
|
|
CloudJob $job,
|
|
string $status,
|
|
$result,
|
|
array $payload
|
|
): void {
|
|
unset($payload);
|
|
if ((int) $migration->cleanup_job_id !== (int) $job->id
|
|
|| (int) $job->instance_id !== (int) $migration->instance_id
|
|
|| (int) $job->host_id !== (int) $migration->source_host_id) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 源清理任务与迁移账本不一致', 409
|
|
);
|
|
}
|
|
if ($status === 'running') {
|
|
return;
|
|
}
|
|
if ($status === 'uncertain') {
|
|
CloudInstanceMigrationSettlement::markCleanupAttentionLocked(
|
|
$migration,
|
|
$job,
|
|
'operation_effect_unknown'
|
|
);
|
|
self::setState($migration, 'source_retained',
|
|
'operation_effect_unknown', 'reconcile_cleanup');
|
|
return;
|
|
}
|
|
if ($status === 'failed') {
|
|
CloudInstanceMigrationSettlement::markCleanupFailedLocked(
|
|
$migration,
|
|
$job,
|
|
'migration_cleanup_failed'
|
|
);
|
|
self::setState($migration, 'source_retained',
|
|
'migration_cleanup_failed');
|
|
return;
|
|
}
|
|
if (!is_array($result)
|
|
|| ($result['deleted'] ?? null) !== true
|
|
|| ($result['managed_artifacts_absent'] ?? null) !== true) {
|
|
throw new CloudError(
|
|
'源 Agent 未确认实例和托管磁盘均已删除', 502
|
|
);
|
|
}
|
|
CloudInstanceMigrationSettlement::releaseSourceLocked(
|
|
$migration,
|
|
$job
|
|
);
|
|
self::setState($migration, 'completed', '');
|
|
$migration->completed_at = time();
|
|
$migration->save();
|
|
}
|
|
private static function setState(
|
|
CloudInstanceMigration $migration,
|
|
string $state,
|
|
string $errorCode,
|
|
string $recoveryAction = ''
|
|
): void {
|
|
CloudInstanceMigrationWorkflow::assertState(
|
|
$state,
|
|
$recoveryAction
|
|
);
|
|
$migration->state = $state;
|
|
$migration->error_code = $errorCode;
|
|
$migration->recovery_action = $recoveryAction;
|
|
$migration->updated_at = time();
|
|
$migration->save();
|
|
}
|
|
|
|
private static function sameDocument($left, $right): bool
|
|
{
|
|
$canonical = static function ($value) use (&$canonical) {
|
|
if (!is_array($value)) {
|
|
return $value;
|
|
}
|
|
if (!array_is_list($value)) {
|
|
ksort($value, SORT_STRING);
|
|
}
|
|
foreach ($value as $key => $child) {
|
|
$value[$key] = $canonical($child);
|
|
}
|
|
return $value;
|
|
};
|
|
return hash_equals(
|
|
hash('sha256', json_encode(
|
|
$canonical($left),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
)),
|
|
hash('sha256', json_encode(
|
|
$canonical($right),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
))
|
|
);
|
|
}
|
|
|
|
/** @return array{endpoints:list<string>,identity_sha256:string} */
|
|
private static function peerCapability(int $hostId, array $actor): array
|
|
{
|
|
$capabilities = CloudAgentSettingsService::clientForHost(
|
|
$hostId, 30.0
|
|
)->request('GET', '/v1/capabilities', $actor);
|
|
$peer = $capabilities['peer_migration'] ?? null;
|
|
$features = $capabilities['features'] ?? null;
|
|
$endpoints = is_array($peer) ? ($peer['endpoints'] ?? null) : null;
|
|
$identity = is_array($peer)
|
|
? ($peer['identity_sha256'] ?? null) : null;
|
|
$version = is_array($peer) ? filter_var(
|
|
$peer['version'] ?? null,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
) : false;
|
|
$chunk = is_array($peer) ? filter_var(
|
|
$peer['chunk_size_bytes'] ?? null,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
) : false;
|
|
if (!is_array($features)
|
|
|| ($features['peer_migration'] ?? null) !== true
|
|
|| !is_array($peer)
|
|
|| $version !== self::PROTOCOL_VERSION
|
|
|| ($peer['pinned_tls'] ?? null) !== true
|
|
|| $chunk !== self::CHUNK_SIZE
|
|
|| !self::validPeerEndpoints($endpoints)
|
|
|| !is_string($identity)
|
|
|| preg_match('/^[a-f0-9]{64}$/D', $identity) !== 1) {
|
|
throw new CloudError(
|
|
'宿主机未就绪 PeerMigration V3、TLS 1.3 身份锁定或固定分块能力',
|
|
409
|
|
);
|
|
}
|
|
return [
|
|
'endpoints' => array_values($endpoints),
|
|
'identity_sha256' => $identity,
|
|
];
|
|
}
|
|
|
|
private static function validPeerEndpoints(mixed $values): bool
|
|
{
|
|
if (!is_array($values) || !array_is_list($values)
|
|
|| count($values) < 1 || count($values) > 16) {
|
|
return false;
|
|
}
|
|
$seen = [];
|
|
foreach ($values as $value) {
|
|
if (!is_string($value) || !self::validPeerEndpoint($value)
|
|
|| isset($seen[$value])) {
|
|
return false;
|
|
}
|
|
$seen[$value] = true;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private static function validPeerEndpoint(string $value): bool
|
|
{
|
|
if ($value === '' || strlen($value) > 2048
|
|
|| trim($value) !== $value
|
|
|| preg_match('/[\\x00\\r\\n\\\\]/', $value) === 1) {
|
|
return false;
|
|
}
|
|
$parts = parse_url($value);
|
|
$host = is_array($parts) ? ($parts['host'] ?? null) : null;
|
|
if (!is_string($host)) {
|
|
return false;
|
|
}
|
|
$literal = str_starts_with($host, '[') && str_ends_with($host, ']')
|
|
? substr($host, 1, -1) : $host;
|
|
return is_array($parts)
|
|
&& ($parts['scheme'] ?? null) === 'https'
|
|
&& filter_var($literal, FILTER_VALIDATE_IP) !== false
|
|
&& filter_var($parts['port'] ?? null, FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1, 'max_range' => 65535]])
|
|
!== false
|
|
&& !isset($parts['user'], $parts['pass'], $parts['query'],
|
|
$parts['fragment'])
|
|
&& (!isset($parts['path']) || $parts['path'] === ''
|
|
|| $parts['path'] === '/');
|
|
}
|
|
|
|
/** @param list<string> $endpoints */
|
|
private static function encodePeerEndpoints(array $endpoints): string
|
|
{
|
|
if (!self::validPeerEndpoints($endpoints)) {
|
|
throw new CloudError('源宿主机的 PeerMigration 地址无效', 409);
|
|
}
|
|
$encoded = json_encode(array_values($endpoints),
|
|
JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
|
if (strlen($encoded) > 2048) {
|
|
throw new CloudError('源宿主机的 PeerMigration 地址过多', 409);
|
|
}
|
|
return $encoded;
|
|
}
|
|
|
|
/** @return list<string> */
|
|
private static function decodePeerEndpoints(string $encoded): array
|
|
{
|
|
try {
|
|
$endpoints = json_decode($encoded, true, 16,
|
|
JSON_THROW_ON_ERROR);
|
|
} catch (\Throwable) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'PeerMigration 源地址目录无法读取', 409
|
|
);
|
|
}
|
|
if (!self::validPeerEndpoints($endpoints)) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'PeerMigration 源地址目录无效', 409
|
|
);
|
|
}
|
|
return array_values($endpoints);
|
|
}
|
|
|
|
private static function catalogAvailable(): bool
|
|
{
|
|
try {
|
|
$schema = (new CloudInstanceMigration())->getConnection()
|
|
->getSchemaBuilder();
|
|
return $schema->hasTable('cloud_instance_migrations')
|
|
&& $schema->hasTable('cloud_peer_migration_artifacts');
|
|
} catch (\Throwable $error) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private static function resources(array $spec): array
|
|
{
|
|
$values = [
|
|
'cpu_cores' => $spec['vcpu'] ?? $spec['cpu_cores'] ?? null,
|
|
'memory_mib' => $spec['memory_mib'] ?? $spec['ram_mib'] ?? null,
|
|
'disk_gib' => $spec['disk_gib'] ?? null,
|
|
'bandwidth_mbps' => $spec['bandwidth_mbps'] ?? null,
|
|
];
|
|
foreach ($values as $field => $value) {
|
|
$parsed = filter_var($value, FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]);
|
|
if ($parsed === false) {
|
|
throw new CloudError('实例缺少可信的资源规格快照', 409);
|
|
}
|
|
$values[$field] = (int) $parsed;
|
|
}
|
|
return $values;
|
|
}
|
|
|
|
private static function finalizeTargetSpec(
|
|
array $source,
|
|
array $body,
|
|
array $vm,
|
|
int $resourcePoolId
|
|
): array {
|
|
if ($resourcePoolId < 1) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标资源池快照无效', 409
|
|
);
|
|
}
|
|
$expected = self::targetRestoreSnapshot($body);
|
|
try {
|
|
$actual = self::targetRestoreSnapshot($vm);
|
|
} catch (CloudError $error) {
|
|
throw new CloudError(
|
|
'目标 Agent 返回的资源或网络快照不一致', 502
|
|
);
|
|
}
|
|
if ($expected !== $actual) {
|
|
throw new CloudError(
|
|
'目标 Agent 返回的资源或网络快照不一致', 502
|
|
);
|
|
}
|
|
$spec = CloudInstanceMetadata::merge($source, $vm);
|
|
foreach ([
|
|
'network', 'public_pool_key', 'public_pool_cidr',
|
|
'public_ip_cidr', 'public_ip', 'gateway', 'dns_servers',
|
|
] as $retired) {
|
|
unset($spec[$retired]);
|
|
}
|
|
return array_replace($spec, [
|
|
'vcpu' => $expected['vcpu'],
|
|
'cpu_cores' => $expected['vcpu'],
|
|
'memory_mib' => $expected['memory_mib'],
|
|
'ram_mib' => $expected['memory_mib'],
|
|
'disk_gib' => $expected['disk_gib'],
|
|
'bandwidth_mbps' => $expected['bandwidth_mbps'],
|
|
'public_network' => $expected['public_network'],
|
|
'resource_pool_id' => $resourcePoolId,
|
|
]);
|
|
}
|
|
|
|
/** @return array<string,mixed> */
|
|
private static function targetRestoreSnapshot(array $value): array
|
|
{
|
|
$integers = [];
|
|
foreach ([
|
|
'vcpu' => [1, 512],
|
|
'memory_mib' => [128, 16 * 1024 * 1024],
|
|
'disk_gib' => [1, 4096],
|
|
'bandwidth_mbps' => [1, 100000],
|
|
] as $field => [$minimum, $maximum]) {
|
|
$parsed = filter_var($value[$field] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => [
|
|
'min_range' => $minimum, 'max_range' => $maximum,
|
|
]]);
|
|
if ($parsed === false) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标资源规格快照无效', 409
|
|
);
|
|
}
|
|
$integers[$field] = (int) $parsed;
|
|
}
|
|
try {
|
|
$publicNetwork = CloudPublicNetworkSnapshot::fromArray(
|
|
$value['public_network'] ?? null
|
|
);
|
|
} catch (\InvalidArgumentException) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'PeerMigration 目标网络规格快照无效', 409
|
|
);
|
|
}
|
|
return $integers + [
|
|
'public_network' => $publicNetwork->toArray(),
|
|
];
|
|
}
|
|
|
|
private static function decodeSpec(string $encoded): array
|
|
{
|
|
$spec = json_decode($encoded, true);
|
|
if (!is_array($spec) || array_is_list($spec)) {
|
|
throw new CloudError('实例规格快照损坏', 409);
|
|
}
|
|
return $spec;
|
|
}
|
|
|
|
private static function uuidV4(): string
|
|
{
|
|
$bytes = random_bytes(16);
|
|
$bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40);
|
|
$bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80);
|
|
$hex = bin2hex($bytes);
|
|
return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-'
|
|
. substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-'
|
|
. substr($hex, 20, 12);
|
|
}
|
|
}
|