You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudMaintenanceMigrationFi...

1280 lines
50 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudInstanceMigration;
use App\Models\CloudInstanceMigrationSourceDebt;
use App\Models\CloudJob;
use App\Services\Cloud\CloudInstanceMigrationService;
use App\Services\Cloud\CloudInstanceMigrationSettlement;
use App\Services\Cloud\CloudPublicNetworkAllocationService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$rejects = static function (callable $callback, string $message): void {
try {
$callback();
} catch (Throwable) {
return;
}
throw new RuntimeException($message);
};
$captures = static function (callable $callback, string $message): Throwable {
try {
$callback();
} catch (Throwable $error) {
return $error;
}
throw new RuntimeException($message);
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
});
$schema->create('cloud_resource_pools', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('line_id');
});
$schema->create('cloud_host_network_attachments', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('line_id');
$table->string('network_selector');
});
$schema->create('cloud_public_ip_pools', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('pool_key');
$table->integer('host_id');
$table->integer('network_attachment_id');
$table->integer('address_family');
$table->string('cidr');
$table->string('gateway');
$table->boolean('gateway_on_link');
$table->integer('guest_prefix_length');
$table->text('dns_servers_json');
$table->text('reserved_ips_json');
$table->string('allocation_start')->nullable();
$table->string('allocation_end')->nullable();
$table->string('allocation_cursor')->nullable();
$table->integer('max_allocations');
});
$schema->create('cloud_public_ips', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('public_ip_pool_id');
$table->integer('address_family');
$table->string('address');
$table->binary('address_binary');
$table->integer('prefix_length');
$table->string('state');
$table->integer('reservation_id')->nullable();
$table->integer('reservation_ip_claim_id')->nullable();
$table->integer('instance_id')->nullable();
$table->string('operation_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_resource_reservation_ips', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('reservation_id');
$table->integer('address_family');
$table->integer('claim_generation')->default(1);
$table->integer('active_claim')->nullable()->default(1);
$table->integer('network_attachment_id');
$table->integer('public_ip_pool_id');
$table->integer('public_ip_id');
$table->string('operation_id');
$table->string('state');
$table->integer('released_at')->nullable();
$table->string('release_reason')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_resource_reservations', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('reservation_key');
$table->string('operation_id');
$table->integer('order_id')->nullable();
$table->integer('backup_restore_id')->nullable();
$table->string('backup_restore_attempt_key')->nullable();
$table->integer('migration_source_debt_id')->nullable();
$table->integer('offer_id')->nullable();
$table->integer('resource_pool_id');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('instance_id')->nullable();
$table->integer('network_attachment_id')->nullable();
$table->integer('public_ip_pool_id')->nullable();
$table->integer('public_ip_id')->nullable();
$table->integer('cpu_cores');
$table->integer('memory_mib');
$table->integer('disk_gib');
$table->string('state');
$table->integer('expires_at')->nullable();
$table->integer('released_at')->nullable();
$table->string('release_reason')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('order_id');
$table->integer('plan_id');
$table->integer('offer_id');
$table->integer('resource_pool_id');
$table->integer('reservation_id');
$table->integer('current_period_id');
$table->integer('expires_at');
$table->decimal('renewal_price', 12, 2);
$table->integer('renewal_days');
$table->integer('renewal_traffic_bytes');
$table->integer('traffic_quota_bytes');
$table->integer('lifetime_download_bytes');
$table->integer('lifetime_upload_bytes');
$table->integer('traffic_synced_at')->default(0);
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->text('spec_json');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_orders', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('user_id');
$table->integer('plan_id');
$table->integer('offer_id');
$table->integer('resource_pool_id');
$table->integer('host_id');
$table->integer('reservation_id');
$table->decimal('amount', 12, 2);
$table->integer('instance_id');
$table->string('status');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_backup_restores', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('target_host_id');
$table->integer('reservation_id')->nullable();
$table->integer('job_id')->nullable();
$table->string('operation_id');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('ready_at')->nullable();
$table->integer('updated_at');
});
$schema->create('cloud_instance_migrations', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('migration_key');
$table->integer('instance_id');
$table->integer('restore_id');
$table->integer('source_host_id');
$table->string('source_uuid');
$table->integer('source_reservation_id');
$table->integer('target_host_id');
$table->string('target_uuid');
$table->integer('target_reservation_id');
$table->integer('vpc_port_id')->nullable();
$table->integer('source_prepare_job_id')->nullable();
$table->integer('target_receive_job_id')->nullable();
$table->integer('firewall_job_id')->nullable();
$table->integer('activate_job_id')->nullable();
$table->integer('rollback_job_id')->nullable();
$table->integer('cleanup_job_id')->nullable();
$table->string('cleanup_mode');
$table->string('state');
$table->string('recovery_action')->default('');
$table->integer('retry_generation')->default(0);
$table->string('manifest_digest')->default('');
$table->boolean('source_was_running')->nullable();
$table->boolean('source_autostart_enabled')->nullable();
$table->string('source_service_status');
$table->string('source_network_status');
$table->string('error_code')->default('');
$table->integer('cutover_at')->nullable();
$table->integer('completed_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_peer_migration_artifacts', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('migration_id');
$table->string('artifact_id');
$table->string('role');
$table->integer('source_snapshot_id')->nullable();
$table->string('snapshot_key')->nullable();
$table->string('name')->default('');
$table->string('consistency');
$table->integer('object_size_bytes');
$table->integer('virtual_size_bytes');
$table->string('sha256');
$table->string('state');
$table->integer('snapshot_created_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instance_runtime', static function (
Blueprint $table
): void {
$table->integer('instance_id')->primary();
$table->integer('host_id');
$table->string('uuid');
$table->string('power_state');
$table->string('sync_state');
$table->integer('last_success_at');
$table->integer('last_attempt_at');
$table->integer('revision');
$table->string('last_error_code');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instance_migration_source_debts', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('debt_key');
$table->integer('migration_id');
$table->integer('source_reservation_id');
$table->integer('source_host_id');
$table->string('source_uuid');
$table->integer('source_network_attachment_id');
$table->text('source_identity_json');
$table->string('source_identity_sha256');
$table->string('source_claim_set_sha256');
$table->text('source_public_network_json');
$table->string('source_public_network_sha256');
$table->string('cleanup_operation_id')->nullable();
$table->integer('cleanup_job_id')->nullable();
$table->string('state');
$table->integer('active_debt')->nullable();
$table->string('last_error_code')->default('');
$table->integer('resolved_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$now = time();
Capsule::table('cloud_hosts')->insert([['id' => 1], ['id' => 2]]);
Capsule::table('cloud_resource_pools')->insert(['id' => 10, 'line_id' => 7]);
Capsule::table('cloud_host_network_attachments')->insert([
['id' => 11, 'host_id' => 1, 'line_id' => 7,
'network_selector' => 'bridge:br0'],
['id' => 12, 'host_id' => 2, 'line_id' => 7,
'network_selector' => 'bridge:br0'],
]);
$pool = static function (int $id, int $hostId, int $attachmentId,
int $family, string $cidr, string $gateway): array {
$network = explode('/', $cidr, 2)[0];
return [
'id' => $id,
'pool_key' => md5('pool-' . $id),
'host_id' => $hostId,
'network_attachment_id' => $attachmentId,
'address_family' => $family,
'cidr' => $cidr,
'gateway' => $gateway,
'gateway_on_link' => 0,
'guest_prefix_length' => $family === 4 ? 24 : 64,
'dns_servers_json' => '["1.1.1.1"]',
'reserved_ips_json' => '[]',
'allocation_start' => $family === 6 ? $network . '100' : null,
'allocation_end' => $family === 6 ? $network . '1ff' : null,
'allocation_cursor' => null,
'max_allocations' => $family === 6 ? 100 : 0,
];
};
Capsule::table('cloud_public_ip_pools')->insert([
$pool(101, 1, 11, 4, '8.8.10.0/24', '8.8.10.1'),
$pool(201, 1, 11, 6, '2404:c140:156:10::/64',
'2404:c140:156:10::1'),
$pool(102, 2, 12, 4, '8.8.20.0/24', '8.8.20.1'),
$pool(202, 2, 12, 6, '2404:c140:156:20::/64',
'2404:c140:156:20::1'),
]);
$seed = static function (int $n) use ($now): array {
$instanceId = 100 + $n;
$orderId = 200 + $n;
$restoreId = 300 + $n;
$migrationId = 400 + $n;
$sourceReservationId = 500 + ($n * 2);
$targetReservationId = $sourceReservationId + 1;
$sourceOperation = md5('source-' . $n);
$targetOperation = md5('target-' . $n);
$sourceUuid = sprintf(
'11111111-1111-4111-8111-%012d',
$instanceId
);
$targetUuid = sprintf(
'22222222-2222-4222-8222-%012d',
$instanceId
);
$sourceAddresses = [
4 => '8.8.10.' . (20 + $n),
6 => '2404:c140:156:10::1' . dechex(20 + $n),
];
$targetAddresses = [
4 => '8.8.20.' . (20 + $n),
6 => '2404:c140:156:20::1' . dechex(20 + $n),
];
$sourceNetwork = [
'contract' => 1,
'network_selector' => 'bridge:br0',
'ipv4' => [
'pool_key' => md5('pool-101'),
'pool_cidr' => '8.8.10.0/24',
'address_cidr' => $sourceAddresses[4] . '/24',
'gateway' => '8.8.10.1',
'gateway_on_link' => false,
],
'ipv6' => [
'pool_key' => md5('pool-201'),
'pool_cidr' => '2404:c140:156:10::/64',
'address_cidr' => $sourceAddresses[6] . '/64',
'gateway' => '2404:c140:156:10::1',
'gateway_on_link' => false,
],
'dns_servers' => ['1.1.1.1'],
];
Capsule::table('cloud_instances')->insert([
'id' => $instanceId, 'uuid' => $sourceUuid, 'host_id' => 1,
'user_id' => 9, 'order_id' => $orderId, 'plan_id' => 31,
'offer_id' => 41, 'resource_pool_id' => 10,
'reservation_id' => $sourceReservationId,
'current_period_id' => 700 + $n, 'expires_at' => $now + 86400,
'renewal_price' => '19.95', 'renewal_days' => 30,
'renewal_traffic_bytes' => 2000000,
'traffic_quota_bytes' => 1500000,
'lifetime_download_bytes' => 800000,
'lifetime_upload_bytes' => 300000,
'name' => 'vm-' . $n, 'status' => 'active',
'service_status' => 'active', 'network_status' => 'active',
'spec_json' => json_encode([
'managed' => true,
'vcpu' => 2,
'cpu_cores' => 2,
'memory_mib' => 2048,
'ram_mib' => 2048,
'disk_gib' => 40,
'bandwidth_mbps' => 1000,
'public_network' => $sourceNetwork,
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
'created_at' => $now - 1000,
'updated_at' => $now,
]);
Capsule::table('cloud_orders')->insert([
'id' => $orderId, 'user_id' => 9, 'plan_id' => 31,
'offer_id' => 41, 'resource_pool_id' => 10, 'host_id' => 1,
'reservation_id' => $sourceReservationId, 'amount' => '12.34',
'instance_id' => $instanceId, 'status' => 'active',
'created_at' => $now - 1000, 'updated_at' => $now,
]);
Capsule::table('cloud_backup_restores')->insert([
'id' => $restoreId, 'target_host_id' => 2,
'reservation_id' => $targetReservationId, 'job_id' => null,
'operation_id' => $targetOperation, 'state' => 'restoring',
'error_code' => '', 'updated_at' => $now,
]);
Capsule::table('cloud_resource_reservations')->insert([
[
'id' => $sourceReservationId,
'reservation_key' => md5('source-reservation-' . $n),
'operation_id' => $sourceOperation, 'order_id' => $orderId,
'backup_restore_id' => null,
'backup_restore_attempt_key' => null,
'migration_source_debt_id' => null, 'offer_id' => 41,
'resource_pool_id' => 10, 'host_id' => 1, 'user_id' => 9,
'instance_id' => $instanceId, 'network_attachment_id' => null,
'public_ip_pool_id' => null, 'public_ip_id' => null,
'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40,
'state' => 'committed', 'expires_at' => null,
'released_at' => null, 'release_reason' => '',
'created_at' => $now - 1000, 'updated_at' => $now,
],
[
'id' => $targetReservationId,
'reservation_key' => md5('target-reservation-' . $n),
'operation_id' => $targetOperation, 'order_id' => null,
'backup_restore_id' => $restoreId,
'backup_restore_attempt_key' => null,
'migration_source_debt_id' => null, 'offer_id' => null,
'resource_pool_id' => 10, 'host_id' => 2, 'user_id' => 9,
'instance_id' => null, 'network_attachment_id' => null,
'public_ip_pool_id' => null, 'public_ip_id' => null,
'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40,
'state' => 'held', 'expires_at' => $now + 600,
'released_at' => null, 'release_reason' => '',
'created_at' => $now - 100, 'updated_at' => $now,
],
]);
foreach ([
[$sourceReservationId, 1, 11, 101, 201,
$sourceAddresses, $sourceOperation, 'committed', $instanceId],
[$targetReservationId, 2, 12, 102, 202,
$targetAddresses, $targetOperation, 'reserved', null],
] as [$reservationId, $hostId, $attachmentId, $v4Pool, $v6Pool,
$addresses, $operation, $state, $ownerInstanceId]) {
foreach ([4 => $v4Pool, 6 => $v6Pool] as $family => $poolId) {
$address = $addresses[$family];
$ipId = Capsule::table('cloud_public_ips')->insertGetId([
'host_id' => $hostId, 'public_ip_pool_id' => $poolId,
'address_family' => $family, 'address' => $address,
'address_binary' => inet_pton($address),
'prefix_length' => $family === 4 ? 24 : 64,
'state' => $state === 'committed' ? 'assigned' : 'reserved',
'reservation_id' => $reservationId,
'reservation_ip_claim_id' => null,
'instance_id' => $ownerInstanceId,
'operation_id' => $operation,
'created_at' => $now, 'updated_at' => $now,
]);
$claimId = Capsule::table('cloud_resource_reservation_ips')
->insertGetId([
'reservation_id' => $reservationId,
'address_family' => $family, 'claim_generation' => 1,
'active_claim' => 1,
'network_attachment_id' => $attachmentId,
'public_ip_pool_id' => $poolId,
'public_ip_id' => $ipId, 'operation_id' => $operation,
'state' => $state, 'released_at' => null,
'release_reason' => '', 'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_public_ips')->where('id', $ipId)->update([
'reservation_ip_claim_id' => $claimId,
]);
}
}
Capsule::table('cloud_instance_migrations')->insert([
'id' => $migrationId, 'migration_key' => md5('migration-' . $n),
'instance_id' => $instanceId, 'restore_id' => $restoreId,
'source_host_id' => 1, 'source_uuid' => $sourceUuid,
'source_reservation_id' => $sourceReservationId,
'target_host_id' => 2, 'target_uuid' => $targetUuid,
'target_reservation_id' => $targetReservationId,
'cleanup_job_id' => null, 'cleanup_mode' => 'delete',
'state' => 'transferring', 'recovery_action' => '',
'retry_generation' => 0, 'source_service_status' => 'active',
'source_network_status' => 'active', 'error_code' => '',
'created_at' => $now - 100, 'updated_at' => $now,
]);
$sourceSet = CloudPublicNetworkAllocationService::currentClaimSet(
$sourceReservationId,
'committed',
$instanceId
);
$targetSet = CloudPublicNetworkAllocationService::currentClaimSet(
$targetReservationId,
'held'
);
Capsule::table('cloud_instances')->where('id', $instanceId)->update([
'spec_json' => json_encode([
'managed' => true,
'vcpu' => 2,
'cpu_cores' => 2,
'memory_mib' => 2048,
'ram_mib' => 2048,
'disk_gib' => 40,
'bandwidth_mbps' => 1000,
'public_network' => $sourceSet->publicNetwork()->toArray(),
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
]);
return compact(
'instanceId', 'orderId', 'restoreId', 'migrationId',
'sourceReservationId', 'targetReservationId',
'sourceOperation', 'targetOperation', 'sourceUuid', 'targetUuid',
'sourceSet', 'targetSet'
);
};
$cutover = static function (array $case): CloudInstanceMigrationSourceDebt {
$handoff = CloudPublicNetworkAllocationService::handoffMigrationBundles(
$case['sourceReservationId'],
$case['targetReservationId'],
$case['instanceId'],
$case['sourceOperation'],
$case['targetOperation']
);
if (!hash_equals(
$case['sourceSet']->digest(),
$handoff['source']->digest()
) || !hash_equals(
$case['targetSet']->digest(),
$handoff['target']->digest()
)) {
throw new RuntimeException('Migration handoff changed claim identity.');
}
Capsule::table('cloud_instances')->where('id', $case['instanceId'])
->update([
'uuid' => $case['targetUuid'],
'host_id' => 2,
'reservation_id' => $case['targetReservationId'],
'spec_json' => json_encode([
'managed' => true,
'public_network' =>
$handoff['target']->publicNetwork()->toArray(),
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
'updated_at' => time(),
]);
$migration = CloudInstanceMigration::findOrFail($case['migrationId']);
return CloudInstanceMigrationSettlement::settleCutoverLocked(
$migration,
$handoff['source']
);
};
$claimIdentity = static function (int $reservationId): array {
return array_map(
static fn (object $row): array => [
(int) $row->id,
(int) $row->reservation_id,
(int) $row->public_ip_id,
(string) $row->operation_id,
(int) $row->claim_generation,
],
Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', $reservationId)
->orderBy('address_family')->orderBy('claim_generation')
->orderBy('id')->get()->all()
);
};
$job = static function (array $case, int $id, string $operation): CloudJob {
$job = new CloudJob();
$job->id = $id;
$job->kind = 'instance_migration_cleanup';
$job->instance_id = $case['instanceId'];
$job->host_id = 1;
$job->backup_restore_id = $case['restoreId'];
$job->operation_id = $operation;
return $job;
};
$migrationState = static function (array $case): array {
$rows = static function ($query): array {
return $query->orderBy('id')->get()
->map(static fn ($row): array => (array) $row)->all();
};
return [
'instance' => (array) Capsule::table('cloud_instances')
->find($case['instanceId']),
'order' => (array) Capsule::table('cloud_orders')
->find($case['orderId']),
'restore' => (array) Capsule::table('cloud_backup_restores')
->find($case['restoreId']),
'migration' => (array) Capsule::table('cloud_instance_migrations')
->find($case['migrationId']),
'reservations' => $rows(Capsule::table('cloud_resource_reservations')
->whereIn('id', [
$case['sourceReservationId'],
$case['targetReservationId'],
])),
'claims' => $rows(Capsule::table('cloud_resource_reservation_ips')
->whereIn('reservation_id', [
$case['sourceReservationId'],
$case['targetReservationId'],
])),
'ips' => $rows(Capsule::table('cloud_public_ips')
->whereIn('reservation_id', [
$case['sourceReservationId'],
$case['targetReservationId'],
])),
'debts' => $rows(Capsule::table(
'cloud_instance_migration_source_debts'
)->where('migration_id', $case['migrationId'])),
'artifacts' => $rows(Capsule::table('cloud_peer_migration_artifacts')
->where('migration_id', $case['migrationId'])),
'runtime' => $rows(Capsule::table('cloud_instance_runtime')
->where('instance_id', $case['instanceId'])),
];
};
// Exercise the real private transfer finalizer under the same outer
// transaction supplied by the worker. A deterministic failure at the runtime
// cache write happens after network handoff, artifact sealing, placement
// switch, commercial settlement, and restore finalization; every one of those
// writes must roll back together.
$atomic = $seed(5);
$atomicArtifactId = md5('migration-root-' . $atomic['migrationId']);
$atomicArtifactSha = hash('sha256', 'migration-root-bytes');
$atomicManifest = hash('sha256', 'migration-manifest');
$atomicJobId = 9105;
Capsule::table('cloud_peer_migration_artifacts')->insert([
'migration_id' => $atomic['migrationId'],
'artifact_id' => $atomicArtifactId,
'role' => 'root',
'source_snapshot_id' => null,
'snapshot_key' => null,
'name' => '',
'consistency' => 'crash',
'object_size_bytes' => 123456,
'virtual_size_bytes' => 40 * 1073741824,
'sha256' => $atomicArtifactSha,
'state' => 'ready',
'snapshot_created_at' => null,
'created_at' => $now - 20,
'updated_at' => $now,
]);
Capsule::table('cloud_instance_migrations')
->where('id', $atomic['migrationId'])->update([
'target_receive_job_id' => $atomicJobId,
'manifest_digest' => $atomicManifest,
'source_was_running' => 1,
'source_autostart_enabled' => 1,
]);
$atomicJob = new CloudJob();
$atomicJob->id = $atomicJobId;
$atomicJob->instance_id = $atomic['instanceId'];
$atomicJob->backup_restore_id = $atomic['restoreId'];
$atomicJob->host_id = 2;
$atomicJob->user_id = 9;
$atomicJob->operation_id = $atomic['targetOperation'];
$atomicTargetNetwork = $atomic['targetSet']->publicNetwork()->toArray();
$atomicPayload = [
'maintenance_migration_id' => $atomic['migrationId'],
'body' => [
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 40,
'bandwidth_mbps' => 1000,
'public_network' => $atomicTargetNetwork,
'vpc_port' => null,
],
];
$atomicResult = [
'migration_id' => md5('migration-5'),
'manifest_digest' => $atomicManifest,
'uuid' => $atomic['targetUuid'],
'status' => 'shutoff',
'staged_powered_off' => true,
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 40,
'bandwidth_mbps' => 1000,
'managed' => true,
'public_network' => $atomicTargetNetwork,
'vpc_port' => null,
'artifacts' => [[
'id' => $atomicArtifactId,
'role' => 'root',
'snapshot_key' => '',
'name' => '',
'consistency' => 'crash',
'size_bytes' => 123456,
'virtual_size_bytes' => 40 * 1073741824,
'sha256' => $atomicArtifactSha,
'created_at' => $now - 20,
]],
];
$finishTransfer = new ReflectionMethod(
CloudInstanceMigrationService::class,
'finishTransferLocked'
);
$beforeAtomic = $migrationState($atomic);
Capsule::statement(
"CREATE TRIGGER fail_migration_runtime_seed\n"
. "BEFORE INSERT ON cloud_instance_runtime\n"
. "BEGIN SELECT RAISE(ABORT, 'forced runtime seed failure'); END"
);
try {
$atomicError = $captures(
static fn () => Capsule::connection()->transaction(
static fn () => $finishTransfer->invoke(
null,
CloudInstanceMigration::findOrFail($atomic['migrationId']),
$atomicJob,
'succeeded',
$atomicResult,
$atomicPayload
)
),
'Runtime-cache failure did not abort the transfer finalizer.'
);
} finally {
Capsule::statement('DROP TRIGGER fail_migration_runtime_seed');
}
$assert(str_contains($atomicError->getMessage(),
'forced runtime seed failure'),
'Transfer finalizer failed before the intended post-settlement fault.');
$assert($beforeAtomic === $migrationState($atomic),
'Outer transaction retained a partial transfer finalization.');
// A confirmed source rollback must restore the original service and release
// the complete prepared target bundle.
$rolledBackCase = $seed(6);
$rollbackJobId = 9206;
Capsule::table('cloud_instance_migrations')
->where('id', $rolledBackCase['migrationId'])->update([
'target_receive_job_id' => 9196,
'rollback_job_id' => $rollbackJobId,
'state' => 'rollback_running',
'source_was_running' => 1,
'source_autostart_enabled' => 1,
'error_code' => 'migration_restore_failed',
]);
$rollbackJob = new CloudJob();
$rollbackJob->id = $rollbackJobId;
$rollbackJob->instance_id = $rolledBackCase['instanceId'];
$rollbackJob->host_id = 1;
$finishRollback = new ReflectionMethod(
CloudInstanceMigrationService::class,
'finishRollbackLocked'
);
$finishRollback->invoke(
null,
CloudInstanceMigration::findOrFail($rolledBackCase['migrationId']),
$rollbackJob,
'succeeded',
[
'uuid' => $rolledBackCase['sourceUuid'],
'running' => true,
'autostart_enabled' => true,
],
[]
);
$rolledBack = CloudInstanceMigration::findOrFail(
$rolledBackCase['migrationId']
);
$assert((string) $rolledBack->state === 'rolled_back'
&& (string) $rolledBack->error_code === 'migration_transfer_aborted'
&& $rolledBack->completed_at !== null
&& (int) CloudInstance::findOrFail(
$rolledBackCase['instanceId']
)->host_id === 1
&& Capsule::table('cloud_public_ips')
->where('reservation_id', $rolledBackCase['sourceReservationId'])
->where('state', 'assigned')->count() === 2
&& Capsule::table('cloud_public_ips')
->where('state', 'available')->count() >= 2,
'Confirmed rollback did not preserve source or release target dual stack.');
// Activation must preserve a powered-off expired service exactly as frozen at
// migration start, including the durable guest hostname.
$poweredOffCase = $seed(7);
$poweredOffDebt = $cutover($poweredOffCase);
$poweredSpec = json_decode((string) CloudInstance::findOrFail(
$poweredOffCase['instanceId']
)->spec_json, true, 16, JSON_THROW_ON_ERROR);
$poweredSpec['guest_hostname'] = 'retained.example';
Capsule::table('cloud_instances')
->where('id', $poweredOffCase['instanceId'])->update([
'service_status' => 'suspended_expired',
'network_status' => 'blocked_expired',
'spec_json' => json_encode($poweredSpec,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
]);
$activationJobId = 9307;
Capsule::table('cloud_instance_migrations')
->where('id', $poweredOffCase['migrationId'])->update([
'activate_job_id' => $activationJobId,
'cleanup_mode' => 'retain',
'state' => 'activation_running',
'source_was_running' => 0,
'source_autostart_enabled' => 1,
'source_service_status' => 'suspended_expired',
'source_network_status' => 'blocked_expired',
]);
$activationJob = new CloudJob();
$activationJob->id = $activationJobId;
$activationJob->instance_id = $poweredOffCase['instanceId'];
$activationJob->host_id = 2;
$activationJob->user_id = 9;
$finishActivation = new ReflectionMethod(
CloudInstanceMigrationService::class,
'finishActivationLocked'
);
$finishActivation->invoke(
null,
CloudInstanceMigration::findOrFail($poweredOffCase['migrationId']),
$activationJob,
'succeeded',
[
'uuid' => $poweredOffCase['targetUuid'],
'running' => false,
'autostart_enabled' => true,
],
[]
);
$poweredOff = CloudInstance::findOrFail($poweredOffCase['instanceId']);
$activated = CloudInstanceMigration::findOrFail(
$poweredOffCase['migrationId']
);
$activatedDebt = CloudInstanceMigrationSourceDebt::findOrFail(
(int) $poweredOffDebt->id
);
$assert((string) $activated->state === 'source_retained'
&& (string) $activatedDebt->state === 'retained'
&& (string) $poweredOff->service_status === 'suspended_expired'
&& (string) $poweredOff->network_status === 'blocked_expired'
&& str_contains((string) $poweredOff->spec_json, 'retained.example'),
'Activation changed powered-off, expired-network, or hostname state.');
// Automatic cleanup: cutover creates prepared debt, then the exact cleanup
// operation releases both source families without moving either claim set.
$automatic = $seed(1);
$historicalSourceOperation = md5('source-history-automatic');
$activeSourceClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', $automatic['sourceReservationId'])
->where('active_claim', 1)->orderBy('address_family')->get()->all();
foreach ($activeSourceClaims as $activeSourceClaim) {
Capsule::table('cloud_resource_reservation_ips')->insert([
'reservation_id' => $automatic['sourceReservationId'],
'address_family' => (int) $activeSourceClaim->address_family,
'claim_generation' => 1,
'active_claim' => null,
'network_attachment_id' =>
(int) $activeSourceClaim->network_attachment_id,
'public_ip_pool_id' => (int) $activeSourceClaim->public_ip_pool_id,
'public_ip_id' => (int) $activeSourceClaim->public_ip_id,
'operation_id' => $historicalSourceOperation,
'state' => 'released',
'released_at' => $now - 60,
'release_reason' => 'network_change_completed',
'created_at' => $now - 120,
'updated_at' => $now - 60,
]);
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $activeSourceClaim->id)
->update(['claim_generation' => 2]);
}
$automatic['sourceSet'] = CloudPublicNetworkAllocationService::currentClaimSet(
$automatic['sourceReservationId'],
'committed',
$automatic['instanceId']
);
$sourceIdentityBefore = $claimIdentity($automatic['sourceReservationId']);
$targetIdentityBefore = $claimIdentity($automatic['targetReservationId']);
$debt = $cutover($automatic);
$migration = CloudInstanceMigration::findOrFail($automatic['migrationId']);
$sameDebt = CloudInstanceMigrationSettlement::settleCutoverLocked(
$migration,
$automatic['sourceSet']
);
$assert((int) $sameDebt->id === (int) $debt->id,
'Cutover retry created a second source debt.');
$cleanup = $job($automatic, 9001, md5('cleanup-automatic'));
$migration->cleanup_job_id = (int) $cleanup->id;
$migration->save();
CloudInstanceMigrationSettlement::bindCleanupLocked($migration, $cleanup);
CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup);
CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup);
$wrongCleanup = $job($automatic, 9010, md5('cleanup-wrong'));
$rejects(
static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked(
$migration,
$wrongCleanup
),
'Released debt accepted a different cleanup generation.'
);
$instance = Capsule::table('cloud_instances')
->find($automatic['instanceId']);
$order = Capsule::table('cloud_orders')->find($automatic['orderId']);
$source = Capsule::table('cloud_resource_reservations')
->find($automatic['sourceReservationId']);
$target = Capsule::table('cloud_resource_reservations')
->find($automatic['targetReservationId']);
$releasedDebt = Capsule::table('cloud_instance_migration_source_debts')
->find((int) $debt->id);
$assert((int) $instance->reservation_id === $automatic['targetReservationId']
&& (int) $order->reservation_id === $automatic['targetReservationId']
&& (int) $target->order_id === $automatic['orderId']
&& $target->backup_restore_id === null
&& (string) $target->state === 'committed'
&& (int) $source->migration_source_debt_id === (int) $debt->id
&& $source->order_id === null
&& (string) $source->state === 'released'
&& (string) $releasedDebt->state === 'released'
&& $releasedDebt->active_debt === null
&& $releasedDebt->resolved_at !== null,
'Migration did not settle target ownership and source cleanup debt.');
$assert($sourceIdentityBefore
=== $claimIdentity($automatic['sourceReservationId'])
&& $targetIdentityBefore
=== $claimIdentity($automatic['targetReservationId'])
&& (int) CloudInstanceMigration::findOrFail(
$automatic['migrationId']
)->source_reservation_id === $automatic['sourceReservationId']
&& (int) CloudInstanceMigration::findOrFail(
$automatic['migrationId']
)->target_reservation_id === $automatic['targetReservationId'],
'Migration moved claims or swapped reservation identity.');
$assert(Capsule::table('cloud_public_ips')
->where('reservation_id', $automatic['sourceReservationId'])
->count() === 0
&& Capsule::table('cloud_public_ips')
->whereIn('id', array_column($sourceIdentityBefore, 2))
->where('state', 'available')->count() === 2
&& Capsule::table('cloud_public_ips')
->where('reservation_id', $automatic['targetReservationId'])
->where('state', 'assigned')->count() === 2,
'Source cleanup did not release exactly its own dual-stack bundle.');
// A retained source may be cleaned later; an unknown cleanup keeps the exact
// original pair fenced and cannot create a second debt or release one family.
$retained = $seed(2);
$retainedDebt = $cutover($retained);
$retainedMigration = CloudInstanceMigration::findOrFail(
$retained['migrationId']
);
CloudInstanceMigrationSettlement::markRetainedLocked($retainedMigration);
$retainedJob = $job($retained, 9002, md5('cleanup-retained'));
$retainedMigration->cleanup_job_id = (int) $retainedJob->id;
$retainedMigration->save();
CloudInstanceMigrationSettlement::bindCleanupLocked(
$retainedMigration,
$retainedJob
);
CloudInstanceMigrationSettlement::markCleanupAttentionLocked(
$retainedMigration,
$retainedJob,
'operation_effect_unknown'
);
$attention = CloudInstanceMigrationSourceDebt::findOrFail(
(int) $retainedDebt->id
);
$assert((string) $attention->state === 'attention'
&& (int) $attention->active_debt === 1
&& (int) $attention->cleanup_job_id === (int) $retainedJob->id
&& Capsule::table('cloud_public_ips')
->where('reservation_id', $retained['sourceReservationId'])
->where('state', 'reserved')->count() === 2,
'Unknown cleanup released or detached the retained source bundle.');
CloudInstanceMigrationSettlement::releaseSourceLocked(
$retainedMigration,
$retainedJob
);
$lateSuccess = CloudInstanceMigrationSourceDebt::findOrFail(
(int) $retainedDebt->id
);
$assert((string) $lateSuccess->state === 'released'
&& $lateSuccess->active_debt === null
&& Capsule::table('cloud_public_ips')
->where('reservation_id', $retained['sourceReservationId'])
->count() === 0,
'Late confirmed cleanup success did not resolve attention debt.');
// A confirmed not-applied cleanup clears only its job claim, leaving the same
// debt and bundle eligible for an explicit successor operation.
$retryable = $seed(3);
$retryableDebt = $cutover($retryable);
$retryableMigration = CloudInstanceMigration::findOrFail(
$retryable['migrationId']
);
$firstCleanup = $job($retryable, 9003, md5('cleanup-failed'));
$retryableMigration->cleanup_job_id = (int) $firstCleanup->id;
$retryableMigration->save();
CloudInstanceMigrationSettlement::bindCleanupLocked(
$retryableMigration,
$firstCleanup
);
CloudInstanceMigrationSettlement::markCleanupFailedLocked(
$retryableMigration,
$firstCleanup,
'migration_cleanup_failed'
);
$afterFailure = CloudInstanceMigrationSourceDebt::findOrFail(
(int) $retryableDebt->id
);
$assert((string) $afterFailure->state === 'retained'
&& $afterFailure->cleanup_job_id === null
&& $afterFailure->cleanup_operation_id === null
&& Capsule::table('cloud_public_ips')
->where('reservation_id', $retryable['sourceReservationId'])
->where('state', 'reserved')->count() === 2,
'Confirmed-not-applied cleanup did not retain a retryable source debt.');
$secondCleanup = $job($retryable, 9004, md5('cleanup-retry'));
$retryableMigration->cleanup_job_id = (int) $secondCleanup->id;
$retryableMigration->save();
CloudInstanceMigrationSettlement::bindCleanupLocked(
$retryableMigration,
$secondCleanup
);
// Corrupt frozen evidence must fail closed without changing either family.
Capsule::table('cloud_instance_migration_source_debts')
->where('id', (int) $retryableDebt->id)
->update(['source_identity_sha256' => str_repeat('0', 64)]);
$beforeDrift = Capsule::table('cloud_public_ips')
->whereIn('reservation_id', [
$retryable['sourceReservationId'],
$retryable['targetReservationId'],
])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all();
$rejects(
static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked(
$retryableMigration,
$secondCleanup
),
'Drifted source identity was released.'
);
$afterDrift = Capsule::table('cloud_public_ips')
->whereIn('reservation_id', [
$retryable['sourceReservationId'],
$retryable['targetReservationId'],
])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all();
$assert($beforeDrift === $afterDrift,
'Failed identity validation changed public-network ownership.');
// A second-family write failure rolls back the complete handoff, including
// the already-updated IPv4 rows.
$rollback = $seed(4);
$beforeRollback = [
'reservations' => Capsule::table('cloud_resource_reservations')
->whereIn('id', [
$rollback['sourceReservationId'],
$rollback['targetReservationId'],
])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(),
'claims' => Capsule::table('cloud_resource_reservation_ips')
->whereIn('reservation_id', [
$rollback['sourceReservationId'],
$rollback['targetReservationId'],
])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(),
'ips' => Capsule::table('cloud_public_ips')
->whereIn('reservation_id', [
$rollback['sourceReservationId'],
$rollback['targetReservationId'],
])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(),
];
Capsule::statement(
"CREATE TRIGGER fail_migration_ipv6_handoff\n"
. "BEFORE UPDATE ON cloud_public_ips\n"
. "WHEN OLD.address_family = 6 AND NEW.state <> OLD.state\n"
. "BEGIN SELECT RAISE(ABORT, 'forced ipv6 failure'); END"
);
$rejects(
static fn () => CloudPublicNetworkAllocationService::
handoffMigrationBundles(
$rollback['sourceReservationId'],
$rollback['targetReservationId'],
$rollback['instanceId'],
$rollback['sourceOperation'],
$rollback['targetOperation']
),
'Second-family fault did not abort migration handoff.'
);
Capsule::statement('DROP TRIGGER fail_migration_ipv6_handoff');
$afterRollback = [
'reservations' => Capsule::table('cloud_resource_reservations')
->whereIn('id', [
$rollback['sourceReservationId'],
$rollback['targetReservationId'],
])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(),
'claims' => Capsule::table('cloud_resource_reservation_ips')
->whereIn('reservation_id', [
$rollback['sourceReservationId'],
$rollback['targetReservationId'],
])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(),
'ips' => Capsule::table('cloud_public_ips')
->whereIn('reservation_id', [
$rollback['sourceReservationId'],
$rollback['targetReservationId'],
])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(),
];
$assert($beforeRollback === $afterRollback,
'Second-family failure left a partial migration handoff.');
// A released replay is still a ledger verification, not an unconditional
// success. Tampering either the paid target authority or the frozen source
// claim history must fail closed even though no allocator mutation remains.
Capsule::table('cloud_resource_reservations')
->where('id', $automatic['targetReservationId'])
->update(['order_id' => null]);
try {
$rejects(
static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked(
$migration,
$cleanup
),
'Released replay accepted drifted target commercial authority.'
);
} finally {
Capsule::table('cloud_resource_reservations')
->where('id', $automatic['targetReservationId'])
->update(['order_id' => $automatic['orderId']]);
}
$releasedSourceClaim = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', $automatic['sourceReservationId'])
->where('operation_id', $automatic['sourceOperation'])
->orderBy('address_family')->first();
$assert($releasedSourceClaim !== null,
'Released source claim history disappeared.');
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $releasedSourceClaim->id)
->update(['operation_id' => md5('tampered-released-claim')]);
try {
$rejects(
static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked(
$migration,
$cleanup
),
'Released replay accepted drifted frozen source claim history.'
);
} finally {
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $releasedSourceClaim->id)
->update(['operation_id' => $automatic['sourceOperation']]);
}
// Historical source generations remain immutable audit rows. Once cleanup is
// released, the concrete addresses may be allocated to a new reservation and
// claim generation; replay must validate the frozen source generation without
// mistaking the new owner for stale source authority.
$reallocatedReservationId = 9901;
$reallocatedOperation = md5('reallocated-source-addresses');
Capsule::table('cloud_resource_reservations')->insert([
'id' => $reallocatedReservationId,
'reservation_key' => md5('reallocated-reservation'),
'operation_id' => $reallocatedOperation,
'order_id' => null,
'backup_restore_id' => null,
'backup_restore_attempt_key' => null,
'migration_source_debt_id' => null,
'offer_id' => null,
'resource_pool_id' => 10,
'host_id' => 1,
'user_id' => 9,
'instance_id' => null,
'network_attachment_id' => null,
'public_ip_pool_id' => null,
'public_ip_id' => null,
'cpu_cores' => 2,
'memory_mib' => 2048,
'disk_gib' => 40,
'state' => 'held',
'expires_at' => $now + 600,
'released_at' => null,
'release_reason' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$releasedSourceClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', $automatic['sourceReservationId'])
->where('operation_id', $automatic['sourceOperation'])
->orderBy('address_family')->get()->keyBy('address_family');
$reallocatedClaims = [];
foreach ([4, 6] as $family) {
$releasedClaim = $releasedSourceClaims->get($family);
$assert(is_object($releasedClaim),
'Released source generation is incomplete before reallocation.');
$newClaimId = Capsule::table('cloud_resource_reservation_ips')
->insertGetId([
'reservation_id' => $reallocatedReservationId,
'address_family' => $family,
'claim_generation' => 1,
'active_claim' => 1,
'network_attachment_id' => 11,
'public_ip_pool_id' => (int) $releasedClaim->public_ip_pool_id,
'public_ip_id' => (int) $releasedClaim->public_ip_id,
'operation_id' => $reallocatedOperation,
'state' => 'reserved',
'released_at' => null,
'release_reason' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$updated = Capsule::table('cloud_public_ips')
->where('id', (int) $releasedClaim->public_ip_id)
->where('state', 'available')
->whereNull('reservation_id')
->whereNull('reservation_ip_claim_id')
->whereNull('operation_id')
->update([
'state' => 'reserved',
'reservation_id' => $reallocatedReservationId,
'reservation_ip_claim_id' => $newClaimId,
'instance_id' => null,
'operation_id' => $reallocatedOperation,
'updated_at' => $now,
]);
$assert($updated === 1,
'Released source address was not legally reallocated.');
$reallocatedClaims[$family] = [
'public_ip_id' => (int) $releasedClaim->public_ip_id,
'source_claim_id' => (int) $releasedClaim->id,
'new_claim_id' => $newClaimId,
];
}
$reallocatedSet = CloudPublicNetworkAllocationService::currentClaimSet(
$reallocatedReservationId,
'held'
);
$assert($reallocatedSet->reservationId() === $reallocatedReservationId,
'Reallocated addresses do not form a valid held dual-stack bundle.');
CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup);
$assert(Capsule::table('cloud_public_ips')
->where('reservation_id', $reallocatedReservationId)
->where('state', 'reserved')->count() === 2
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', $reallocatedReservationId)
->where('active_claim', 1)->count() === 2,
'Released replay rejected or changed a legal address reallocation.');
$reallocatedIpv4 = $reallocatedClaims[4];
foreach ([
'reservation_id' => $automatic['sourceReservationId'],
'reservation_ip_claim_id' => $reallocatedIpv4['source_claim_id'],
'operation_id' => $automatic['sourceOperation'],
] as $field => $staleValue) {
Capsule::table('cloud_public_ips')
->where('id', $reallocatedIpv4['public_ip_id'])
->update([$field => $staleValue]);
try {
$rejects(
static fn () =>
CloudInstanceMigrationSettlement::releaseSourceLocked(
$migration,
$cleanup
),
'Released replay accepted a stale source ' . $field . ' pointer.'
);
} finally {
$replacement = match ($field) {
'reservation_id' => $reallocatedReservationId,
'reservation_ip_claim_id' => $reallocatedIpv4['new_claim_id'],
'operation_id' => $reallocatedOperation,
};
Capsule::table('cloud_public_ips')
->where('id', $reallocatedIpv4['public_ip_id'])
->update([$field => $replacement]);
}
}
CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup);
echo "Maintenance migration dual-stack source-debt settlement passed.\n";