@ -833,6 +833,36 @@ namespace {
null,
9002
);
$migrationTargetClean = $insertJob(
'instance_migration_target_clean',
static fn (int $id, string $operationId): array => [
'method' => 'POST',
'request_target' => '/v1/vms/'
. rawurlencode($migrationTargetUuid)
. '/peer-migration/cleanup-target',
'body' => [
'operation_id' => $operationId,
'migration_id' => $migrationKey,
'prior_operation_id' =>
$migrationTransfer['operation_id'],
'cleanup_plan_id' => str_repeat('c', 64),
],
'owner_id' => $ownerId,
'can_manage' => true,
'maintenance_migration_id' => 1,
'prior_job_id' => $migrationTransfer['id'],
'cleanup_plan_id' => str_repeat('c', 64),
'cleanup_steps' => ['delete_managed_volumes'],
],
$instanceId,
null,
$migrationTransfer['id'],
null,
'agent',
$ownerId,
null,
9002
);
$migrationCleanup = $insertJob(
'instance_migration_cleanup',
static fn (int $id, string $operationId): array => [
@ -1017,6 +1047,14 @@ namespace {
'staged_powered_off' => true,
]
);
$addFixture(
'instance_migration_target_clean',
'target-residue-cleanup',
$migrationTargetClean,
'cleanup-peer-migration-target:' . $migrationKey,
$migrationTargetUuid,
$deleteProjection
);
$addFixture(
'instance_migration_cleanup',
'source-cleanup',
@ -2181,6 +2219,34 @@ namespace {
CloudAdminOperationRecoveryService::class,
'inspection'
);
$transferIdentity = CloudOperationRecoveryIdentity::expectedFromPayload(
$migrationTransfer['job'],
$migrationTransfer['payload']
);
$cleanupInspection = $inspectionMethod->invoke(null, [
'operation_id' => $migrationTransfer['operation_id'],
'action' => $transferIdentity['action'],
'resource_id' => $transferIdentity['resource_id'],
'request_fingerprint' =>
$transferIdentity['request_fingerprint'],
'operation_status' => 'failed',
'checked_at' => time(),
'outcome' => 'uncertain',
'summary_code' => 'cleanup_required',
'effect_resolution' => 'partial',
'fence_state' => 'active',
'fence_resolution_required' => true,
'abort_job' => ['available' => false],
'cleanup' => [
'available' => true,
'plan_id' => str_repeat('c', 64),
'steps' => ['delete_managed_volumes'],
],
], $migrationTransfer['operation_id'],
'instance_migration_transfer', true);
$assert(($cleanupInspection['cleanup']['plan_id'] ?? null)
=== str_repeat('c', 64),
'Validated target cleanup evidence was dropped before recovery.');
$upgradeNotAppliedRejected = false;
try {
$inspectionMethod->invoke(null, [
@ -2727,5 +2793,187 @@ namespace {
)->where('id', '< >', $migrationRollback['id'])->count() === 1,
'Abort prepare created more than one compensating rollback task.');
// A partial target-receive inspection queues one independent, plan-bound
// cleanup generation. Re-entering automatic recovery must reuse that
// durable child rather than replaying the receive or creating duplicates.
$nextJobId = max(
$nextJobId,
(int) Capsule::table('cloud_jobs')->max('id') + 1
);
$residueMigrationKey = str_repeat('9', 32);
$residueTargetUuid = '88888888-8888-4888-8888-888888888888';
$residueTransfer = $insertJob(
'instance_migration_transfer',
static fn (int $id, string $operationId): array => [
'method' => 'POST',
'request_target' => '/v1/vms/'
. rawurlencode($residueTargetUuid)
. '/peer-migration/receive',
'body' => [
'operation_id' => $operationId,
'public_ip_cidr' => '203.0.113.30/24',
],
'owner_id' => $ownerId,
'can_manage' => true,
'maintenance_migration_id' => 2,
],
$instanceId,
null,
null,
null,
'agent',
$ownerId,
null,
9012
);
Capsule::table('cloud_backup_restores')->insert([
'id' => 9012,
'restore_key' => str_repeat('f', 32),
'backup_id' => null,
'target_host_id' => $hostId,
'target_instance_id' => null,
'reservation_id' => 9004,
'job_id' => $residueTransfer['id'],
'operation_id' => $residueTransfer['operation_id'],
'new_uuid' => $residueTargetUuid,
'new_name' => 'residue-target',
'state' => 'uncertain',
'error_code' => 'operation_effect_unknown',
'ready_at' => null,
'created_at' => $now - 120,
'updated_at' => $now,
]);
Capsule::table('cloud_instance_migrations')->insert([
'id' => 2,
'migration_key' => $residueMigrationKey,
'instance_id' => $instanceId,
'restore_id' => 9012,
'source_host_id' => $hostId,
'source_uuid' => $instanceUuid,
'source_reservation_id' => 9003,
'target_host_id' => $hostId,
'target_uuid' => $residueTargetUuid,
'target_reservation_id' => 9004,
'source_prepare_job_id' => null,
'target_receive_job_id' => $residueTransfer['id'],
'firewall_job_id' => null,
'activate_job_id' => null,
'rollback_job_id' => null,
'cleanup_job_id' => null,
'cleanup_mode' => 'delete',
'state' => 'attention_required',
'recovery_action' => 'reconcile_transfer',
'retry_generation' => 0,
'source_endpoint' => '["https://192.0.2.10:9443"]',
'peer_trust_id' => str_repeat('1', 64),
'manifest_digest' => str_repeat('2', 64),
'source_was_running' => true,
'source_autostart_enabled' => true,
'source_service_status' => 'active',
'source_network_status' => 'active',
'error_code' => 'operation_effect_unknown',
'cutover_at' => null,
'completed_at' => null,
'created_at' => $now - 120,
'updated_at' => $now,
]);
Capsule::table('cloud_resource_reservations')->where('id', 9004)
->update([
'backup_restore_id' => 9012,
'operation_id' => $residueTransfer['operation_id'],
'instance_id' => $instanceId,
'state' => 'uncertain',
'updated_at' => $now,
]);
Capsule::table('cloud_public_ips')->where('id', 9007)->update([
'state' => 'uncertain',
'instance_id' => null,
'operation_id' => $residueTransfer['operation_id'],
'updated_at' => $now,
]);
$residueInspection = [
'cleanup' => [
'available' => true,
'plan_id' => str_repeat('3', 64),
'steps' => ['delete_managed_volumes'],
],
];
$assert(CloudInstanceMigrationService::ensureTargetResidueCleanup(
CloudJob::findOrFail($residueTransfer['id']),
$residueInspection
), 'A safe target-residue plan did not queue cleanup.');
$targetCleanup = CloudJob::where(
'parent_job_id', $residueTransfer['id']
)->where('kind', 'instance_migration_target_clean')->first();
$targetCleanupPayload = $targetCleanup === null ? null
: CloudSecrets::decrypt(
(string) $targetCleanup->sealed_payload,
'job-payload'
);
$assert($targetCleanup !== null
& & (int) $targetCleanup->host_id === $hostId
& & (int) $targetCleanup->instance_id === $instanceId
& & ($targetCleanupPayload['body']['prior_operation_id'] ?? null)
=== $residueTransfer['operation_id']
& & ($targetCleanupPayload['body']['cleanup_plan_id'] ?? null)
=== str_repeat('3', 64)
& & str_ends_with(
(string) ($targetCleanupPayload['request_target'] ?? ''),
'/peer-migration/cleanup-target'
), 'Target-residue cleanup did not retain its exact plan and parent.');
$assert(CloudInstanceMigrationService::ensureTargetResidueCleanup(
CloudJob::findOrFail($residueTransfer['id']),
$residueInspection
) & & CloudJob::where(
'parent_job_id', $residueTransfer['id']
)->where('kind', 'instance_migration_target_clean')->count() === 1,
'Repeated recovery created duplicate target-residue cleanups.');
// Model the next recovery pass after the independent cleanup succeeds and
// the original receive inspection proves the target is absent. The normal
// transfer finalizer must restore every target ledger to one retryable
// operation identity instead of leaving uncertain capacity or IP claims.
Capsule::table('cloud_jobs')->where('id', $targetCleanup->id)->update([
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'terminal_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_jobs')->where('id', $residueTransfer['id'])
->update([
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'not_applied',
'effect_resolution_reason' => 'system_inspection',
'updated_at' => $now,
]);
CloudInstanceMigrationService::finishJobLocked(
CloudJob::findOrFail($residueTransfer['id']),
'failed',
null,
$residueTransfer['payload']
);
$repairedMigration = CloudInstanceMigration::findOrFail(2);
$repairedRestore = Capsule::table('cloud_backup_restores')
->where('id', 9012)->first();
$repairedReservation = Capsule::table('cloud_resource_reservations')
->where('id', 9004)->first();
$repairedIp = Capsule::table('cloud_public_ips')
->where('id', 9007)->first();
$assert((string) $repairedMigration->state === 'attention_required'
& & (string) $repairedMigration->recovery_action === 'retry_transfer'
& & (string) $repairedMigration->error_code
=== 'peer_transfer_failed'
& & (string) $repairedRestore->state === 'reserved'
& & (string) $repairedRestore->error_code === 'peer_transfer_failed'
& & (string) $repairedReservation->state === 'held'
& & (string) $repairedIp->state === 'reserved'
& & $repairedIp->instance_id === null,
'Target cleanup did not repair migration, restore, capacity, and IP ledgers for a safe retry.');
echo "Cloud operation recovery matrix tests passed\n";
}