From 8c76dd2ddb514503ccf4ebe0856c20d293fe615b Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 4 Nov 2026 01:24:53 +0800 Subject: [PATCH] feat: settle dual-stack maintenance migrations --- docs/DEPLOYMENT.md | 2 +- docs/PORTABLE_VM_BACKUPS.md | 10 +- docs/VALIDATION.md | 2 +- .../Cloud/CloudInstanceMigrationService.php | 100 +- .../CloudInstanceMigrationSettlement.php | 937 ++++++---- .../Cloud/CloudOperationRecoveryIdentity.php | 17 + .../Services/Cloud/CloudPlacementService.php | 3 +- .../CloudPublicNetworkAllocationService.php | 342 +++- .../CloudPublicNetworkTransitionGate.php | 6 +- web/app/Services/Cloud/CloudWorkerService.php | 21 +- web/public/cloud/admin-instance-migration.js | 4 +- web/resources/views/venix/layout_assets.tpl | 2 +- web/sql/cloud.sql | 1 + web/tests/CloudFinancialWorkflowTest.php | 74 + .../CloudInstanceMigrationWorkflowTest.php | 7 +- ...CloudMaintenanceMigrationFinalizerTest.php | 1522 +++++++++++++---- ...aintenanceMigrationReceiveContractTest.php | 4 +- .../CloudOperationRecoveryMatrixTest.php | 385 ++++- ...CloudPortableBackupAdminUiContractTest.php | 10 +- ...loudPortableBackupRecoveryEnvelopeTest.php | 71 + ...licNetworkLegacyWriterGateContractTest.php | 37 +- ...loudRestoreMigrationLedgerContractTest.php | 2 +- 22 files changed, 2695 insertions(+), 864 deletions(-) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 055caea1..3e0559f2 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -160,7 +160,7 @@ libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存 公网网络接入在宿主机的“网络接入”中按资源池线路独立配置,可使用 `bridge:br0`、其他 Linux 网桥或 `network:default` 这样的 libvirt network。名称以字母或数字开头,后续只能包含字母、数字、点、下划线和短横线,bridge 名称为 1~15 字符,libvirt network 名称为 1~64 字符。宿主机上的公网网桥或 libvirt network 必须由运维预先准备。Agent 只发现、验证和使用现有公网网络,不创建公网网桥、不启动 libvirt network,也不修改公网地址、上游网关或挂载。`cloud_hosts.default_network` 仅保留为兼容展示值,不参与 Agent 就绪、调度、恢复、迁移或换 IP 决策;真正的运行时权威是资源池线路、当前 Agent 代次已验证的网络接入,以及绑定该接入的公网地址池。VPC 使用下述独立的 Agent 受管 bridge 和 L3 路由,不改变这一公网边界。 -全新公网账本 schema 版本为 `go-agent-wss-v33-portable-restore-attempts`。每个资源预留的当前公网权威只来自 `cloud_resource_reservation_ips.active_claim = 1` 的一组 IPv4+IPv6 精确声明;两族允许具有各自的 generation 和 operation ID,但必须绑定同一资源预留和网络接入,并各自与具体 IP、地址池、宿主机、实例状态双向一致。`cloud_resource_reservations` 中旧的单地址列只保留审计形态,不参与当前地址读取、释放或管理员投影。释放时历史声明保留并把 `active_claim` 置空,具体 IP 同时清空 claim、operation、reservation 和 instance 归属。独立恢复只允许 `cloud_backup_restore_attempts` 拥有、携带合法 `attempt_key` 且通过完整双栈信封校验的新任务;旧 `CloudPlacementService` 单地址恢复、缺少或损坏 attempt 身份的任务仍会在派发 CAS 前拒绝。尚未迁移到双栈账本的普通换 IP 与维护迁移继续保持关闭,不能通过旧单地址写入路径绕过门禁。 +全新公网账本 schema 版本为 `go-agent-wss-v33-portable-restore-attempts`。每个资源预留的当前公网权威只来自 `cloud_resource_reservation_ips.active_claim = 1` 的一组 IPv4+IPv6 精确声明;两族允许具有各自的 generation 和 operation ID,但必须绑定同一资源预留和网络接入,并各自与具体 IP、地址池、宿主机、实例状态双向一致。`cloud_resource_reservations` 中旧的单地址列只保留审计形态,不参与当前地址读取、释放或管理员投影。释放时历史声明保留并把 `active_claim` 置空,具体 IP 同时清空 claim、operation、reservation 和 instance 归属。普通换 IP、独立恢复和维护迁移分别只允许 transition ledger、restore attempt 与 migration source debt 所有的完整双栈流程;旧 `CloudPlacementService` 单地址恢复、旧换 IP writer,以及缺少或损坏冻结身份的任务仍会在派发 CAS 前拒绝,不能绕过新账本。 Agent 的 capabilities 响应仍使用 `network_selectors` 字段,但它是自动发现的当前可用 `bridge:<名称>` / `network:<名称>` 列表,排序去重且最多 64 项,不再来自配置文件。Linux 网桥必须有 sysfs bridge 目录且接口为 UP;libvirt network 必须处于 active 状态。完整 Agent 测试保存当前代次的候选列表,但不要求某个宿主机全局默认值必须存在。管理员只能从该代次已测试列表创建网络接入,并必须对每个接入执行精确验证;创建、恢复、迁移、重装和换 IP 会在锁内再次核对本次资源池线路、接入、公网池与 Agent 代次。修改兼容 `default_network` 不会撤销 Agent 就绪结果,修改或失效真实网络接入则由接入自身的验证状态阻止运行时使用。 diff --git a/docs/PORTABLE_VM_BACKUPS.md b/docs/PORTABLE_VM_BACKUPS.md index 0e3088cc..709cbaba 100644 --- a/docs/PORTABLE_VM_BACKUPS.md +++ b/docs/PORTABLE_VM_BACKUPS.md @@ -44,20 +44,20 @@ ## 维护迁移 -维护迁移是独立恢复之外的管理员流程,只允许选择同一资源池内的其他宿主机。它使用独立的 `PeerMigration V3` 冷迁移协议,不经过面板、Gateway 或对象存储传输磁盘,也不改变普通云备份的单系统盘格式。控制面先冻结当前全部已就绪快照目录,并按系统盘加全部快照容量锁定目标资源和新公网 IP;任何未就绪、目录漂移或不完整回执都会阻止切换。 +维护迁移是独立恢复之外的管理员流程,只允许选择同一资源池内的其他宿主机。它使用独立的 `PeerMigration V3` 冷迁移协议,不经过面板、Gateway 或对象存储传输磁盘,也不改变普通云备份的单系统盘格式。控制面先冻结当前全部已就绪快照目录,并按系统盘加全部快照容量锁定目标资源和同一接入下全新的公网 IPv4+IPv6;任何未就绪、目录漂移、单族地址不足或不完整回执都会阻止切换。 源 Agent 先把原运行状态和 autostart 写入本地持久恢复意图,再正常关闭实例并禁用 autostart。实例完全停机后,Agent 对稳定系统盘和全部快照卷计算大小、虚拟容量和 SHA-256,发布一个不含宿主机路径的短期只读清单。每台 Agent 首次启动时在私有状态目录自动生成并持久化独立 TLS 1.3 身份;面板通过已认证控制通道把源身份的精确 SHA-256 锁定值交给目标。目标只能用该锁定身份和控制面生成的 32 字节随机票据读取本次迁移,票据不进入 URL,清单和每个工件都不能泄露本机路径。 -目标 Agent 主动从源 Agent 的 HTTPS 地址目录拉取数据,优先选择目标本机同网段的地址,再回退公网地址;因此当前可使用公网,后续增加物理内网地址时无需改面板配置。传输使用固定 8 MiB HTTP Range 分块、`.partial` 断点续传和完整 SHA-256 校验。暂存目录由 Agent 自动派生为第一个 `storage_path_roots` 下的 `.peer-migration-staging`,不会落入 Agent Journal 所在目录。导入完成并重建全部快照元数据后,目标只定义关机且禁用 autostart 的 Domain。控制面确认新 UUID、新公网 IP、系统盘摘要、每个快照回执和资源预留后,才切换原 `cloud_instances` 记录及快照宿主机归属。迁移不会创建第二份账单实例,也不会改动原订单、套餐、服务期、到期时间、续费参数或当前流量周期。 +目标 Agent 主动从源 Agent 的 HTTPS 地址目录拉取数据,优先选择目标本机同网段的地址,再回退公网地址;因此当前可使用公网,后续增加物理内网地址时无需改面板配置。传输使用固定 8 MiB HTTP Range 分块、`.partial` 断点续传和完整 SHA-256 校验。暂存目录由 Agent 自动派生为第一个 `storage_path_roots` 下的 `.peer-migration-staging`,不会落入 Agent Journal 所在目录。导入完成并重建全部快照元数据后,目标只定义关机且禁用 autostart 的 Domain。控制面确认新 UUID、新公网 IPv4+IPv6、系统盘摘要、每个快照回执和资源预留后,才在一个数据库事务内切换原 `cloud_instances` 记录及快照宿主机归属。迁移不会创建第二份账单实例,也不会改动原订单、套餐、服务期、到期时间、续费参数或当前流量周期。 -目标使用已冻结的目标资源池线路、该线路当前 Agent 已验证的网络接入和新公网 IPv4。原累计流量作为新 Agent 计数器世代的基数继续单调累计,不能因宿主机和 UUID 改变而清零或重复计费。修改过的来宾主机名从备份时冻结的 `guest_hostname` 生成新 seed,不回退为实例显示名。目标保持源实例的服务状态、网络状态和电源状态;源网络已因流量或到期被限制时,目标持久网卡从定义开始就是 `link down`。 +目标使用已冻结的目标资源池线路、该线路当前 Agent 已验证的网络接入和新公网 IPv4+IPv6,并同时建立两族防伪身份。原累计流量作为新 Agent 计数器世代的基数继续单调累计,不能因宿主机和 UUID 改变而清零或重复计费。迁移不重新注入主机名、密码或 SSH 公钥,来宾磁盘中的当前值保持不变;面板保存的 `guest_hostname` 只随实例规格元数据迁移。目标保持源实例的服务状态、网络状态和电源状态;源网络已因流量或到期被限制时,目标持久网卡从定义开始就是 `link down`。 目标在整个恢复和防火墙阶段保持关机。若实例启用了用户防火墙,控制面必须先在关机目标上重新应用并确认同一策略;没有策略或策略已禁用时也必须完成恢复终结。只有这些步骤完成后,独立的目标激活任务才恢复原 autostart 和运行/关机状态,随后才允许进入源副本清理。因此目标不会在用户防火墙生效前短暂联网启动。 管理员可在开始时选择两种源副本策略: -- “目标验证后自动清理”会在目标恢复及防火墙收敛后删除源 Domain 和面板托管磁盘,只有 Agent 同时确认 Domain 与全部托管产物均不存在时,才释放原公网 IP 和源容量。 -- “保留源副本”会保持源 Domain 关机且禁用 autostart,并继续锁定原公网 IP 和源容量;管理员稍后可从同一页面显式发起清理。 +- “目标验证后自动清理”会在目标恢复及防火墙收敛后删除源 Domain 和面板托管磁盘,只有 Agent 同时确认 Domain 与全部托管产物均不存在时,才释放原公网 IPv4、IPv6 和源容量。 +- “保留源副本”会保持源 Domain 关机且禁用 autostart,并通过独立 source-debt 账本继续锁定原公网 IPv4、IPv6 和源容量;管理员稍后可从同一页面显式发起清理。 - 在源副本清理或异常核对完成前,面板会阻止普通实例变更和套餐升级;到期策略仍可断网,但不会只删除目标实例而遗留源副本。 任何派发后的超时、断连或回执丢失都进入原 operation 的只读核对,系统不会重放源停机、目标创建或源删除,也不会提前释放资源。传输明确失败且目标创建被证明没有发生时,管理员可以“重试传输”并从已有分块继续,或“中止传输”恢复源机原电源/autostart 状态并释放目标预留。防火墙、目标激活和源回滚的确定失败分别提供同阶段重试;源清理失败回到 `source_retained`,允许再次显式清理。只有不确定效果仍由任务安全核对栅栏阻止重放,因此业务状态不会依靠一个无出口的永久软件锁。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index f1760716..1dea8faf 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -138,7 +138,7 @@ jq empty deploy/*.json.example - `PeerMigration V3` 必须验证当前全部快照都进入同一不可变清单、任一快照缺失或回执漂移即拒绝切换、目标恢复后快照目录整体改绑。源 Agent 自动生成持久 TLS 1.3 身份并发布本机地址目录,目标必须精确锁定源证书 SHA-256 并携带一次性 Bearer 票据;公开清单、任务结果和浏览器响应均不得出现宿主机路径、私钥或票据。目标必须优先尝试同网段地址并可回退公网地址,按固定 8 MiB Range 分块续传、复用合法 `.partial` 并在导入前核对每个完整 SHA-256。 - 目标分块暂存必须由 Agent 派生到第一个 `storage_path_roots` 下的 `.peer-migration-staging`,不能落在 Journal 系统盘。故障注入分块断连后应从已有偏移继续;成功导入删除暂存,过期中断会话只能由有界本地清理回收,不能删除不匹配目录。 - 目标在恢复和用户防火墙收敛期间必须保持关机;源机的运行/关机、autostart、断网状态与修改后主机名必须在独立激活步骤后保持一致。传输确定失败须同时提供续传和安全回滚,防火墙、激活、回滚、源清理的确定失败均有显式恢复出口。若源准备已存在精确成功回执但 PHP 最终化失败,只能将原准备效果收敛为 `applied` 后创建新的补偿回滚任务;Agent 明确确认源机恢复前不得释放目标预留或 IP。未知效果只能核对原 operation,不能重放。普通云备份仍只能产生单系统盘 S3 对象,PeerMigration 不得创建 S3 迁移对象。 -- 维护迁移阶段注册表必须通过完整性测试:每个 Agent 任务种类只能对应一个账本字段、完成处理和不确定结果核对动作,每个人工恢复动作只能对应一个执行入口。页面必须直接使用后端返回的恢复选项和阶段进度;接口在加锁后使用同一判定重新授权。资源结算模块只能在源删除或目标接管得到明确证明后移动订单预留和公网 IP。没有持久化字节遥测时只能显示阶段进度,不能合成传输百分比。 +- 维护迁移阶段注册表必须通过完整性测试:每个 Agent 任务种类只能对应一个账本字段、完成处理和不确定结果核对动作,每个人工恢复动作只能对应一个执行入口。页面必须直接使用后端返回的恢复选项和阶段进度;接口在加锁后使用同一判定重新授权。目标接管得到明确证明后,目标 reservation 才能原位接管订单及目标双栈 claims,源 reservation 同时原位转成 source-debt;reservation、claim 和具体地址 ID 均不得在源与目标之间交换。源删除只有在 Agent 明确确认 Domain 与托管磁盘都不存在后才释放原 IPv4+IPv6;未知效果必须保留同一 cleanup operation、两族地址和源容量,晚到成功回执才能终结债务。没有持久化字节遥测时只能显示阶段进度,不能合成传输百分比。 - 独立恢复可选择原宿主机或其他合格宿主机,且目标身份必须精确到宿主机与资源池;无论目标是否为原宿主机,都必须单独校验并预留容量,以及同一已验证 active attachment 下全新的 IPv4+IPv6。校验下载对象字节数与 SHA-256;新实例 UUID、MAC、seed、NVRAM、网络过滤器、两族防伪身份和流量状态均不得复用源实例,且不得继承订单、账单、服务期、到期时间或流量周期。 - 独立恢复还必须故障注入 attempt/payload 不一致、预签名失败和本地信封失败:这些故障不得联系 Agent;存在确定 `not_dispatched` 证据时必须原子释放容量、IPv4、IPv6 和两族 claims,账本损坏或派发效果未知时则必须保留完整双栈 reservation、claims 与 operation 围栏。后续只能查询原 operation,或使用已经持久化的精确回执执行 PHP-only finalization,绝不重放原 mutation。预签名 URL 只能存在于单次 wire payload,不得进入数据库任务、回执、浏览器响应或恢复信封。 - 公网地址池批量替换只能选择同宿主机、同线路、当前 Agent revision 已验证 attachment 上的 active/applied 目标池。启动事务必须同时将源池转为 `draining`、为全部实例冻结不重复的目标 IP、建立父任务及全部子任务。验收全成功停用源池、全部明确未应用恢复源池、已知部分成功保持排空、任一未知效果保持 active claim 和 IP 围栏。管理员只能核对原子任务,不得批量重放。 diff --git a/web/app/Services/Cloud/CloudInstanceMigrationService.php b/web/app/Services/Cloud/CloudInstanceMigrationService.php index 55535b8e..177bbc23 100644 --- a/web/app/Services/Cloud/CloudInstanceMigrationService.php +++ b/web/app/Services/Cloud/CloudInstanceMigrationService.php @@ -27,7 +27,6 @@ final class CloudInstanceMigrationService CloudJob $receiveJob, array $inspection ): bool { - CloudPublicNetworkTransitionGate::assertMigrationReady(); $cleanup = $inspection['cleanup'] ?? null; $planId = is_array($cleanup) ? ($cleanup['plan_id'] ?? null) : null; $steps = is_array($cleanup) ? ($cleanup['steps'] ?? null) : null; @@ -174,7 +173,6 @@ final class CloudInstanceMigrationService array $input ): array { CloudAccess::manage($actor); - CloudPublicNetworkTransitionGate::assertMigrationReady(); CloudInput::only($input, [ 'target_host_id', 'target_resource_pool_id', 'cleanup_mode', 'confirm_maintenance_migration', @@ -194,7 +192,9 @@ final class CloudInstanceMigrationService ); } if (($input['confirm_maintenance_migration'] ?? false) !== true) { - throw new CloudError('请确认维护迁移会关闭源实例并更换公网 IP'); + throw new CloudError( + '请确认维护迁移会关闭源实例并更换公网 IPv4 和 IPv6' + ); } $requestKey = $actor['request_idempotency_key'] ?? null; if (!is_string($requestKey) || $requestKey === '') { @@ -722,7 +722,11 @@ final class CloudInstanceMigrationService $action = 'abort_prepare'; } if ((string) $stage['key'] === 'cleanup') { - CloudInstanceMigrationSettlement::retainSourceLocked($migration); + CloudInstanceMigrationSettlement::markCleanupAttentionLocked( + $migration, + $job, + 'finalization_incident' + ); self::setState($migration, 'source_retained', 'finalization_incident', $action); return; @@ -874,7 +878,6 @@ final class CloudInstanceMigrationService array $input ): CloudJob { CloudAccess::manage($actor); - CloudPublicNetworkTransitionGate::assertMigrationReady(); CloudInput::only($input, ['action', 'confirm_recovery']); $action = $input['action'] ?? null; if (!is_string($action) @@ -1030,7 +1033,6 @@ final class CloudInstanceMigrationService array $input ): CloudJob { CloudAccess::manage($actor); - CloudPublicNetworkTransitionGate::assertMigrationReady(); CloudInput::only($input, ['confirm_source_cleanup']); if (($input['confirm_source_cleanup'] ?? false) !== true) { throw new CloudError('请确认删除源宿主机上的旧副本'); @@ -1530,14 +1532,36 @@ final class CloudInstanceMigrationService if ($rootDiskGib === false) { throw new CloudError('目标 Agent 的系统盘容量回执无效', 502); } - $committedNetwork = CloudPublicNetworkAllocationService::commitBundle( - (int) $target->id, - (int) $instance->id, - (string) $restore->operation_id + $sourceHint = CloudResourceReservation::find( + (int) $migration->source_reservation_id ); + if ($sourceHint === null + || preg_match('/^[a-f0-9]{32}$/D', + (string) $sourceHint->operation_id) !== 1) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源双栈网络操作身份无效', 409 + ); + } + $handoff = CloudPublicNetworkAllocationService:: + handoffMigrationBundles( + (int) $sourceHint->id, + (int) $target->id, + (int) $instance->id, + (string) $sourceHint->operation_id, + (string) $restore->operation_id + ); + $sourceClaimSet = $handoff['source'] ?? null; + $committedTargetClaimSet = $handoff['target'] ?? null; + if (!$sourceClaimSet instanceof CloudActivePublicNetworkClaimSet + || !$committedTargetClaimSet + instanceof CloudActivePublicNetworkClaimSet) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 双栈网络切换回执无效', 409 + ); + } if (!hash_equals( $targetClaimSet->publicNetwork()->fingerprint(), - $committedNetwork->fingerprint() + $committedTargetClaimSet->publicNetwork()->fingerprint() )) { throw new CloudRecoveryInstanceStateConflict( 'PeerMigration 目标双栈公网提交身份发生变化', 409 @@ -1554,19 +1578,18 @@ final class CloudInstanceMigrationService ); $expectedTargetClaim = (int) $rootDiskGib + $snapshotDiskGib; if ($target === null || $source === null - || !in_array((string) $target->state, - ['held', 'uncertain'], true) - || $target->instance_id !== null + || (string) $target->state !== 'committed' + || (int) $target->instance_id !== (int) $instance->id || !hash_equals((string) $restore->operation_id, (string) $target->operation_id) || (int) $target->disk_gib !== $expectedTargetClaim - || (string) $source->state !== 'committed' + || (string) $source->state !== 'held' || (int) $source->host_id !== (int) $migration->source_host_id || (int) $source->resource_pool_id !== (int) $instance->resource_pool_id || (int) $target->resource_pool_id !== (int) $source->resource_pool_id - || (int) $source->instance_id !== (int) $instance->id + || $source->instance_id !== null || (int) $source->disk_gib !== (int) $rootDiskGib || (int) $instance->host_id !== (int) $migration->source_host_id || !hash_equals(strtolower((string) $instance->uuid), @@ -1592,11 +1615,6 @@ final class CloudInstanceMigrationService $source->disk_gib = $expectedTargetClaim; $source->updated_at = $now; $source->save(); - $target->state = 'committed'; - $target->instance_id = (int) $instance->id; - $target->expires_at = null; - $target->released_at = null; - $target->release_reason = ''; $target->updated_at = $now; $target->save(); @@ -1632,6 +1650,18 @@ final class CloudInstanceMigrationService $instance->traffic_synced_at = 0; $instance->updated_at = $now; $instance->save(); + CloudInstanceMigrationSettlement::settleCutoverLocked( + $migration, + $sourceClaimSet + ); + $restore = CloudBackupRestore::where( + 'id', (int) $restore->id + )->lockForUpdate()->first(); + if ($restore === null || $restore->reservation_id !== null) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 目标恢复预留未完成结算', 409 + ); + } $restore->state = 'ready'; $restore->error_code = ''; $restore->ready_at = $now; @@ -2413,7 +2443,7 @@ final class CloudInstanceMigrationService ]); return; } - CloudInstanceMigrationSettlement::retainSourceLocked($migration); + CloudInstanceMigrationSettlement::markRetainedLocked($migration); self::setState($migration, 'source_retained', ''); } @@ -2462,6 +2492,7 @@ final class CloudInstanceMigrationService $job->backup_restore_id = (int) $migration->restore_id; $job->save(); $migration->cleanup_job_id = (int) $job->id; + CloudInstanceMigrationSettlement::bindCleanupLocked($migration, $job); self::setState($migration, 'cleanup_running', ''); return $job; } @@ -2485,13 +2516,21 @@ final class CloudInstanceMigrationService return; } if ($status === 'uncertain') { - CloudInstanceMigrationSettlement::retainSourceLocked($migration); + CloudInstanceMigrationSettlement::markCleanupAttentionLocked( + $migration, + $job, + 'operation_effect_unknown' + ); self::setState($migration, 'source_retained', 'operation_effect_unknown', 'reconcile_cleanup'); return; } if ($status === 'failed') { - CloudInstanceMigrationSettlement::retainSourceLocked($migration); + CloudInstanceMigrationSettlement::markCleanupFailedLocked( + $migration, + $job, + 'migration_cleanup_failed' + ); self::setState($migration, 'source_retained', 'migration_cleanup_failed'); return; @@ -2503,15 +2542,10 @@ final class CloudInstanceMigrationService '源 Agent 未确认实例和托管磁盘均已删除', 502 ); } - if (CloudInstanceMigrationSettlement::isSettled($migration)) { - CloudInstanceMigrationSettlement::releaseRetainedSourceLocked( - $migration - ); - } else { - CloudInstanceMigrationSettlement::finalizeSourceDeletionLocked( - $migration - ); - } + CloudInstanceMigrationSettlement::releaseSourceLocked( + $migration, + $job + ); self::setState($migration, 'completed', ''); $migration->completed_at = time(); $migration->save(); diff --git a/web/app/Services/Cloud/CloudInstanceMigrationSettlement.php b/web/app/Services/Cloud/CloudInstanceMigrationSettlement.php index 53a9b7bb..04313734 100644 --- a/web/app/Services/Cloud/CloudInstanceMigrationSettlement.php +++ b/web/app/Services/Cloud/CloudInstanceMigrationSettlement.php @@ -2,373 +2,708 @@ namespace App\Services\Cloud; +use App\Models\CloudBackupRestore; use App\Models\CloudInstance; use App\Models\CloudInstanceMigration; +use App\Models\CloudInstanceMigrationSourceDebt; +use App\Models\CloudJob; use App\Models\CloudOrder; -use App\Models\CloudPublicIp; use App\Models\CloudResourceReservation; +use Illuminate\Database\Capsule\Manager as DB; /** - * Resource-ledger settlement for maintenance migration. + * Maintenance-migration resource settlement. * - * Stage execution decides when a result is proven. This module alone moves - * order ownership, reservations and public IP assignments after that proof. + * Reservation and claim identities never move. The target becomes the paid + * authority in place while the original source becomes a held cleanup debt. */ final class CloudInstanceMigrationSettlement { - /** - * Lock the exact maintenance-migration evidence that accounts for a - * stopped source duplicate on one host. - * - * The caller already owns the host and public-pool locks. Migration rows - * are locked before their restore reservations, matching every migration - * finalizer before the allocator later locks concrete IP rows. - * - * @param list $candidatePoolIds - * @return array - */ - public static function lockRetainedSourcePublicIpEvidence( - int $hostId, - array $candidatePoolIds - ): array { - $candidatePoolIds = array_values(array_unique(array_map( - static fn ($id): int => (int) $id, - $candidatePoolIds - ))); - sort($candidatePoolIds, SORT_NUMERIC); - if ($hostId < 1 || $candidatePoolIds === [] - || $candidatePoolIds[0] < 1) { - throw new CloudError('保留源副本公网账本范围无效', 409); - } - - $migrations = CloudInstanceMigration::where('source_host_id', $hostId) - ->whereIn('state', ['source_retained', 'cleanup_running']) - ->orderBy('id')->lockForUpdate()->get(); - if ($migrations->isEmpty()) { - return []; - } - $reservationIds = $migrations->pluck('source_reservation_id')->map( - static fn ($id): int => (int) $id - )->unique()->sort()->values()->all(); - if ($reservationIds === [] || $reservationIds[0] < 1) { - throw new CloudError('保留源副本缺少资源预留证据', 409); - } - $reservations = CloudResourceReservation::whereIn('id', $reservationIds) - ->orderBy('id')->lockForUpdate()->get()->keyBy('id'); - if ($reservations->count() !== count($reservationIds)) { - throw new CloudError('保留源副本资源预留证据不完整', 409); - } - - $candidatePools = array_fill_keys($candidatePoolIds, true); - $evidence = []; - foreach ($migrations as $migration) { - $reservationId = (int) $migration->source_reservation_id; - $reservation = $reservations->get($reservationId); - $operationId = (string) ($reservation?->operation_id ?? ''); - if (!$reservation instanceof CloudResourceReservation - || (int) $migration->instance_id < 1 - || (int) $migration->restore_id < 1 - || (int) $migration->source_host_id !== $hostId - || (int) $migration->target_host_id < 1 - || (int) $migration->target_reservation_id < 1 - || (int) $reservation->host_id !== $hostId - || (int) ($reservation->backup_restore_id ?? 0) - !== (int) $migration->restore_id - || $reservation->order_id !== null - || (string) $reservation->state !== 'committed' - || $reservation->instance_id !== null - || (int) ($reservation->network_attachment_id ?? 0) < 1 - || (int) ($reservation->public_ip_pool_id ?? 0) < 1 - || (int) ($reservation->public_ip_id ?? 0) < 1 - || preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) { - throw new CloudError( - '保留源副本迁移与资源预留证据不一致', - 409 - ); - } - $poolId = (int) $reservation->public_ip_pool_id; - if (!isset($candidatePools[$poolId])) { - continue; - } - $publicIpId = (int) $reservation->public_ip_id; - if (isset($evidence[$publicIpId])) { - throw new CloudError('保留源副本公网地址证据重复', 409); - } - $evidence[$publicIpId] = [ - 'migration_id' => (int) $migration->id, - 'migration_state' => (string) $migration->state, - 'reservation_id' => $reservationId, - 'host_id' => (int) $reservation->host_id, - 'network_attachment_id' => - (int) $reservation->network_attachment_id, - 'public_ip_pool_id' => $poolId, - 'public_ip_id' => $publicIpId, - 'operation_id' => $operationId, - ]; - } - ksort($evidence, SORT_NUMERIC); - return $evidence; - } - - /** - * Make the order-linked reservation authoritative on the target while a - * separate restore reservation accounts for the stopped source duplicate. - */ - public static function retainSourceLocked( - CloudInstanceMigration $migration - ): void { - if (self::isSettled($migration)) { - return; - } - $instance = CloudInstance::where( - 'id', (int) $migration->instance_id - )->lockForUpdate()->first(); + public static function settleCutoverLocked( + CloudInstanceMigration $migration, + CloudActivePublicNetworkClaimSet $frozenSourceClaimSet + ): CloudInstanceMigrationSourceDebt { + $instance = CloudInstance::where('id', (int) $migration->instance_id) + ->lockForUpdate()->first(); $source = CloudResourceReservation::where( 'id', (int) $migration->source_reservation_id )->lockForUpdate()->first(); $target = CloudResourceReservation::where( 'id', (int) $migration->target_reservation_id )->lockForUpdate()->first(); + $restore = CloudBackupRestore::where('id', (int) $migration->restore_id) + ->lockForUpdate()->first(); $order = $instance === null || $instance->order_id === null ? null : CloudOrder::where('id', (int) $instance->order_id) ->lockForUpdate()->first(); - $sourceIp = $source === null ? null : CloudPublicIp::where( - 'id', (int) $source->public_ip_id - )->lockForUpdate()->first(); - $targetIp = $target === null ? null : CloudPublicIp::where( - 'id', (int) $target->public_ip_id - )->lockForUpdate()->first(); if ($instance === null || $source === null || $target === null - || $order === null || $sourceIp === null || $targetIp === null + || $restore === null || $order === null) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 切换缺少资源账本', 409 + ); + } + $existing = CloudInstanceMigrationSourceDebt::where( + 'migration_id', (int) $migration->id + )->lockForUpdate()->first(); + if ($existing !== null) { + self::assertActiveAuthorityLocked( + $migration, $existing, $instance, $source, $target, + $order, $restore + ); + return $existing; + } + + $sourceClaimSet = CloudPublicNetworkAllocationService::currentClaimSet( + (int) $source->id, + 'held' + ); + $targetClaimSet = CloudPublicNetworkAllocationService::currentClaimSet( + (int) $target->id, + 'committed', + (int) $instance->id + ); + if ($frozenSourceClaimSet->reservationId() !== (int) $source->id + || $frozenSourceClaimSet->hostId() + !== (int) $migration->source_host_id + || $sourceClaimSet->reservationId() !== (int) $source->id + || $sourceClaimSet->hostId() !== (int) $source->host_id + || $sourceClaimSet->networkAttachmentId() + !== $frozenSourceClaimSet->networkAttachmentId() + || !hash_equals( + $sourceClaimSet->digest(), + $frozenSourceClaimSet->digest() + ) + || !hash_equals( + $sourceClaimSet->publicNetwork()->fingerprint(), + $frozenSourceClaimSet->publicNetwork()->fingerprint() + ) + || $targetClaimSet->reservationId() !== (int) $target->id + || $targetClaimSet->hostId() !== (int) $migration->target_host_id || (int) $instance->host_id !== (int) $migration->target_host_id - || !hash_equals(strtolower((string) $instance->uuid), - strtolower((string) $migration->target_uuid)) + || !hash_equals( + strtolower((string) $instance->uuid), + strtolower((string) $migration->target_uuid) + ) || (int) $instance->reservation_id !== (int) $target->id - || (int) $instance->resource_pool_id - !== (int) $source->resource_pool_id - || (int) $target->resource_pool_id - !== (int) $source->resource_pool_id || (int) $source->host_id !== (int) $migration->source_host_id || (int) $target->host_id !== (int) $migration->target_host_id + || (string) $source->state !== 'held' + || $source->instance_id !== null || (int) $source->order_id !== (int) $order->id - || (int) $order->resource_pool_id - !== (int) $source->resource_pool_id + || (int) $source->offer_id !== (int) $order->offer_id || $source->backup_restore_id !== null - || (int) $target->backup_restore_id !== (int) $migration->restore_id - || $target->order_id !== null - || (string) $source->state !== 'committed' + || $source->backup_restore_attempt_key !== null + || $source->migration_source_debt_id !== null || (string) $target->state !== 'committed' - || (int) $sourceIp->host_id !== (int) $source->host_id - || (int) $targetIp->host_id !== (int) $target->host_id - || (int) $sourceIp->public_ip_pool_id - !== (int) $source->public_ip_pool_id - || (int) $targetIp->public_ip_pool_id - !== (int) $target->public_ip_pool_id - || (int) $sourceIp->reservation_id !== (int) $source->id - || (int) $targetIp->reservation_id !== (int) $target->id) { + || (int) $target->instance_id !== (int) $instance->id + || (int) $target->backup_restore_id !== (int) $restore->id + || $target->order_id !== null || $target->offer_id !== null + || $target->backup_restore_attempt_key !== null + || $target->migration_source_debt_id !== null + || (int) $restore->reservation_id !== (int) $target->id + || (int) $order->reservation_id !== (int) $source->id + || (int) $order->host_id !== (int) $source->host_id + || (int) $order->resource_pool_id + !== (int) $source->resource_pool_id + || (int) $instance->resource_pool_id + !== (int) $target->resource_pool_id + || (int) $source->resource_pool_id + !== (int) $target->resource_pool_id + || (int) $source->user_id !== (int) $instance->user_id + || (int) $target->user_id !== (int) $instance->user_id) { throw new CloudRecoveryInstanceStateConflict( - 'PeerMigration 保留源副本时资源账本不一致', 409 + 'PeerMigration 切换资源账本不一致', 409 ); } + + $frozen = CloudFrozenPublicNetworkIdentity::fromClaimSet( + (string) $migration->source_uuid, + $frozenSourceClaimSet + ); $now = time(); - $sourceFields = []; - $targetFields = []; - foreach (['resource_pool_id', 'host_id', 'network_attachment_id', - 'public_ip_pool_id', 'public_ip_id', 'cpu_cores', 'memory_mib', - 'disk_gib'] as $field) { - $sourceFields[$field] = $source->{$field}; - $targetFields[$field] = $target->{$field}; - } - foreach ($targetFields as $field => $value) { - $source->{$field} = $value; - } - $source->instance_id = (int) $instance->id; + $debt = new CloudInstanceMigrationSourceDebt(); + $debt->debt_key = bin2hex(random_bytes(16)); + $debt->migration_id = (int) $migration->id; + $debt->source_reservation_id = (int) $source->id; + $debt->source_host_id = (int) $source->host_id; + $debt->source_uuid = strtolower((string) $migration->source_uuid); + $debt->source_network_attachment_id = + $frozenSourceClaimSet->networkAttachmentId(); + $debt->source_identity_json = $frozen->identityJson(); + $debt->source_identity_sha256 = $frozen->identitySha256(); + $debt->source_claim_set_sha256 = $frozen->claimSetSha256(); + $debt->source_public_network_json = $frozen->publicNetworkJson(); + $debt->source_public_network_sha256 = + $frozen->publicNetworkSha256(); + $debt->cleanup_operation_id = null; + $debt->cleanup_job_id = null; + $debt->state = 'prepared'; + $debt->active_debt = 1; + $debt->last_error_code = ''; + $debt->resolved_at = null; + $debt->created_at = $now; + $debt->updated_at = $now; + $debt->save(); + + // Free the unique commercial source before assigning it to target. + $source->order_id = null; + $source->offer_id = null; + $source->migration_source_debt_id = (int) $debt->id; $source->updated_at = $now; $source->save(); - foreach ($sourceFields as $field => $value) { - $target->{$field} = $value; - } - $target->instance_id = null; + $target->backup_restore_id = null; + $target->order_id = (int) $order->id; + $target->offer_id = (int) $order->offer_id; $target->updated_at = $now; $target->save(); - - $targetIp->reservation_id = (int) $source->id; - $targetIp->instance_id = (int) $instance->id; - $targetIp->operation_id = (string) $source->operation_id; - $targetIp->state = 'assigned'; - $targetIp->updated_at = $now; - $targetIp->save(); - $sourceIp->reservation_id = (int) $target->id; - $sourceIp->instance_id = null; - $sourceIp->operation_id = (string) $target->operation_id; - $sourceIp->state = 'assigned'; - $sourceIp->updated_at = $now; - $sourceIp->save(); - - $instance->reservation_id = (int) $source->id; - $instance->resource_pool_id = (int) $source->resource_pool_id; - $instance->updated_at = $now; - $instance->save(); - $order->host_id = (int) $source->host_id; - $order->resource_pool_id = (int) $source->resource_pool_id; - $order->reservation_id = (int) $source->id; + $restore->reservation_id = null; + $restore->updated_at = $now; + $restore->save(); + $order->host_id = (int) $target->host_id; + $order->resource_pool_id = (int) $target->resource_pool_id; + $order->reservation_id = (int) $target->id; $order->updated_at = $now; $order->save(); - $oldSourceId = (int) $migration->source_reservation_id; - $migration->source_reservation_id = (int) $target->id; - $migration->target_reservation_id = $oldSourceId; - $migration->updated_at = $now; - $migration->save(); + return $debt; } - public static function isSettled( + public static function markRetainedLocked( CloudInstanceMigration $migration - ): bool { - $instance = CloudInstance::find((int) $migration->instance_id); - $target = CloudResourceReservation::find( - (int) $migration->target_reservation_id - ); - return $instance !== null && $target !== null - && (int) $instance->reservation_id === (int) $target->id - && $target->order_id !== null - && $target->backup_restore_id === null - && (int) $target->host_id === (int) $migration->target_host_id - && (int) $target->resource_pool_id - === (int) $instance->resource_pool_id; + ): void { + [$debt] = self::activeAuthorityLocked($migration); + if ((string) $debt->state === 'retained') { + return; + } + if ((string) $debt->state !== 'prepared' + || $debt->cleanup_operation_id !== null + || $debt->cleanup_job_id !== null) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源副本不能转为保留状态', 409 + ); + } + $debt->state = 'retained'; + $debt->last_error_code = ''; + $debt->updated_at = time(); + $debt->save(); } - /** Source deletion is proven; transfer its order-linked reservation. */ - public static function finalizeSourceDeletionLocked( - CloudInstanceMigration $migration + public static function bindCleanupLocked( + CloudInstanceMigration $migration, + CloudJob $job + ): void { + [$debt] = self::activeAuthorityLocked($migration); + self::assertCleanupJob($migration, $job); + if ((string) $debt->state === 'cleanup_running' + && (int) $debt->cleanup_job_id === (int) $job->id + && hash_equals( + (string) $debt->cleanup_operation_id, + (string) $job->operation_id + )) { + return; + } + if (!in_array((string) $debt->state, + ['prepared', 'retained'], true) + || $debt->cleanup_operation_id !== null + || $debt->cleanup_job_id !== null) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源清理债务已被其他任务领取', 409 + ); + } + $debt->cleanup_operation_id = (string) $job->operation_id; + $debt->cleanup_job_id = (int) $job->id; + $debt->state = 'cleanup_running'; + $debt->last_error_code = ''; + $debt->updated_at = time(); + $debt->save(); + } + + public static function markCleanupAttentionLocked( + CloudInstanceMigration $migration, + CloudJob $job, + string $errorCode ): void { - $instance = CloudInstance::where( - 'id', (int) $migration->instance_id + [$debt] = self::activeAuthorityLocked($migration); + self::assertBoundCleanup($migration, $debt, $job); + if (!in_array((string) $debt->state, + ['cleanup_running', 'attention'], true)) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源清理核对状态不一致', 409 + ); + } + $debt->state = 'attention'; + $debt->last_error_code = $errorCode; + $debt->updated_at = time(); + $debt->save(); + } + + public static function markCleanupFailedLocked( + CloudInstanceMigration $migration, + CloudJob $job, + string $errorCode + ): void { + [$debt] = self::activeAuthorityLocked($migration); + self::assertBoundCleanup($migration, $debt, $job); + if (!in_array((string) $debt->state, + ['cleanup_running', 'attention'], true)) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源清理失败状态不一致', 409 + ); + } + $debt->cleanup_operation_id = null; + $debt->cleanup_job_id = null; + $debt->state = 'retained'; + $debt->last_error_code = $errorCode; + $debt->updated_at = time(); + $debt->save(); + } + + public static function releaseSourceLocked( + CloudInstanceMigration $migration, + CloudJob $job + ): void { + $migration->getConnection()->transaction(static function () use ( + $migration, + $job + ): void { + // Validate the migration-owned projection before entering the + // allocator. Do not lock its reservation first: allocator order is + // host -> network context -> reservation -> claims -> addresses. + $debt = CloudInstanceMigrationSourceDebt::where( + 'migration_id', (int) $migration->id + )->first(); + if ($debt === null) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源副本缺少清理债务', 409 + ); + } + if ((string) $debt->state === 'released') { + self::assertReleasedAuthority($migration, $debt); + self::assertBoundCleanup($migration, $debt, $job); + return; + } + $instance = CloudInstance::find((int) $migration->instance_id); + $source = CloudResourceReservation::find( + (int) $migration->source_reservation_id + ); + $target = CloudResourceReservation::find( + (int) $migration->target_reservation_id + ); + $order = $instance === null || $instance->order_id === null + ? null : CloudOrder::find((int) $instance->order_id); + $restore = CloudBackupRestore::find((int) $migration->restore_id); + if ($instance === null || $source === null || $target === null + || $order === null || $restore === null) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源副本债务账本不完整', 409 + ); + } + self::assertActiveAuthorityLocked( + $migration, $debt, $instance, $source, $target, + $order, $restore + ); + self::assertBoundCleanup($migration, $debt, $job); + if (!in_array((string) $debt->state, + ['cleanup_running', 'attention'], true)) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源清理完成状态不一致', 409 + ); + } + CloudPublicNetworkAllocationService::releaseBundle( + (int) $source->id, + (string) $source->operation_id, + 'maintenance_migration_source_removed', + true + ); + $now = time(); + $debt = CloudInstanceMigrationSourceDebt::where( + 'id', (int) $debt->id + )->lockForUpdate()->first(); + $source = CloudResourceReservation::where( + 'id', (int) $source->id + )->lockForUpdate()->first(); + if ($debt === null || $source === null + || (string) $source->state !== 'held' + || $source->instance_id !== null + || (int) $source->migration_source_debt_id + !== (int) $debt->id + || !in_array((string) $debt->state, + ['cleanup_running', 'attention'], true) + || (int) $debt->cleanup_job_id !== (int) $job->id + || !hash_equals( + (string) $debt->cleanup_operation_id, + (string) $job->operation_id + )) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源副本资源预留已变化', 409 + ); + } + $source->state = 'released'; + $source->released_at = $now; + $source->release_reason = + 'maintenance_migration_source_removed'; + $source->expires_at = null; + $source->updated_at = $now; + $source->save(); + $debt->state = 'released'; + $debt->active_debt = null; + $debt->last_error_code = ''; + $debt->resolved_at = $now; + $debt->updated_at = $now; + $debt->save(); + }, 3); + } + + /** + * @return array{0:CloudInstanceMigrationSourceDebt, + * 1:CloudResourceReservation,2:CloudResourceReservation, + * 3:CloudInstance,4:CloudOrder,5:CloudBackupRestore} + */ + private static function activeAuthorityLocked( + CloudInstanceMigration $migration, + ?CloudInstanceMigrationSourceDebt $knownDebt = null + ): array { + $debt = $knownDebt ?? CloudInstanceMigrationSourceDebt::where( + 'migration_id', (int) $migration->id )->lockForUpdate()->first(); + $instance = CloudInstance::where('id', (int) $migration->instance_id) + ->lockForUpdate()->first(); $source = CloudResourceReservation::where( 'id', (int) $migration->source_reservation_id )->lockForUpdate()->first(); $target = CloudResourceReservation::where( 'id', (int) $migration->target_reservation_id )->lockForUpdate()->first(); - $sourceIp = $source === null ? null : CloudPublicIp::where( - 'id', (int) $source->public_ip_id - )->lockForUpdate()->first(); - $targetIp = $target === null ? null : CloudPublicIp::where( - 'id', (int) $target->public_ip_id - )->lockForUpdate()->first(); $order = $instance === null || $instance->order_id === null ? null : CloudOrder::where('id', (int) $instance->order_id) ->lockForUpdate()->first(); - if ($instance === null || $source === null || $target === null - || $sourceIp === null || $targetIp === null || $order === null - || (int) $instance->host_id !== (int) $migration->target_host_id - || !hash_equals(strtolower((string) $instance->uuid), - strtolower((string) $migration->target_uuid)) + $restore = CloudBackupRestore::where('id', (int) $migration->restore_id) + ->lockForUpdate()->first(); + if ($debt === null || $instance === null || $source === null + || $target === null || $order === null || $restore === null) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源副本债务账本不完整', 409 + ); + } + self::assertActiveAuthorityLocked( + $migration, $debt, $instance, $source, $target, $order, $restore + ); + return [$debt, $source, $target, $instance, $order, $restore]; + } + + private static function assertActiveAuthorityLocked( + CloudInstanceMigration $migration, + CloudInstanceMigrationSourceDebt $debt, + CloudInstance $instance, + CloudResourceReservation $source, + CloudResourceReservation $target, + CloudOrder $order, + CloudBackupRestore $restore + ): void { + if ((int) $debt->migration_id !== (int) $migration->id + || (int) $debt->source_reservation_id !== (int) $source->id + || (int) $debt->source_host_id !== (int) $migration->source_host_id + || !hash_equals( + strtolower((string) $debt->source_uuid), + strtolower((string) $migration->source_uuid) + ) + || (int) ($debt->active_debt ?? 0) !== 1 + || (int) $source->migration_source_debt_id !== (int) $debt->id + || $source->order_id !== null || $source->offer_id !== null + || $source->backup_restore_id !== null + || $source->backup_restore_attempt_key !== null + || (string) $source->state !== 'held' + || $source->instance_id !== null + || (int) $source->host_id !== (int) $migration->source_host_id + || (int) $target->order_id !== (int) $order->id + || (int) $target->offer_id !== (int) $order->offer_id + || $target->backup_restore_id !== null + || $target->backup_restore_attempt_key !== null + || $target->migration_source_debt_id !== null + || (string) $target->state !== 'committed' + || (int) $target->instance_id !== (int) $instance->id + || (int) $target->host_id !== (int) $migration->target_host_id + || (int) $instance->host_id !== (int) $target->host_id + || !hash_equals( + strtolower((string) $instance->uuid), + strtolower((string) $migration->target_uuid) + ) || (int) $instance->reservation_id !== (int) $target->id || (int) $instance->resource_pool_id - !== (int) $source->resource_pool_id - || (int) $target->resource_pool_id - !== (int) $source->resource_pool_id + !== (int) $target->resource_pool_id + || (int) $order->reservation_id !== (int) $target->id + || (int) $order->host_id !== (int) $target->host_id + || (int) $order->resource_pool_id + !== (int) $target->resource_pool_id + || $restore->reservation_id !== null) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源副本债务权威链不一致', 409 + ); + } + $sourceClaimSet = CloudPublicNetworkAllocationService::currentClaimSet( + (int) $source->id, + 'held' + ); + try { + CloudFrozenPublicNetworkIdentity::fromStored( + (int) $source->id, + (int) $source->host_id, + (string) $migration->source_uuid, + (int) $debt->source_network_attachment_id, + (string) $debt->source_identity_json, + (string) $debt->source_identity_sha256, + (string) $debt->source_public_network_json, + (string) $debt->source_public_network_sha256, + (string) $debt->source_claim_set_sha256, + $sourceClaimSet + ); + } catch (\InvalidArgumentException) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源副本冻结双栈身份不一致', 409 + ); + } + CloudPublicNetworkAllocationService::currentClaimSet( + (int) $target->id, + 'committed', + (int) $instance->id + ); + } + + private static function assertCleanupJob( + CloudInstanceMigration $migration, + CloudJob $job + ): void { + if ((string) $job->kind !== 'instance_migration_cleanup' + || (int) $job->id < 1 + || (int) $job->instance_id !== (int) $migration->instance_id + || (int) $job->host_id !== (int) $migration->source_host_id + || (int) $job->backup_restore_id !== (int) $migration->restore_id + || preg_match('/^[a-f0-9]{32}$/D', + (string) $job->operation_id) !== 1) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源清理任务身份不一致', 409 + ); + } + } + + private static function assertBoundCleanup( + CloudInstanceMigration $migration, + CloudInstanceMigrationSourceDebt $debt, + CloudJob $job + ): void { + self::assertCleanupJob($migration, $job); + if ((int) $debt->cleanup_job_id !== (int) $job->id + || !hash_equals( + (string) $debt->cleanup_operation_id, + (string) $job->operation_id + )) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 源清理债务与任务不一致', 409 + ); + } + } + + private static function assertReleasedAuthority( + CloudInstanceMigration $migration, + CloudInstanceMigrationSourceDebt $debt + ): void { + $instance = CloudInstance::find((int) $migration->instance_id); + $source = CloudResourceReservation::find( + (int) $migration->source_reservation_id + ); + $target = CloudResourceReservation::find( + (int) $migration->target_reservation_id + ); + $order = $instance === null || $instance->order_id === null + ? null : CloudOrder::find((int) $instance->order_id); + $restore = CloudBackupRestore::find((int) $migration->restore_id); + if ((int) $debt->migration_id !== (int) $migration->id + || (int) $debt->source_reservation_id + !== (int) $migration->source_reservation_id + || (int) $debt->source_host_id + !== (int) $migration->source_host_id + || !hash_equals( + strtolower((string) $debt->source_uuid), + strtolower((string) $migration->source_uuid) + ) + || (string) $debt->state !== 'released' + || $debt->active_debt !== null + || $debt->resolved_at === null + || $debt->cleanup_operation_id === null + || $debt->cleanup_job_id === null + || $source === null || $target === null || $instance === null + || $order === null || $restore === null + || (int) $source->migration_source_debt_id !== (int) $debt->id + || $source->order_id !== null || $source->offer_id !== null + || $source->backup_restore_id !== null + || $source->backup_restore_attempt_key !== null + || (string) $source->state !== 'released' + || $source->released_at === null || $source->expires_at !== null + || $source->instance_id !== null || (int) $source->host_id !== (int) $migration->source_host_id + || (string) $source->release_reason + !== 'maintenance_migration_source_removed' + || (int) $target->order_id !== (int) $order->id + || (int) $target->offer_id !== (int) $order->offer_id + || $target->backup_restore_id !== null + || $target->backup_restore_attempt_key !== null + || $target->migration_source_debt_id !== null + || (string) $target->state !== 'committed' + || (int) $target->instance_id !== (int) $instance->id || (int) $target->host_id !== (int) $migration->target_host_id - || (int) $source->order_id !== (int) $order->id + || (int) $instance->host_id !== (int) $target->host_id + || !hash_equals( + strtolower((string) $instance->uuid), + strtolower((string) $migration->target_uuid) + ) + || (int) $instance->reservation_id !== (int) $target->id + || (int) $instance->resource_pool_id + !== (int) $target->resource_pool_id + || (int) $order->reservation_id !== (int) $target->id + || (int) $order->host_id !== (int) $target->host_id || (int) $order->resource_pool_id - !== (int) $source->resource_pool_id - || (string) $source->state !== 'committed' - || (int) $target->backup_restore_id !== (int) $migration->restore_id - || (string) $target->state !== 'committed' - || (int) $sourceIp->host_id !== (int) $source->host_id - || (int) $targetIp->host_id !== (int) $target->host_id - || (int) $sourceIp->public_ip_pool_id - !== (int) $source->public_ip_pool_id - || (int) $targetIp->public_ip_pool_id - !== (int) $target->public_ip_pool_id - || (int) $sourceIp->reservation_id !== (int) $source->id - || (int) $targetIp->reservation_id !== (int) $target->id) { + !== (int) $target->resource_pool_id + || $restore->reservation_id !== null) { throw new CloudRecoveryInstanceStateConflict( - 'PeerMigration 清理后的资源预留账本不一致', 409 + 'PeerMigration 源副本释放账本不一致', 409 ); } - $now = time(); - $sourceIp->state = 'available'; - $sourceIp->reservation_id = null; - $sourceIp->instance_id = null; - $sourceIp->operation_id = null; - $sourceIp->updated_at = $now; - $sourceIp->save(); - foreach (['resource_pool_id', 'host_id', 'network_attachment_id', - 'public_ip_pool_id', 'public_ip_id', 'cpu_cores', 'memory_mib', - 'disk_gib'] as $field) { - $source->{$field} = $target->{$field}; - } - $source->instance_id = (int) $instance->id; - $source->updated_at = $now; - $source->save(); - $targetIp->reservation_id = (int) $source->id; - $targetIp->instance_id = (int) $instance->id; - $targetIp->operation_id = (string) $source->operation_id; - $targetIp->state = 'assigned'; - $targetIp->updated_at = $now; - $targetIp->save(); - $target->instance_id = null; - $target->network_attachment_id = null; - $target->public_ip_pool_id = null; - $target->public_ip_id = null; - $target->state = 'released'; - $target->released_at = $now; - $target->release_reason = 'maintenance_migration_transferred'; - $target->updated_at = $now; - $target->save(); - $instance->reservation_id = (int) $source->id; - $instance->resource_pool_id = (int) $source->resource_pool_id; - $instance->updated_at = $now; - $instance->save(); - $order->host_id = (int) $source->host_id; - $order->resource_pool_id = (int) $source->resource_pool_id; - $order->reservation_id = (int) $source->id; - $order->updated_at = $now; - $order->save(); - $migration->source_reservation_id = (int) $target->id; - $migration->target_reservation_id = (int) $source->id; - $migration->updated_at = $now; - $migration->save(); + self::assertReleasedSourceNetworkAuthority( + $migration, + $debt, + $source + ); + CloudPublicNetworkAllocationService::currentClaimSet( + (int) $target->id, + 'committed', + (int) $instance->id + ); } - public static function releaseRetainedSourceLocked( - CloudInstanceMigration $migration + private static function assertReleasedSourceNetworkAuthority( + CloudInstanceMigration $migration, + CloudInstanceMigrationSourceDebt $debt, + CloudResourceReservation $source ): void { - $debt = CloudResourceReservation::where( - 'id', (int) $migration->source_reservation_id - )->lockForUpdate()->first(); - $ip = $debt === null ? null : CloudPublicIp::where( - 'id', (int) $debt->public_ip_id - )->lockForUpdate()->first(); - if ($debt === null || $ip === null - || (int) $debt->host_id !== (int) $migration->source_host_id - || (int) $debt->backup_restore_id !== (int) $migration->restore_id - || (string) $debt->state !== 'committed' - || (int) $ip->reservation_id !== (int) $debt->id - || !hash_equals((string) $debt->operation_id, - (string) $ip->operation_id)) { + try { + $networkValue = json_decode( + (string) $debt->source_public_network_json, + true, + 64, + JSON_THROW_ON_ERROR + ); + $network = CloudPublicNetworkSnapshot::fromArray($networkValue); + } catch (\Throwable) { throw new CloudRecoveryInstanceStateConflict( - 'PeerMigration 源副本债务账本不一致', 409 + 'PeerMigration 已释放源双栈快照损坏', 409 ); } - $now = time(); - $ip->state = 'available'; - $ip->reservation_id = null; - $ip->instance_id = null; - $ip->operation_id = null; - $ip->updated_at = $now; - $ip->save(); - $debt->state = 'released'; - $debt->released_at = $now; - $debt->release_reason = 'maintenance_migration_source_removed'; - $debt->expires_at = null; - $debt->instance_id = null; - $debt->updated_at = $now; - $debt->save(); + $claimRows = DB::table('cloud_resource_reservation_ips') + ->where('reservation_id', (int) $source->id) + ->where('operation_id', (string) $source->operation_id) + ->orderBy('address_family')->orderBy('id')->get()->all(); + if (DB::table('cloud_resource_reservation_ips') + ->where('reservation_id', (int) $source->id) + ->where('active_claim', 1)->exists()) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 已释放源仍有活动双栈声明', 409 + ); + } + $claims = []; + foreach ($claimRows as $claim) { + $family = (int) ($claim->address_family ?? 0); + if (!in_array($family, [4, 6], true) + || isset($claims[$family]) + || !hash_equals( + (string) $source->operation_id, + (string) ($claim->operation_id ?? '') + ) + || !hash_equals( + (string) $source->release_reason, + (string) ($claim->release_reason ?? '') + )) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 已释放源双栈声明不一致', 409 + ); + } + $claims[$family] = $claim; + } + try { + $claimIdentity = CloudPublicNetworkClaimSetIdentity:: + fromReleasedLedger((int) $source->id, $claims, $network); + } catch (\Throwable) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 已释放源双栈声明损坏', 409 + ); + } + $canonicalIdentity = [ + 'contract' => 1, + 'reservation_id' => (int) $source->id, + 'host_id' => (int) $source->host_id, + 'uuid' => strtolower((string) $migration->source_uuid), + 'network_attachment_id' => + (int) $debt->source_network_attachment_id, + 'claim_set_sha256' => $claimIdentity->digest(), + 'public_network_sha256' => $network->fingerprint(), + ]; + $canonicalJson = json_encode( + $canonicalIdentity, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + ); + if ((int) $debt->source_network_attachment_id < 1 + || (int) $claims[4]->network_attachment_id + !== (int) $debt->source_network_attachment_id + || (int) $claims[6]->network_attachment_id + !== (int) $debt->source_network_attachment_id + || !hash_equals( + $claimIdentity->digest(), + (string) $debt->source_claim_set_sha256 + ) + || !hash_equals( + $network->fingerprint(), + (string) $debt->source_public_network_sha256 + ) + || !hash_equals( + $canonicalJson, + (string) $debt->source_identity_json + ) + || !hash_equals( + hash('sha256', $canonicalJson), + (string) $debt->source_identity_sha256 + )) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 已释放源冻结双栈身份不一致', 409 + ); + } + + $addresses = DB::table('cloud_public_ips')->whereIn('id', [ + (int) $claims[4]->public_ip_id, + (int) $claims[6]->public_ip_id, + ])->orderBy('id')->get()->keyBy('id'); + foreach ([4, 6] as $family) { + $claim = $claims[$family]; + $address = $addresses->get((int) $claim->public_ip_id); + if (!is_object($address) + || (int) $address->host_id !== (int) $source->host_id + || (int) $address->address_family !== $family + || (int) $address->public_ip_pool_id + !== (int) $claim->public_ip_pool_id + || !hash_equals( + (string) $address->address . '/' + . (int) $address->prefix_length, + $network->address($family) + ) + || (int) ($address->reservation_id ?? 0) + === (int) $source->id + || (int) ($address->reservation_ip_claim_id ?? 0) + === (int) $claim->id + || hash_equals( + (string) $source->operation_id, + (string) ($address->operation_id ?? '') + )) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 已释放源公网地址仍保留旧归属', 409 + ); + } + } } } diff --git a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php index f341adeb..f64696a6 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php @@ -6,6 +6,7 @@ use App\Models\CloudDeploymentSnapshot; use App\Models\CloudInstance; use App\Models\CloudInstanceBackup; use App\Models\CloudInstanceMigration; +use App\Models\CloudInstanceMigrationSourceDebt; use App\Models\CloudInstanceFirewallVersion; use App\Models\CloudInstanceSnapshot; use App\Models\CloudPeerMigrationArtifact; @@ -655,6 +656,22 @@ final class CloudOperationRecoveryIdentity 409 ); } + $sourceDebt = CloudInstanceMigrationSourceDebt::where( + 'migration_id', (int) $migration->id + )->first(); + if ($sourceDebt === null + || !in_array((string) $sourceDebt->state, + ['cleanup_running', 'attention', 'released'], true) + || (int) $sourceDebt->cleanup_job_id !== (int) $job->id + || !hash_equals( + (string) $sourceDebt->cleanup_operation_id, + (string) $job->operation_id + )) { + throw new CloudRecoveryInstanceStateConflict( + '维护迁移源清理任务与资源债务不一致', + 409 + ); + } $uuid = strtolower((string) $migration->source_uuid); $base = '/v1/vms/' . rawurlencode($uuid); self::expectRequest($method, $target, 'POST', diff --git a/web/app/Services/Cloud/CloudPlacementService.php b/web/app/Services/Cloud/CloudPlacementService.php index 304036be..4f8be51c 100644 --- a/web/app/Services/Cloud/CloudPlacementService.php +++ b/web/app/Services/Cloud/CloudPlacementService.php @@ -1817,7 +1817,8 @@ final class CloudPlacementService CloudBackupRestore $restore ): void { if ($restore->backup_id === null) { - CloudPublicNetworkTransitionGate::assertMigrationReady(); + CloudPublicNetworkTransitionGate:: + assertLegacyMigrationWriterRetired(); } CloudPublicNetworkTransitionGate::assertPortableRestoreReady(); } diff --git a/web/app/Services/Cloud/CloudPublicNetworkAllocationService.php b/web/app/Services/Cloud/CloudPublicNetworkAllocationService.php index 034d1c8c..12e2377e 100644 --- a/web/app/Services/Cloud/CloudPublicNetworkAllocationService.php +++ b/web/app/Services/Cloud/CloudPublicNetworkAllocationService.php @@ -377,6 +377,280 @@ final class CloudPublicNetworkAllocationService }); } + /** + * Atomically hand one migration instance from its source bundle to target. + * + * Both hosts and every network row are locked in a single deterministic + * order. Claim and address identities never move between reservations. + * + * @return array{source:CloudActivePublicNetworkClaimSet, + * target:CloudActivePublicNetworkClaimSet} + */ + public static function handoffMigrationBundles( + int $sourceReservationId, + int $targetReservationId, + int $instanceId, + string $sourceOperationId, + string $targetOperationId + ): array { + self::assertIdentity($sourceReservationId, $sourceOperationId); + self::assertIdentity($targetReservationId, $targetOperationId); + if ($instanceId < 1) { + throw new CloudError('实例网络归属无效', 409); + } + if ($sourceReservationId === $targetReservationId) { + throw new CloudError('迁移源和目标资源预留必须不同', 409); + } + return self::connection()->transaction(static function () use ( + $sourceReservationId, + $targetReservationId, + $instanceId, + $sourceOperationId, + $targetOperationId + ): array { + $reservationIds = [$sourceReservationId, $targetReservationId]; + sort($reservationIds, SORT_NUMERIC); + $reservationHints = DB::table('cloud_resource_reservations') + ->whereIn('id', $reservationIds)->orderBy('id')->get()->all(); + if (count($reservationHints) !== 2) { + throw new CloudError('双栈公网地址账本不完整', 409); + } + $reservationHintsById = []; + $hostIds = []; + foreach ($reservationHints as $hint) { + $reservationHintsById[(int) $hint->id] = $hint; + $hostIds[(int) $hint->host_id] = true; + } + if (!isset($reservationHintsById[$sourceReservationId], + $reservationHintsById[$targetReservationId]) + || count($hostIds) !== 2) { + throw new CloudError('迁移双栈网络宿主机边界无效', 409); + } + $hostIds = array_keys($hostIds); + sort($hostIds, SORT_NUMERIC); + $hosts = DB::table('cloud_hosts')->whereIn('id', $hostIds) + ->orderBy('id')->lockForUpdate()->get()->all(); + if (count($hosts) !== 2) { + throw new CloudError('公网网络宿主机不存在', 409); + } + + $claimHints = DB::table('cloud_resource_reservation_ips') + ->whereIn('reservation_id', $reservationIds) + ->where('active_claim', 1)->orderBy('reservation_id') + ->orderBy('address_family')->orderBy('id')->get()->all(); + if (count($claimHints) !== 4) { + throw new CloudError('双栈公网地址账本不完整', 409); + } + $attachmentIds = []; + $poolIds = []; + foreach ($claimHints as $claim) { + $attachmentIds[(int) $claim->network_attachment_id] = true; + $poolIds[(int) $claim->public_ip_pool_id] = true; + } + $attachmentIds = array_keys($attachmentIds); + $poolIds = array_keys($poolIds); + sort($attachmentIds, SORT_NUMERIC); + sort($poolIds, SORT_NUMERIC); + DB::table('cloud_host_network_attachments') + ->whereIn('id', $attachmentIds)->orderBy('id') + ->lockForUpdate()->get(); + DB::table('cloud_public_ip_pools')->whereIn('id', $poolIds) + ->orderBy('id')->lockForUpdate()->get(); + + $lockedReservations = DB::table('cloud_resource_reservations') + ->whereIn('id', $reservationIds)->orderBy('id') + ->lockForUpdate()->get()->all(); + $reservations = []; + foreach ($lockedReservations as $reservation) { + $reservations[(int) $reservation->id] = $reservation; + } + $source = $reservations[$sourceReservationId] ?? null; + $target = $reservations[$targetReservationId] ?? null; + self::assertReservation( + $source, + (int) $reservationHintsById[$sourceReservationId]->host_id, + $sourceOperationId, + ['committed'] + ); + self::assertReservation( + $target, + (int) $reservationHintsById[$targetReservationId]->host_id, + $targetOperationId, + ['held', 'uncertain'] + ); + if ((int) ($source->instance_id ?? 0) !== $instanceId + || $target->instance_id !== null) { + throw new CloudError('迁移双栈网络归属不一致', 409); + } + + $lockedClaims = DB::table('cloud_resource_reservation_ips') + ->whereIn('reservation_id', $reservationIds) + ->where('active_claim', 1)->orderBy('reservation_id') + ->orderBy('address_family')->orderBy('id')->lockForUpdate() + ->get()->all(); + $claimIdentity = static function (object $claim): array { + return [ + (int) $claim->id, + (int) $claim->reservation_id, + (string) $claim->address_family, + (int) $claim->network_attachment_id, + (int) $claim->public_ip_pool_id, + (int) $claim->public_ip_id, + (string) $claim->operation_id, + (int) $claim->claim_generation, + (string) $claim->state, + ]; + }; + if (array_map($claimIdentity, $claimHints) + !== array_map($claimIdentity, $lockedClaims)) { + // The network context was selected from the unlocked hints. + // A changed claim could otherwise introduce a late attachment + // or pool lock and break the canonical global lock order. + throw new CloudError('迁移双栈网络账本已变化,请重新核对', 409); + } + $claimsByReservation = [ + $sourceReservationId => [], + $targetReservationId => [], + ]; + foreach ($lockedClaims as $claim) { + $claimsByReservation[(int) $claim->reservation_id][] = $claim; + } + $sourceClaims = $claimsByReservation[$sourceReservationId]; + $targetClaims = $claimsByReservation[$targetReservationId]; + $targetClaimState = self::claimStateForReservation($target); + self::assertClaimSet( + $sourceClaims, + $sourceOperationId, + ['committed'] + ); + self::assertClaimSet( + $targetClaims, + $targetOperationId, + [$targetClaimState] + ); + $sourceSelection = self::lockClaimNetworkContext( + $sourceClaims, + (int) $source->host_id + ); + $targetSelection = self::lockClaimNetworkContext( + $targetClaims, + (int) $target->host_id + ); + self::assertClaimsUseSelection($sourceClaims, $sourceSelection); + self::assertClaimsUseSelection($targetClaims, $targetSelection); + $addresses = self::lockClaimAddresses($lockedClaims); + self::assertClaimAddresses( + $sourceClaims, + $addresses, + $sourceSelection, + $sourceOperationId, + ['committed' => ['assigned']], + $instanceId + ); + self::assertClaimAddresses( + $targetClaims, + $addresses, + $targetSelection, + $targetOperationId, + [ + 'reserved' => ['reserved'], + 'uncertain' => ['uncertain'], + ] + ); + $sourceClaimSet = self::claimSet( + $sourceClaims, + 'committed', + $instanceId, + (int) $source->host_id + ); + + $now = time(); + foreach ($sourceClaims as $claim) { + $ipUpdated = DB::table('cloud_public_ips') + ->where('id', (int) $claim->public_ip_id) + ->where('reservation_id', $sourceReservationId) + ->where('reservation_ip_claim_id', (int) $claim->id) + ->where('operation_id', $sourceOperationId) + ->where('instance_id', $instanceId) + ->where('state', 'assigned')->update([ + 'state' => 'reserved', + 'instance_id' => null, + 'updated_at' => $now, + ]); + $claimUpdated = DB::table('cloud_resource_reservation_ips') + ->where('id', (int) $claim->id) + ->where('active_claim', 1) + ->where('operation_id', $sourceOperationId) + ->where('state', 'committed')->update([ + 'state' => 'reserved', + 'updated_at' => $now, + ]); + if ($ipUpdated !== 1 || $claimUpdated !== 1) { + throw new CloudError('迁移源双栈地址未精确转为清理债务', 409); + } + } + foreach ($targetClaims as $claim) { + $ipUpdated = DB::table('cloud_public_ips') + ->where('id', (int) $claim->public_ip_id) + ->where('reservation_id', $targetReservationId) + ->where('reservation_ip_claim_id', (int) $claim->id) + ->where('operation_id', $targetOperationId) + ->whereNull('instance_id') + ->where('state', $targetClaimState)->update([ + 'state' => 'assigned', + 'instance_id' => $instanceId, + 'updated_at' => $now, + ]); + $claimUpdated = DB::table('cloud_resource_reservation_ips') + ->where('id', (int) $claim->id) + ->where('active_claim', 1) + ->where('operation_id', $targetOperationId) + ->where('state', $targetClaimState)->update([ + 'state' => 'committed', + 'updated_at' => $now, + ]); + if ($ipUpdated !== 1 || $claimUpdated !== 1) { + throw new CloudError('迁移目标双栈地址未精确提交', 409); + } + } + $sourceUpdated = DB::table('cloud_resource_reservations') + ->where('id', $sourceReservationId) + ->where('operation_id', $sourceOperationId) + ->where('instance_id', $instanceId) + ->where('state', 'committed')->update([ + 'state' => 'held', + 'instance_id' => null, + 'expires_at' => null, + 'updated_at' => $now, + ]); + $targetUpdated = DB::table('cloud_resource_reservations') + ->where('id', $targetReservationId) + ->where('operation_id', $targetOperationId) + ->whereNull('instance_id') + ->where('state', (string) $target->state)->update([ + 'state' => 'committed', + 'instance_id' => $instanceId, + 'expires_at' => null, + 'released_at' => null, + 'release_reason' => '', + 'updated_at' => $now, + ]); + if ($sourceUpdated !== 1 || $targetUpdated !== 1) { + throw new CloudError('迁移源资源预留未精确转为清理债务', 409); + } + $targetClaimSet = self::claimSet( + self::lockedClaims($targetReservationId), + 'committed', + $instanceId, + (int) $target->host_id + ); + return [ + 'source' => $sourceClaimSet, + 'target' => $targetClaimSet, + ]; + }); + } + public static function fenceBundle(int $reservationId, string $operationId): void { @@ -901,16 +1175,11 @@ final class CloudPublicNetworkAllocationService } private static function assertPoolLedger(object $pool, - CloudPublicIpPoolContract $contract, array $rows, - array $retainedSourceEvidence): void + CloudPublicIpPoolContract $contract, array $rows): void { foreach ($rows as $row) { self::assertConcreteAddress($row, $pool, $contract); - self::assertAddressOwnership( - $row, - $pool, - $retainedSourceEvidence - ); + self::assertAddressOwnership($row); } } @@ -974,8 +1243,7 @@ final class CloudPublicNetworkAllocationService } } - private static function assertAddressOwnership(object $row, - ?object $pool = null, array $retainedSourceEvidence = []): void + private static function assertAddressOwnership(object $row): void { $reservationId = $row->reservation_id === null ? null : (int) $row->reservation_id; @@ -995,14 +1263,9 @@ final class CloudPublicNetworkAllocationService && $claimId !== null && $claimId > 0 && $instanceId === null && $validOperation, 'assigned' => $reservationId !== null && $reservationId > 0 - && $claimId !== null && $claimId > 0 && $validOperation && ( - ($instanceId !== null && $instanceId > 0) - || self::matchesRetainedSourceEvidence( - $row, - $pool, - $retainedSourceEvidence - ) - ), + && $claimId !== null && $claimId > 0 + && $instanceId !== null && $instanceId > 0 + && $validOperation, 'changing', 'uncertain' => $reservationId !== null && $reservationId > 0 && $claimId !== null && $claimId > 0 && ($instanceId === null || $instanceId > 0) @@ -1014,37 +1277,6 @@ final class CloudPublicNetworkAllocationService } } - private static function matchesRetainedSourceEvidence(object $row, - ?object $pool, array $evidenceByPublicIpId): bool - { - if ($pool === null || $row->instance_id !== null) { - return false; - } - $evidence = $evidenceByPublicIpId[(int) ($row->id ?? 0)] ?? null; - return is_array($evidence) - && in_array( - $evidence['migration_state'] ?? null, - ['source_retained', 'cleanup_running'], - true - ) - && (int) ($evidence['reservation_id'] ?? 0) - === (int) ($row->reservation_id ?? 0) - && (int) ($evidence['host_id'] ?? 0) === (int) $row->host_id - && (int) ($evidence['host_id'] ?? 0) === (int) $pool->host_id - && (int) ($evidence['network_attachment_id'] ?? 0) - === (int) $pool->network_attachment_id - && (int) ($evidence['public_ip_pool_id'] ?? 0) - === (int) $row->public_ip_pool_id - && (int) ($evidence['public_ip_pool_id'] ?? 0) - === (int) $pool->id - && (int) ($evidence['public_ip_id'] ?? 0) === (int) $row->id - && is_string($evidence['operation_id'] ?? null) - && hash_equals( - (string) $evidence['operation_id'], - (string) ($row->operation_id ?? '') - ); - } - private static function assertClaimAddresses(array $claims, array $addresses, array $selection, ?string $operationId, array $allowedPairs, ?int $instanceId = null): void @@ -1229,17 +1461,10 @@ final class CloudPublicNetworkAllocationService if (!$complete) { throw new CloudError('宿主机双栈公网地址池目录不完整', 409); } - $poolIds = array_map( - static fn (object $pool): int => (int) $pool->id, - $poolRows - ); - $retainedSourceEvidence = CloudInstanceMigrationSettlement:: - lockRetainedSourcePublicIpEvidence($hostId, $poolIds); return [ 'attachments' => $eligible, 'pools' => $pools, 'contracts' => $contracts, - 'retained_source_evidence' => $retainedSourceEvidence, ]; } @@ -1270,12 +1495,7 @@ final class CloudPublicNetworkAllocationService if (!$contract instanceof CloudPublicIpPoolContract) { throw new CloudError('公网地址池合同缺失', 409); } - self::assertPoolLedger( - $pool, - $contract, - $rowsByPool[$poolId], - $catalog['retained_source_evidence'] ?? [] - ); + self::assertPoolLedger($pool, $contract, $rowsByPool[$poolId]); } if ($constraint !== null) { diff --git a/web/app/Services/Cloud/CloudPublicNetworkTransitionGate.php b/web/app/Services/Cloud/CloudPublicNetworkTransitionGate.php index 2983cc91..34cc8ebb 100644 --- a/web/app/Services/Cloud/CloudPublicNetworkTransitionGate.php +++ b/web/app/Services/Cloud/CloudPublicNetworkTransitionGate.php @@ -24,10 +24,10 @@ final class CloudPublicNetworkTransitionGate ); } - public static function assertMigrationReady(): void + public static function assertLegacyMigrationWriterRetired(): void { throw new CloudError( - '维护迁移正在升级为双栈公网网络流程,暂不可提交', + '旧维护迁移公网写入口已停用', 409 ); } @@ -68,7 +68,7 @@ final class CloudPublicNetworkTransitionGate return; } if ((int) ($reservation->migration_source_debt_id ?? 0) > 0) { - self::assertMigrationReady(); + self::assertLegacyMigrationWriterRetired(); } throw new CloudError('实例资源预留来源无法确认', 409); } diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 1a5c5275..33d5b929 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -437,7 +437,17 @@ final class CloudWorkerService (string) $job->kind, 'instance_migration_' )) { - CloudPublicNetworkTransitionGate::assertMigrationReady(); + // Canonical PeerMigration rows carry an exact recoverable + // identity. Reject stale scalar-IP or torn ledger jobs before + // the dispatch CAS so they can never contact an Agent. This + // must validate the live request directly: expected() also + // accepts a post-effect recovery escrow, which is never a + // valid shape for an accepted, not-yet-dispatched job. + CloudOperationRecoveryIdentity::expectedFromPayload( + $job, + $payload, + false + ); } elseif (in_array( (string) $job->kind, ['delete', 'delete_force', 'expiry_delete'], @@ -1728,13 +1738,8 @@ final class CloudWorkerService 'memory_mib' => $body['memory_mib'] ?? null, 'disk_gib' => $body['disk_gib'] ?? null, 'bandwidth_mbps' => $body['bandwidth_mbps'] ?? null, - 'network' => $body['network'] ?? null, - 'public_ip_cidr' => $body['public_ip_cidr'] ?? null, - 'public_pool_key' => $body['public_pool_key'] ?? null, - 'public_pool_cidr' => - $body['public_pool_cidr'] ?? null, - 'gateway' => $body['gateway'] ?? null, - 'dns_servers' => $body['dns_servers'] ?? null, + 'public_network' => $body['public_network'] ?? null, + 'vpc_port' => $body['vpc_port'] ?? null, ], ]; } diff --git a/web/public/cloud/admin-instance-migration.js b/web/public/cloud/admin-instance-migration.js index 9fbf67df..4edcbb82 100644 --- a/web/public/cloud/admin-instance-migration.js +++ b/web/public/cloud/admin-instance-migration.js @@ -89,7 +89,7 @@ export function createAdminInstanceMigration({ ? 'alert alert-danger fs-13' : 'alert alert-warning fs-13', migration.recovery_action === 'reconcile_cleanup' ? '源副本删除效果正在按原操作编号核对,资源保持占用且不会盲目重放删除。' - : '目标实例已经接管原账单、到期时间和流量周期;源宿主机副本及原公网 IP 仍被保留并占用资源。')); + : '目标实例已经接管原账单、到期时间和流量周期;源宿主机副本及原公网 IPv4、IPv6 仍被保留并占用资源。')); } const recoveryOptions = Array.isArray(migration.recovery_options) ? migration.recovery_options.filter(option => option && typeof option === 'object' @@ -211,7 +211,7 @@ export function createAdminInstanceMigration({ } if (!confirm.checked) { feedback.hidden = false; feedback.className = 'text-danger fs-13 mb-0 mt-2'; - feedback.textContent = '请确认冷迁移、原服务账本和新公网 IP 规则。'; return; + feedback.textContent = '请确认冷迁移、原服务账本和新公网 IPv4、IPv6 规则。'; return; } for (const control of form.querySelectorAll('input, select, button')) control.disabled = true; feedback.hidden = false; feedback.className = 'text-muted fs-13 mb-0 mt-2'; diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 0a3b1337..62d5f52a 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026110303' scope='global'} +{assign var=venix_asset_version value='2026110401' scope='global'} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 63c823a0..9aa9d908 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -1533,6 +1533,7 @@ CREATE TABLE cloud_instance_migration_source_debts ( AND resolved_at IS NULL) OR (state = 'attention' AND active_debt IS NOT NULL AND active_debt = 1 + AND cleanup_operation_id IS NOT NULL AND cleanup_job_id IS NOT NULL AND resolved_at IS NULL) OR (state = 'released' AND active_debt IS NULL AND cleanup_operation_id IS NOT NULL AND cleanup_job_id IS NOT NULL diff --git a/web/tests/CloudFinancialWorkflowTest.php b/web/tests/CloudFinancialWorkflowTest.php index 6339d866..0454b595 100644 --- a/web/tests/CloudFinancialWorkflowTest.php +++ b/web/tests/CloudFinancialWorkflowTest.php @@ -3082,6 +3082,80 @@ namespace { && (int) $legacyWorkerJob->fresh()->dispatch_attempts === 0, 'Legacy network task was replayed after local rejection.'); + // A stale migration row that still carries the retired scalar public-IP + // request shape must fail before the accepted-to-dispatching CAS. The + // immutable migration/instance/reservation ledgers are the only authority + // for a PeerMigration receive request. + $legacyMigrationJob = new CloudJob(); + $legacyMigrationJob->operation_id = bin2hex(random_bytes(16)); + $legacyMigrationJob->request_idempotency_key = + 'legacy-migration-worker-gate-0001'; + $legacyMigrationJob->request_fingerprint = hash( + 'sha256', + 'legacy-migration-worker-gate-0001' + ); + $legacyMigrationJob->instance_id = (int) $legacyNetworkInstance->id; + $legacyMigrationJob->user_id = (int) $legacyNetworkInstance->user_id; + $legacyMigrationJob->host_id = (int) $legacyNetworkInstance->host_id; + $legacyMigrationJob->kind = 'instance_migration_transfer'; + $legacyMigrationJob->execution_mode = 'agent'; + $legacyMigrationJob->status = 'queued'; + $legacyMigrationStoredIdentity = [ + 'version' => 1, + 'action' => 'peer-migration-receive', + 'resource_id' => 'migration:987654', + 'request_fingerprint' => str_repeat('a', 64), + ]; + $legacyMigrationJob->sealed_payload = + \App\Services\Cloud\CloudSecrets::encrypt([ + \App\Services\Cloud\CloudOperationRecoveryIdentity::payloadKey() + => $legacyMigrationStoredIdentity, + '_operation_recovery' => [ + 'version' => 1, + 'kind' => 'instance_migration_transfer', + 'prepared_marker' => 'prepared:test:legacy-migration', + 'operation' => [ + 'operation_id' => + (string) $legacyMigrationJob->operation_id, + 'action' => $legacyMigrationStoredIdentity['action'], + 'resource_id' => + $legacyMigrationStoredIdentity['resource_id'], + 'request_fingerprint' => + $legacyMigrationStoredIdentity[ + 'request_fingerprint' + ], + ], + ], + 'method' => 'POST', + 'request_target' => '/v1/vms/' + . rawurlencode((string) $legacyNetworkInstance->uuid) + . '/peer-migration/receive', + 'body' => [ + 'operation_id' => (string) $legacyMigrationJob->operation_id, + 'public_ip_cidr' => '9.9.9.10/24', + 'gateway' => '9.9.9.1', + ], + 'maintenance_migration_id' => 987654, + 'owner_id' => (int) $legacyNetworkInstance->user_id, + 'can_manage' => false, + ], 'job-payload'); + $legacyMigrationJob->available_at = $legacyMigrationJob->created_at = + $legacyMigrationJob->updated_at = time(); + $legacyMigrationJob->save(); + $beforeLegacyMigration = count($history); + $assert($worker->tick(), + 'Legacy migration task was not consumed by local admission.'); + $legacyMigrationJob->refresh(); + $assert(count($history) === $beforeLegacyMigration + && (int) $legacyMigrationJob->dispatch_attempts === 0 + && $legacyMigrationJob->phase === 'terminal' + && $legacyMigrationJob->outcome === 'failed', + 'Legacy migration task reached the Agent or remained claimable.'); + $worker->tick(); + $assert(count($history) === $beforeLegacyMigration + && (int) $legacyMigrationJob->fresh()->dispatch_attempts === 0, + 'Legacy migration task was replayed after local rejection.'); + // The ordinary job enqueue path is permanently closed to legacy scalar // network changes. Canonical transitions have their own durable workflow // and are tested in the dedicated begin/finalization behavior suites. diff --git a/web/tests/CloudInstanceMigrationWorkflowTest.php b/web/tests/CloudInstanceMigrationWorkflowTest.php index 1ed4004b..2027da71 100644 --- a/web/tests/CloudInstanceMigrationWorkflowTest.php +++ b/web/tests/CloudInstanceMigrationWorkflowTest.php @@ -289,10 +289,11 @@ $assert(substr_count($serviceSource, '$field = match ((string) $job->kind)') && str_contains($serviceSource, 'CloudInstanceMigrationWorkflow::recoveryActions(') - && !str_contains($serviceSource, - "'maintenance_migration_transferred'") && str_contains($settlementSource, - "'maintenance_migration_transferred'") + "'maintenance_migration_source_removed'") + && str_contains($settlementSource, + 'migration_source_debt_id') + && !str_contains($settlementSource, 'CloudPublicIp') && str_contains($serviceSource, 'CloudVpcPortService::migrationPortIdLocked(') && str_contains($serviceSource, diff --git a/web/tests/CloudMaintenanceMigrationFinalizerTest.php b/web/tests/CloudMaintenanceMigrationFinalizerTest.php index c19c30e5..493e83b3 100644 --- a/web/tests/CloudMaintenanceMigrationFinalizerTest.php +++ b/web/tests/CloudMaintenanceMigrationFinalizerTest.php @@ -4,9 +4,11 @@ require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudInstance; use App\Models\CloudInstanceMigration; +use App\Models\CloudInstanceMigrationSourceDebt; use App\Models\CloudJob; -use App\Services\Cloud\CloudInstanceMigrationSettlement; use App\Services\Cloud\CloudInstanceMigrationService; +use App\Services\Cloud\CloudInstanceMigrationSettlement; +use App\Services\Cloud\CloudPublicNetworkAllocationService; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; @@ -15,15 +17,130 @@ $assert = static function (bool $condition, string $message): void { throw new RuntimeException($message); } }; +$rejects = static function (callable $callback, string $message): void { + try { + $callback(); + } catch (Throwable) { + return; + } + throw new RuntimeException($message); +}; +$captures = static function (callable $callback, string $message): Throwable { + try { + $callback(); + } catch (Throwable $error) { + return $error; + } + throw new RuntimeException($message); +}; $capsule = new Capsule(); $capsule->addConnection([ - 'driver' => 'sqlite', 'database' => ':memory:', + 'driver' => 'sqlite', + 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); +$schema->create('cloud_hosts', static function (Blueprint $table): void { + $table->increments('id'); +}); +$schema->create('cloud_resource_pools', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('line_id'); +}); +$schema->create('cloud_host_network_attachments', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('host_id'); + $table->integer('line_id'); + $table->string('network_selector'); +}); +$schema->create('cloud_public_ip_pools', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('pool_key'); + $table->integer('host_id'); + $table->integer('network_attachment_id'); + $table->integer('address_family'); + $table->string('cidr'); + $table->string('gateway'); + $table->boolean('gateway_on_link'); + $table->integer('guest_prefix_length'); + $table->text('dns_servers_json'); + $table->text('reserved_ips_json'); + $table->string('allocation_start')->nullable(); + $table->string('allocation_end')->nullable(); + $table->string('allocation_cursor')->nullable(); + $table->integer('max_allocations'); +}); +$schema->create('cloud_public_ips', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('host_id'); + $table->integer('public_ip_pool_id'); + $table->integer('address_family'); + $table->string('address'); + $table->binary('address_binary'); + $table->integer('prefix_length'); + $table->string('state'); + $table->integer('reservation_id')->nullable(); + $table->integer('reservation_ip_claim_id')->nullable(); + $table->integer('instance_id')->nullable(); + $table->string('operation_id')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_resource_reservation_ips', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('reservation_id'); + $table->integer('address_family'); + $table->integer('claim_generation')->default(1); + $table->integer('active_claim')->nullable()->default(1); + $table->integer('network_attachment_id'); + $table->integer('public_ip_pool_id'); + $table->integer('public_ip_id'); + $table->string('operation_id'); + $table->string('state'); + $table->integer('released_at')->nullable(); + $table->string('release_reason')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_resource_reservations', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('reservation_key'); + $table->string('operation_id'); + $table->integer('order_id')->nullable(); + $table->integer('backup_restore_id')->nullable(); + $table->string('backup_restore_attempt_key')->nullable(); + $table->integer('migration_source_debt_id')->nullable(); + $table->integer('offer_id')->nullable(); + $table->integer('resource_pool_id'); + $table->integer('host_id'); + $table->integer('user_id'); + $table->integer('instance_id')->nullable(); + $table->integer('network_attachment_id')->nullable(); + $table->integer('public_ip_pool_id')->nullable(); + $table->integer('public_ip_id')->nullable(); + $table->integer('cpu_cores'); + $table->integer('memory_mib'); + $table->integer('disk_gib'); + $table->string('state'); + $table->integer('expires_at')->nullable(); + $table->integer('released_at')->nullable(); + $table->string('release_reason')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); +}); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); @@ -42,6 +159,7 @@ $schema->create('cloud_instances', static function (Blueprint $table): void { $table->integer('traffic_quota_bytes'); $table->integer('lifetime_download_bytes'); $table->integer('lifetime_upload_bytes'); + $table->integer('traffic_synced_at')->default(0); $table->string('name'); $table->string('status'); $table->string('service_status'); @@ -64,296 +182,575 @@ $schema->create('cloud_orders', static function (Blueprint $table): void { $table->integer('created_at'); $table->integer('updated_at'); }); -$schema->create('cloud_resource_reservations', - static function (Blueprint $table): void { - $table->increments('id'); - $table->string('reservation_key'); - $table->string('operation_id'); - $table->integer('order_id')->nullable(); - $table->integer('backup_restore_id')->nullable(); - $table->integer('offer_id')->nullable(); - $table->integer('resource_pool_id'); - $table->integer('host_id'); - $table->integer('user_id'); - $table->integer('instance_id')->nullable(); - $table->integer('network_attachment_id')->nullable(); - $table->integer('public_ip_pool_id')->nullable(); - $table->integer('public_ip_id')->nullable(); - $table->integer('cpu_cores'); - $table->integer('memory_mib'); - $table->integer('disk_gib'); - $table->string('state'); - $table->integer('expires_at')->nullable(); - $table->integer('released_at')->nullable(); - $table->string('release_reason'); - $table->integer('created_at'); - $table->integer('updated_at'); - }); -$schema->create('cloud_public_ips', static function (Blueprint $table): void { +$schema->create('cloud_backup_restores', static function ( + Blueprint $table +): void { $table->increments('id'); - $table->integer('host_id'); - $table->integer('public_ip_pool_id'); - $table->string('address'); - $table->integer('prefix_length'); - $table->string('state'); + $table->integer('target_host_id'); $table->integer('reservation_id')->nullable(); - $table->integer('instance_id')->nullable(); - $table->string('operation_id')->nullable(); + $table->integer('job_id')->nullable(); + $table->string('operation_id'); + $table->string('state'); + $table->string('error_code')->default(''); + $table->integer('ready_at')->nullable(); + $table->integer('updated_at'); +}); +$schema->create('cloud_instance_migrations', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('migration_key'); + $table->integer('instance_id'); + $table->integer('restore_id'); + $table->integer('source_host_id'); + $table->string('source_uuid'); + $table->integer('source_reservation_id'); + $table->integer('target_host_id'); + $table->string('target_uuid'); + $table->integer('target_reservation_id'); + $table->integer('vpc_port_id')->nullable(); + $table->integer('source_prepare_job_id')->nullable(); + $table->integer('target_receive_job_id')->nullable(); + $table->integer('firewall_job_id')->nullable(); + $table->integer('activate_job_id')->nullable(); + $table->integer('rollback_job_id')->nullable(); + $table->integer('cleanup_job_id')->nullable(); + $table->string('cleanup_mode'); + $table->string('state'); + $table->string('recovery_action')->default(''); + $table->integer('retry_generation')->default(0); + $table->string('manifest_digest')->default(''); + $table->boolean('source_was_running')->nullable(); + $table->boolean('source_autostart_enabled')->nullable(); + $table->string('source_service_status'); + $table->string('source_network_status'); + $table->string('error_code')->default(''); + $table->integer('cutover_at')->nullable(); + $table->integer('completed_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); -$schema->create('cloud_hosts', static function (Blueprint $table): void { +$schema->create('cloud_peer_migration_artifacts', static function ( + Blueprint $table +): void { $table->increments('id'); + $table->integer('migration_id'); + $table->string('artifact_id'); + $table->string('role'); + $table->integer('source_snapshot_id')->nullable(); + $table->string('snapshot_key')->nullable(); + $table->string('name')->default(''); + $table->string('consistency'); + $table->integer('object_size_bytes'); + $table->integer('virtual_size_bytes'); + $table->string('sha256'); + $table->string('state'); + $table->integer('snapshot_created_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_instance_runtime', static function ( + Blueprint $table +): void { + $table->integer('instance_id')->primary(); + $table->integer('host_id'); + $table->string('uuid'); + $table->string('power_state'); + $table->string('sync_state'); + $table->integer('last_success_at'); + $table->integer('last_attempt_at'); + $table->integer('revision'); + $table->string('last_error_code'); + $table->integer('created_at'); + $table->integer('updated_at'); }); -$schema->create('cloud_host_network_attachments', - static function (Blueprint $table): void { - $table->increments('id'); - $table->integer('host_id'); - $table->integer('line_id'); - $table->string('network_selector'); - }); -$schema->create('cloud_resource_pools', - static function (Blueprint $table): void { - $table->increments('id'); - $table->integer('line_id'); - }); -$schema->create('cloud_host_resource_pool_memberships', - static function (Blueprint $table): void { - $table->increments('id'); - $table->integer('host_id'); - $table->integer('resource_pool_id'); - $table->string('scheduler_state'); - }); -$schema->create('cloud_public_ip_pools', - static function (Blueprint $table): void { - $table->increments('id'); - $table->integer('host_id'); - $table->integer('network_attachment_id'); - $table->string('pool_key'); - $table->string('cidr'); - $table->string('gateway'); - $table->text('dns_servers_json'); - }); -$schema->create('cloud_backup_restores', static function (Blueprint $table): void { +$schema->create('cloud_instance_migration_source_debts', static function ( + Blueprint $table +): void { $table->increments('id'); - $table->integer('target_host_id')->nullable(); - $table->string('operation_id')->nullable(); - $table->integer('reservation_id')->nullable(); - $table->integer('job_id')->nullable(); + $table->string('debt_key'); + $table->integer('migration_id'); + $table->integer('source_reservation_id'); + $table->integer('source_host_id'); + $table->string('source_uuid'); + $table->integer('source_network_attachment_id'); + $table->text('source_identity_json'); + $table->string('source_identity_sha256'); + $table->string('source_claim_set_sha256'); + $table->text('source_public_network_json'); + $table->string('source_public_network_sha256'); + $table->string('cleanup_operation_id')->nullable(); + $table->integer('cleanup_job_id')->nullable(); $table->string('state'); - $table->string('error_code')->default(''); - $table->integer('updated_at')->nullable(); + $table->integer('active_debt')->nullable(); + $table->string('last_error_code')->default(''); + $table->integer('resolved_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); }); -$schema->create('cloud_instance_migrations', - static function (Blueprint $table): void { - $table->increments('id'); - $table->string('migration_key'); - $table->integer('instance_id'); - $table->integer('vpc_port_id')->nullable(); - $table->integer('restore_id'); - $table->integer('source_host_id'); - $table->string('source_uuid'); - $table->integer('source_reservation_id'); - $table->integer('target_host_id'); - $table->string('target_uuid'); - $table->integer('target_reservation_id'); - $table->integer('source_prepare_job_id')->nullable(); - $table->integer('target_receive_job_id')->nullable(); - $table->integer('firewall_job_id')->nullable(); - $table->integer('activate_job_id')->nullable(); - $table->integer('rollback_job_id')->nullable(); - $table->integer('cleanup_job_id')->nullable(); - $table->string('cleanup_mode'); - $table->string('state'); - $table->string('recovery_action')->default(''); - $table->integer('retry_generation')->default(0); - $table->string('source_endpoint')->default('["https://192.0.2.10:9443"]'); - $table->string('peer_trust_id')->default(''); - $table->string('manifest_digest')->default(''); - $table->boolean('source_was_running')->nullable(); - $table->boolean('source_autostart_enabled')->nullable(); - $table->string('source_service_status'); - $table->string('source_network_status'); - $table->string('error_code'); - $table->integer('cutover_at')->nullable(); - $table->integer('completed_at')->nullable(); - $table->integer('created_at'); - $table->integer('updated_at'); - }); $now = time(); -$sourceUuid = '11111111-1111-4111-8111-111111111111'; -$targetUuid = '22222222-2222-4222-8222-222222222222'; -$capsule->table('cloud_instances')->insert([ - 'id' => 11, 'uuid' => $targetUuid, 'host_id' => 2, 'user_id' => 9, - 'order_id' => 21, 'plan_id' => 31, 'offer_id' => 41, - 'resource_pool_id' => 101, 'reservation_id' => 62, - 'current_period_id' => 71, 'expires_at' => $now + 86400, - 'renewal_price' => '19.95', 'renewal_days' => 30, - 'renewal_traffic_bytes' => 2_000_000, - 'traffic_quota_bytes' => 1_500_000, - 'lifetime_download_bytes' => 800_000, - 'lifetime_upload_bytes' => 300_000, - 'name' => 'same-service', 'status' => 'active', - 'service_status' => 'active', 'network_status' => 'active', - 'spec_json' => '{}', 'created_at' => $now - 1000, - 'updated_at' => $now, -]); -$capsule->table('cloud_orders')->insert([ - 'id' => 21, 'user_id' => 9, 'plan_id' => 31, 'offer_id' => 41, - 'resource_pool_id' => 101, 'host_id' => 1, 'reservation_id' => 61, - 'amount' => '12.34', 'instance_id' => 11, 'status' => 'active', - 'created_at' => $now - 1000, 'updated_at' => $now, +Capsule::table('cloud_hosts')->insert([['id' => 1], ['id' => 2]]); +Capsule::table('cloud_resource_pools')->insert(['id' => 10, 'line_id' => 7]); +Capsule::table('cloud_host_network_attachments')->insert([ + ['id' => 11, 'host_id' => 1, 'line_id' => 7, + 'network_selector' => 'bridge:br0'], + ['id' => 12, 'host_id' => 2, 'line_id' => 7, + 'network_selector' => 'bridge:br0'], ]); -$capsule->table('cloud_resource_reservations')->insert([ - [ - 'id' => 61, 'reservation_key' => str_repeat('a', 32), - 'operation_id' => str_repeat('b', 32), 'order_id' => 21, - 'backup_restore_id' => null, 'offer_id' => 41, - 'resource_pool_id' => 101, 'host_id' => 1, 'user_id' => 9, - 'instance_id' => 11, 'network_attachment_id' => 81, - 'public_ip_pool_id' => 91, 'public_ip_id' => 101, - 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 90, - 'state' => 'committed', 'expires_at' => null, - 'released_at' => null, 'release_reason' => '', - 'created_at' => $now - 1000, 'updated_at' => $now, - ], - [ - 'id' => 62, 'reservation_key' => str_repeat('c', 32), - 'operation_id' => str_repeat('d', 32), 'order_id' => null, - 'backup_restore_id' => 51, 'offer_id' => null, - 'resource_pool_id' => 101, 'host_id' => 2, 'user_id' => 9, - 'instance_id' => 11, 'network_attachment_id' => 82, - 'public_ip_pool_id' => 92, 'public_ip_id' => 102, - 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, - 'state' => 'committed', 'expires_at' => null, - 'released_at' => null, 'release_reason' => '', - 'created_at' => $now - 100, 'updated_at' => $now, - ], + +$pool = static function (int $id, int $hostId, int $attachmentId, + int $family, string $cidr, string $gateway): array { + $network = explode('/', $cidr, 2)[0]; + return [ + 'id' => $id, + 'pool_key' => md5('pool-' . $id), + 'host_id' => $hostId, + 'network_attachment_id' => $attachmentId, + 'address_family' => $family, + 'cidr' => $cidr, + 'gateway' => $gateway, + 'gateway_on_link' => 0, + 'guest_prefix_length' => $family === 4 ? 24 : 64, + 'dns_servers_json' => '["1.1.1.1"]', + 'reserved_ips_json' => '[]', + 'allocation_start' => $family === 6 ? $network . '100' : null, + 'allocation_end' => $family === 6 ? $network . '1ff' : null, + 'allocation_cursor' => null, + 'max_allocations' => $family === 6 ? 100 : 0, + ]; +}; +Capsule::table('cloud_public_ip_pools')->insert([ + $pool(101, 1, 11, 4, '8.8.10.0/24', '8.8.10.1'), + $pool(201, 1, 11, 6, '2404:c140:156:10::/64', + '2404:c140:156:10::1'), + $pool(102, 2, 12, 4, '8.8.20.0/24', '8.8.20.1'), + $pool(202, 2, 12, 6, '2404:c140:156:20::/64', + '2404:c140:156:20::1'), ]); -$capsule->table('cloud_public_ips')->insert([ - [ - 'id' => 101, 'host_id' => 1, 'public_ip_pool_id' => 91, - 'address' => '198.51.100.10', 'prefix_length' => 24, - 'state' => 'assigned', 'reservation_id' => 61, - 'instance_id' => 11, 'operation_id' => str_repeat('b', 32), + +$seed = static function (int $n) use ($now): array { + $instanceId = 100 + $n; + $orderId = 200 + $n; + $restoreId = 300 + $n; + $migrationId = 400 + $n; + $sourceReservationId = 500 + ($n * 2); + $targetReservationId = $sourceReservationId + 1; + $sourceOperation = md5('source-' . $n); + $targetOperation = md5('target-' . $n); + $sourceUuid = sprintf( + '11111111-1111-4111-8111-%012d', + $instanceId + ); + $targetUuid = sprintf( + '22222222-2222-4222-8222-%012d', + $instanceId + ); + $sourceAddresses = [ + 4 => '8.8.10.' . (20 + $n), + 6 => '2404:c140:156:10::1' . dechex(20 + $n), + ]; + $targetAddresses = [ + 4 => '8.8.20.' . (20 + $n), + 6 => '2404:c140:156:20::1' . dechex(20 + $n), + ]; + $sourceNetwork = [ + 'contract' => 1, + 'network_selector' => 'bridge:br0', + 'ipv4' => [ + 'pool_key' => md5('pool-101'), + 'pool_cidr' => '8.8.10.0/24', + 'address_cidr' => $sourceAddresses[4] . '/24', + 'gateway' => '8.8.10.1', + 'gateway_on_link' => false, + ], + 'ipv6' => [ + 'pool_key' => md5('pool-201'), + 'pool_cidr' => '2404:c140:156:10::/64', + 'address_cidr' => $sourceAddresses[6] . '/64', + 'gateway' => '2404:c140:156:10::1', + 'gateway_on_link' => false, + ], + 'dns_servers' => ['1.1.1.1'], + ]; + Capsule::table('cloud_instances')->insert([ + 'id' => $instanceId, 'uuid' => $sourceUuid, 'host_id' => 1, + 'user_id' => 9, 'order_id' => $orderId, 'plan_id' => 31, + 'offer_id' => 41, 'resource_pool_id' => 10, + 'reservation_id' => $sourceReservationId, + 'current_period_id' => 700 + $n, 'expires_at' => $now + 86400, + 'renewal_price' => '19.95', 'renewal_days' => 30, + 'renewal_traffic_bytes' => 2000000, + 'traffic_quota_bytes' => 1500000, + 'lifetime_download_bytes' => 800000, + 'lifetime_upload_bytes' => 300000, + 'name' => 'vm-' . $n, 'status' => 'active', + 'service_status' => 'active', 'network_status' => 'active', + 'spec_json' => json_encode([ + 'managed' => true, + 'vcpu' => 2, + 'cpu_cores' => 2, + 'memory_mib' => 2048, + 'ram_mib' => 2048, + 'disk_gib' => 40, + 'bandwidth_mbps' => 1000, + 'public_network' => $sourceNetwork, + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), + 'created_at' => $now - 1000, + 'updated_at' => $now, + ]); + Capsule::table('cloud_orders')->insert([ + 'id' => $orderId, 'user_id' => 9, 'plan_id' => 31, + 'offer_id' => 41, 'resource_pool_id' => 10, 'host_id' => 1, + 'reservation_id' => $sourceReservationId, 'amount' => '12.34', + 'instance_id' => $instanceId, 'status' => 'active', 'created_at' => $now - 1000, 'updated_at' => $now, - ], - [ - 'id' => 102, 'host_id' => 2, 'public_ip_pool_id' => 92, - 'address' => '203.0.113.20', 'prefix_length' => 24, - 'state' => 'assigned', 'reservation_id' => 62, - 'instance_id' => 11, 'operation_id' => str_repeat('d', 32), + ]); + Capsule::table('cloud_backup_restores')->insert([ + 'id' => $restoreId, 'target_host_id' => 2, + 'reservation_id' => $targetReservationId, 'job_id' => null, + 'operation_id' => $targetOperation, 'state' => 'restoring', + 'error_code' => '', 'updated_at' => $now, + ]); + Capsule::table('cloud_resource_reservations')->insert([ + [ + 'id' => $sourceReservationId, + 'reservation_key' => md5('source-reservation-' . $n), + 'operation_id' => $sourceOperation, 'order_id' => $orderId, + 'backup_restore_id' => null, + 'backup_restore_attempt_key' => null, + 'migration_source_debt_id' => null, 'offer_id' => 41, + 'resource_pool_id' => 10, 'host_id' => 1, 'user_id' => 9, + 'instance_id' => $instanceId, 'network_attachment_id' => null, + 'public_ip_pool_id' => null, 'public_ip_id' => null, + 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, + 'state' => 'committed', 'expires_at' => null, + 'released_at' => null, 'release_reason' => '', + 'created_at' => $now - 1000, 'updated_at' => $now, + ], + [ + 'id' => $targetReservationId, + 'reservation_key' => md5('target-reservation-' . $n), + 'operation_id' => $targetOperation, 'order_id' => null, + 'backup_restore_id' => $restoreId, + 'backup_restore_attempt_key' => null, + 'migration_source_debt_id' => null, 'offer_id' => null, + 'resource_pool_id' => 10, 'host_id' => 2, 'user_id' => 9, + 'instance_id' => null, 'network_attachment_id' => null, + 'public_ip_pool_id' => null, 'public_ip_id' => null, + 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, + 'state' => 'held', 'expires_at' => $now + 600, + 'released_at' => null, 'release_reason' => '', + 'created_at' => $now - 100, 'updated_at' => $now, + ], + ]); + foreach ([ + [$sourceReservationId, 1, 11, 101, 201, + $sourceAddresses, $sourceOperation, 'committed', $instanceId], + [$targetReservationId, 2, 12, 102, 202, + $targetAddresses, $targetOperation, 'reserved', null], + ] as [$reservationId, $hostId, $attachmentId, $v4Pool, $v6Pool, + $addresses, $operation, $state, $ownerInstanceId]) { + foreach ([4 => $v4Pool, 6 => $v6Pool] as $family => $poolId) { + $address = $addresses[$family]; + $ipId = Capsule::table('cloud_public_ips')->insertGetId([ + 'host_id' => $hostId, 'public_ip_pool_id' => $poolId, + 'address_family' => $family, 'address' => $address, + 'address_binary' => inet_pton($address), + 'prefix_length' => $family === 4 ? 24 : 64, + 'state' => $state === 'committed' ? 'assigned' : 'reserved', + 'reservation_id' => $reservationId, + 'reservation_ip_claim_id' => null, + 'instance_id' => $ownerInstanceId, + 'operation_id' => $operation, + 'created_at' => $now, 'updated_at' => $now, + ]); + $claimId = Capsule::table('cloud_resource_reservation_ips') + ->insertGetId([ + 'reservation_id' => $reservationId, + 'address_family' => $family, 'claim_generation' => 1, + 'active_claim' => 1, + 'network_attachment_id' => $attachmentId, + 'public_ip_pool_id' => $poolId, + 'public_ip_id' => $ipId, 'operation_id' => $operation, + 'state' => $state, 'released_at' => null, + 'release_reason' => '', 'created_at' => $now, + 'updated_at' => $now, + ]); + Capsule::table('cloud_public_ips')->where('id', $ipId)->update([ + 'reservation_ip_claim_id' => $claimId, + ]); + } + } + Capsule::table('cloud_instance_migrations')->insert([ + 'id' => $migrationId, 'migration_key' => md5('migration-' . $n), + 'instance_id' => $instanceId, 'restore_id' => $restoreId, + 'source_host_id' => 1, 'source_uuid' => $sourceUuid, + 'source_reservation_id' => $sourceReservationId, + 'target_host_id' => 2, 'target_uuid' => $targetUuid, + 'target_reservation_id' => $targetReservationId, + 'cleanup_job_id' => null, 'cleanup_mode' => 'delete', + 'state' => 'transferring', 'recovery_action' => '', + 'retry_generation' => 0, 'source_service_status' => 'active', + 'source_network_status' => 'active', 'error_code' => '', 'created_at' => $now - 100, 'updated_at' => $now, - ], -]); -$capsule->table('cloud_instance_migrations')->insert([ - 'id' => 1, 'migration_key' => str_repeat('e', 32), - 'instance_id' => 11, 'restore_id' => 51, - 'source_host_id' => 1, 'source_uuid' => $sourceUuid, - 'source_reservation_id' => 61, 'target_host_id' => 2, - 'target_uuid' => $targetUuid, 'target_reservation_id' => 62, - 'source_prepare_job_id' => 91, 'target_receive_job_id' => 92, - 'firewall_job_id' => null, 'activate_job_id' => null, - 'rollback_job_id' => null, - 'cleanup_job_id' => 93, - 'cleanup_mode' => 'delete', 'state' => 'cleanup_running', - 'source_was_running' => true, 'source_autostart_enabled' => true, - 'source_service_status' => 'active', - 'source_network_status' => 'active', 'error_code' => '', - 'cutover_at' => $now - 10, 'completed_at' => null, - 'created_at' => $now - 200, 'updated_at' => $now, -]); + ]); + $sourceSet = CloudPublicNetworkAllocationService::currentClaimSet( + $sourceReservationId, + 'committed', + $instanceId + ); + $targetSet = CloudPublicNetworkAllocationService::currentClaimSet( + $targetReservationId, + 'held' + ); + Capsule::table('cloud_instances')->where('id', $instanceId)->update([ + 'spec_json' => json_encode([ + 'managed' => true, + 'vcpu' => 2, + 'cpu_cores' => 2, + 'memory_mib' => 2048, + 'ram_mib' => 2048, + 'disk_gib' => 40, + 'bandwidth_mbps' => 1000, + 'public_network' => $sourceSet->publicNetwork()->toArray(), + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), + ]); + return compact( + 'instanceId', 'orderId', 'restoreId', 'migrationId', + 'sourceReservationId', 'targetReservationId', + 'sourceOperation', 'targetOperation', 'sourceUuid', 'targetUuid', + 'sourceSet', 'targetSet' + ); +}; -$before = CloudInstance::findOrFail(11)->only([ - 'id', 'user_id', 'order_id', 'plan_id', 'offer_id', - 'current_period_id', 'expires_at', 'renewal_price', 'renewal_days', - 'renewal_traffic_bytes', 'traffic_quota_bytes', - 'lifetime_download_bytes', 'lifetime_upload_bytes', 'created_at', -]); -CloudInstanceMigrationSettlement::finalizeSourceDeletionLocked( - CloudInstanceMigration::findOrFail(1) -); +$cutover = static function (array $case): CloudInstanceMigrationSourceDebt { + $handoff = CloudPublicNetworkAllocationService::handoffMigrationBundles( + $case['sourceReservationId'], + $case['targetReservationId'], + $case['instanceId'], + $case['sourceOperation'], + $case['targetOperation'] + ); + if (!hash_equals( + $case['sourceSet']->digest(), + $handoff['source']->digest() + ) || !hash_equals( + $case['targetSet']->digest(), + $handoff['target']->digest() + )) { + throw new RuntimeException('Migration handoff changed claim identity.'); + } + Capsule::table('cloud_instances')->where('id', $case['instanceId']) + ->update([ + 'uuid' => $case['targetUuid'], + 'host_id' => 2, + 'reservation_id' => $case['targetReservationId'], + 'spec_json' => json_encode([ + 'managed' => true, + 'public_network' => + $handoff['target']->publicNetwork()->toArray(), + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), + 'updated_at' => time(), + ]); + $migration = CloudInstanceMigration::findOrFail($case['migrationId']); + return CloudInstanceMigrationSettlement::settleCutoverLocked( + $migration, + $handoff['source'] + ); +}; -$instance = CloudInstance::findOrFail(11); -$after = $instance->only(array_keys($before)); -$source = $capsule->table('cloud_resource_reservations')->find(61); -$target = $capsule->table('cloud_resource_reservations')->find(62); -$sourceIp = $capsule->table('cloud_public_ips')->find(101); -$targetIp = $capsule->table('cloud_public_ips')->find(102); -$order = $capsule->table('cloud_orders')->find(21); +$claimIdentity = static function (int $reservationId): array { + return array_map( + static fn (object $row): array => [ + (int) $row->id, + (int) $row->reservation_id, + (int) $row->public_ip_id, + (string) $row->operation_id, + (int) $row->claim_generation, + ], + Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', $reservationId) + ->orderBy('address_family')->orderBy('claim_generation') + ->orderBy('id')->get()->all() + ); +}; +$job = static function (array $case, int $id, string $operation): CloudJob { + $job = new CloudJob(); + $job->id = $id; + $job->kind = 'instance_migration_cleanup'; + $job->instance_id = $case['instanceId']; + $job->host_id = 1; + $job->backup_restore_id = $case['restoreId']; + $job->operation_id = $operation; + return $job; +}; -$assert($before === $after, - 'Maintenance migration modified identity, billing, expiry, period, or traffic ownership fields.'); -$assert((int) $instance->reservation_id === 61 - && (int) $instance->resource_pool_id === 101 - && (int) $source->host_id === 2 - && (int) $source->resource_pool_id === 101 - && (int) $source->public_ip_id === 102 - && (int) $source->network_attachment_id === 82 - && (int) $source->disk_gib === 40 - && (string) $source->state === 'committed', - 'The original paid reservation did not take ownership of target placement.'); -$assert((string) $sourceIp->state === 'available' - && $sourceIp->reservation_id === null - && $sourceIp->instance_id === null - && (string) $targetIp->state === 'assigned' - && (int) $targetIp->reservation_id === 61 - && (int) $targetIp->instance_id === 11, - 'Maintenance migration did not release the old IP and transfer the new IP atomically.'); -$assert((string) $target->state === 'released' - && $target->instance_id === null - && $target->public_ip_id === null - && (string) $target->release_reason - === 'maintenance_migration_transferred', - 'Temporary restore reservation remained billable after placement transfer.'); -$assert((int) $order->host_id === 2 - && (int) $order->resource_pool_id === 101 - && (int) $order->reservation_id === 61 - && (int) $order->plan_id === 31 - && (int) $order->offer_id === 41 - && (string) $order->amount === '12.34', - 'Order placement changed without preserving its commercial terms.'); +$migrationState = static function (array $case): array { + $rows = static function ($query): array { + return $query->orderBy('id')->get() + ->map(static fn ($row): array => (array) $row)->all(); + }; + return [ + 'instance' => (array) Capsule::table('cloud_instances') + ->find($case['instanceId']), + 'order' => (array) Capsule::table('cloud_orders') + ->find($case['orderId']), + 'restore' => (array) Capsule::table('cloud_backup_restores') + ->find($case['restoreId']), + 'migration' => (array) Capsule::table('cloud_instance_migrations') + ->find($case['migrationId']), + 'reservations' => $rows(Capsule::table('cloud_resource_reservations') + ->whereIn('id', [ + $case['sourceReservationId'], + $case['targetReservationId'], + ])), + 'claims' => $rows(Capsule::table('cloud_resource_reservation_ips') + ->whereIn('reservation_id', [ + $case['sourceReservationId'], + $case['targetReservationId'], + ])), + 'ips' => $rows(Capsule::table('cloud_public_ips') + ->whereIn('reservation_id', [ + $case['sourceReservationId'], + $case['targetReservationId'], + ])), + 'debts' => $rows(Capsule::table( + 'cloud_instance_migration_source_debts' + )->where('migration_id', $case['migrationId'])), + 'artifacts' => $rows(Capsule::table('cloud_peer_migration_artifacts') + ->where('migration_id', $case['migrationId'])), + 'runtime' => $rows(Capsule::table('cloud_instance_runtime') + ->where('instance_id', $case['instanceId'])), + ]; +}; -$capsule->table('cloud_instances')->insert([ - 'id' => 12, 'uuid' => $sourceUuid, 'host_id' => 1, 'user_id' => 9, - 'order_id' => 22, 'plan_id' => 31, 'offer_id' => 41, - 'resource_pool_id' => 101, 'reservation_id' => 63, - 'current_period_id' => 72, 'expires_at' => $now + 86400, - 'renewal_price' => '19.95', 'renewal_days' => 30, - 'renewal_traffic_bytes' => 2_000_000, - 'traffic_quota_bytes' => 1_500_000, - 'lifetime_download_bytes' => 100, 'lifetime_upload_bytes' => 50, - 'name' => 'rollback-service', 'status' => 'active', - 'service_status' => 'active', 'network_status' => 'active', - 'spec_json' => '{}', 'created_at' => $now - 1000, - 'updated_at' => $now, -]); -$capsule->table('cloud_instance_migrations')->insert([ - 'id' => 2, 'migration_key' => str_repeat('f', 32), - 'instance_id' => 12, 'restore_id' => 52, - 'source_host_id' => 1, 'source_uuid' => $sourceUuid, - 'source_reservation_id' => 63, 'target_host_id' => 2, - 'target_uuid' => $targetUuid, 'target_reservation_id' => 64, - 'source_prepare_job_id' => 191, 'target_receive_job_id' => 192, - 'firewall_job_id' => null, 'activate_job_id' => null, - 'rollback_job_id' => 200, - 'cleanup_job_id' => null, 'cleanup_mode' => 'delete', - 'state' => 'rollback_running', 'source_was_running' => true, - 'source_autostart_enabled' => true, - 'source_service_status' => 'active', - 'source_network_status' => 'active', - 'error_code' => 'migration_restore_failed', 'cutover_at' => null, - 'completed_at' => null, 'created_at' => $now - 200, +// Exercise the real private transfer finalizer under the same outer +// transaction supplied by the worker. A deterministic failure at the runtime +// cache write happens after network handoff, artifact sealing, placement +// switch, commercial settlement, and restore finalization; every one of those +// writes must roll back together. +$atomic = $seed(5); +$atomicArtifactId = md5('migration-root-' . $atomic['migrationId']); +$atomicArtifactSha = hash('sha256', 'migration-root-bytes'); +$atomicManifest = hash('sha256', 'migration-manifest'); +$atomicJobId = 9105; +Capsule::table('cloud_peer_migration_artifacts')->insert([ + 'migration_id' => $atomic['migrationId'], + 'artifact_id' => $atomicArtifactId, + 'role' => 'root', + 'source_snapshot_id' => null, + 'snapshot_key' => null, + 'name' => '', + 'consistency' => 'crash', + 'object_size_bytes' => 123456, + 'virtual_size_bytes' => 40 * 1073741824, + 'sha256' => $atomicArtifactSha, + 'state' => 'ready', + 'snapshot_created_at' => null, + 'created_at' => $now - 20, 'updated_at' => $now, ]); -$capsule->table('cloud_backup_restores')->insert([ - 'id' => 52, - 'state' => 'failed', -]); +Capsule::table('cloud_instance_migrations') + ->where('id', $atomic['migrationId'])->update([ + 'target_receive_job_id' => $atomicJobId, + 'manifest_digest' => $atomicManifest, + 'source_was_running' => 1, + 'source_autostart_enabled' => 1, + ]); +$atomicJob = new CloudJob(); +$atomicJob->id = $atomicJobId; +$atomicJob->instance_id = $atomic['instanceId']; +$atomicJob->backup_restore_id = $atomic['restoreId']; +$atomicJob->host_id = 2; +$atomicJob->user_id = 9; +$atomicJob->operation_id = $atomic['targetOperation']; +$atomicTargetNetwork = $atomic['targetSet']->publicNetwork()->toArray(); +$atomicPayload = [ + 'maintenance_migration_id' => $atomic['migrationId'], + 'body' => [ + 'vcpu' => 2, + 'memory_mib' => 2048, + 'disk_gib' => 40, + 'bandwidth_mbps' => 1000, + 'public_network' => $atomicTargetNetwork, + 'vpc_port' => null, + ], +]; +$atomicResult = [ + 'migration_id' => md5('migration-5'), + 'manifest_digest' => $atomicManifest, + 'uuid' => $atomic['targetUuid'], + 'status' => 'shutoff', + 'staged_powered_off' => true, + 'vcpu' => 2, + 'memory_mib' => 2048, + 'disk_gib' => 40, + 'bandwidth_mbps' => 1000, + 'managed' => true, + 'public_network' => $atomicTargetNetwork, + 'vpc_port' => null, + 'artifacts' => [[ + 'id' => $atomicArtifactId, + 'role' => 'root', + 'snapshot_key' => '', + 'name' => '', + 'consistency' => 'crash', + 'size_bytes' => 123456, + 'virtual_size_bytes' => 40 * 1073741824, + 'sha256' => $atomicArtifactSha, + 'created_at' => $now - 20, + ]], +]; +$finishTransfer = new ReflectionMethod( + CloudInstanceMigrationService::class, + 'finishTransferLocked' +); +$beforeAtomic = $migrationState($atomic); +Capsule::statement( + "CREATE TRIGGER fail_migration_runtime_seed\n" + . "BEFORE INSERT ON cloud_instance_runtime\n" + . "BEGIN SELECT RAISE(ABORT, 'forced runtime seed failure'); END" +); +try { + $atomicError = $captures( + static fn () => Capsule::connection()->transaction( + static fn () => $finishTransfer->invoke( + null, + CloudInstanceMigration::findOrFail($atomic['migrationId']), + $atomicJob, + 'succeeded', + $atomicResult, + $atomicPayload + ) + ), + 'Runtime-cache failure did not abort the transfer finalizer.' + ); +} finally { + Capsule::statement('DROP TRIGGER fail_migration_runtime_seed'); +} +$assert(str_contains($atomicError->getMessage(), + 'forced runtime seed failure'), + 'Transfer finalizer failed before the intended post-settlement fault.'); +$assert($beforeAtomic === $migrationState($atomic), + 'Outer transaction retained a partial transfer finalization.'); + +// A confirmed source rollback must restore the original service and release +// the complete prepared target bundle. +$rolledBackCase = $seed(6); +$rollbackJobId = 9206; +Capsule::table('cloud_instance_migrations') + ->where('id', $rolledBackCase['migrationId'])->update([ + 'target_receive_job_id' => 9196, + 'rollback_job_id' => $rollbackJobId, + 'state' => 'rollback_running', + 'source_was_running' => 1, + 'source_autostart_enabled' => 1, + 'error_code' => 'migration_restore_failed', + ]); $rollbackJob = new CloudJob(); -$rollbackJob->id = 200; -$rollbackJob->instance_id = 12; +$rollbackJob->id = $rollbackJobId; +$rollbackJob->instance_id = $rolledBackCase['instanceId']; $rollbackJob->host_id = 1; $finishRollback = new ReflectionMethod( CloudInstanceMigrationService::class, @@ -361,106 +758,61 @@ $finishRollback = new ReflectionMethod( ); $finishRollback->invoke( null, - CloudInstanceMigration::findOrFail(2), + CloudInstanceMigration::findOrFail($rolledBackCase['migrationId']), $rollbackJob, 'succeeded', [ - 'uuid' => $sourceUuid, - 'state' => 'running', + 'uuid' => $rolledBackCase['sourceUuid'], 'running' => true, 'autostart_enabled' => true, ], [] ); -$rolledBack = CloudInstanceMigration::findOrFail(2); +$rolledBack = CloudInstanceMigration::findOrFail( + $rolledBackCase['migrationId'] +); $assert((string) $rolledBack->state === 'rolled_back' && (string) $rolledBack->error_code === 'migration_transfer_aborted' && $rolledBack->completed_at !== null - && (int) CloudInstance::findOrFail(12)->host_id === 1, - 'A failed target restore did not confirm recovery of the original source service.'); + && (int) CloudInstance::findOrFail( + $rolledBackCase['instanceId'] + )->host_id === 1 + && Capsule::table('cloud_public_ips') + ->where('reservation_id', $rolledBackCase['sourceReservationId']) + ->where('state', 'assigned')->count() === 2 + && Capsule::table('cloud_public_ips') + ->where('state', 'available')->count() >= 2, + 'Confirmed rollback did not preserve source or release target dual stack.'); -$capsule->table('cloud_instances')->insert([ - 'id' => 13, 'uuid' => $targetUuid, 'host_id' => 2, 'user_id' => 9, - 'order_id' => 23, 'plan_id' => 31, 'offer_id' => 41, - 'resource_pool_id' => 101, 'reservation_id' => 65, - 'current_period_id' => 73, 'expires_at' => $now + 86400, - 'renewal_price' => '19.95', 'renewal_days' => 30, - 'renewal_traffic_bytes' => 2_000_000, - 'traffic_quota_bytes' => 1_500_000, - 'lifetime_download_bytes' => 100, 'lifetime_upload_bytes' => 50, - 'name' => 'powered-off-service', 'status' => 'active', - 'service_status' => 'suspended_expired', - 'network_status' => 'blocked_expired', - 'spec_json' => '{"guest_hostname":"retained.example"}', - 'created_at' => $now - 1000, 'updated_at' => $now, -]); -$capsule->table('cloud_orders')->insert([ - 'id' => 23, 'user_id' => 9, 'plan_id' => 31, 'offer_id' => 41, - 'resource_pool_id' => 101, 'host_id' => 1, 'reservation_id' => 66, - 'amount' => '19.95', 'instance_id' => 13, 'status' => 'active', - 'created_at' => $now - 1000, 'updated_at' => $now, -]); -$capsule->table('cloud_resource_reservations')->insert([ - [ - 'id' => 66, 'reservation_key' => str_repeat('6', 32), - 'operation_id' => str_repeat('7', 32), 'order_id' => 23, - 'backup_restore_id' => null, 'offer_id' => 41, - 'resource_pool_id' => 101, 'host_id' => 1, 'user_id' => 9, - 'instance_id' => 13, 'network_attachment_id' => 81, - 'public_ip_pool_id' => 91, 'public_ip_id' => 103, - 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, - 'state' => 'committed', 'expires_at' => null, - 'released_at' => null, 'release_reason' => '', - 'created_at' => $now - 1000, 'updated_at' => $now, - ], - [ - 'id' => 65, 'reservation_key' => str_repeat('8', 32), - 'operation_id' => str_repeat('9', 32), 'order_id' => null, - 'backup_restore_id' => 53, 'offer_id' => null, - 'resource_pool_id' => 101, 'host_id' => 2, 'user_id' => 9, - 'instance_id' => 13, 'network_attachment_id' => 82, - 'public_ip_pool_id' => 92, 'public_ip_id' => 104, - 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, - 'state' => 'committed', 'expires_at' => null, - 'released_at' => null, 'release_reason' => '', - 'created_at' => $now - 100, 'updated_at' => $now, - ], -]); -$capsule->table('cloud_public_ips')->insert([ - [ - 'id' => 103, 'host_id' => 1, 'public_ip_pool_id' => 91, - 'address' => '198.51.100.30', 'prefix_length' => 24, - 'state' => 'assigned', 'reservation_id' => 66, - 'instance_id' => 13, 'operation_id' => str_repeat('7', 32), - 'created_at' => $now - 1000, 'updated_at' => $now, - ], - [ - 'id' => 104, 'host_id' => 2, 'public_ip_pool_id' => 92, - 'address' => '203.0.113.40', 'prefix_length' => 24, - 'state' => 'assigned', 'reservation_id' => 65, - 'instance_id' => 13, 'operation_id' => str_repeat('9', 32), - 'created_at' => $now - 100, 'updated_at' => $now, - ], -]); -$capsule->table('cloud_instance_migrations')->insert([ - 'id' => 3, 'migration_key' => str_repeat('1', 32), - 'instance_id' => 13, 'restore_id' => 53, - 'source_host_id' => 1, 'source_uuid' => $sourceUuid, - 'source_reservation_id' => 66, 'target_host_id' => 2, - 'target_uuid' => $targetUuid, 'target_reservation_id' => 65, - 'source_prepare_job_id' => 291, 'target_receive_job_id' => 292, - 'firewall_job_id' => null, 'activate_job_id' => 300, - 'rollback_job_id' => null, 'cleanup_job_id' => null, - 'cleanup_mode' => 'retain', 'state' => 'activation_running', - 'source_was_running' => false, 'source_autostart_enabled' => true, - 'source_service_status' => 'suspended_expired', - 'source_network_status' => 'blocked_expired', 'error_code' => '', - 'cutover_at' => $now - 10, 'completed_at' => null, - 'created_at' => $now - 200, 'updated_at' => $now, -]); +// Activation must preserve a powered-off expired service exactly as frozen at +// migration start, including the durable guest hostname. +$poweredOffCase = $seed(7); +$poweredOffDebt = $cutover($poweredOffCase); +$poweredSpec = json_decode((string) CloudInstance::findOrFail( + $poweredOffCase['instanceId'] +)->spec_json, true, 16, JSON_THROW_ON_ERROR); +$poweredSpec['guest_hostname'] = 'retained.example'; +Capsule::table('cloud_instances') + ->where('id', $poweredOffCase['instanceId'])->update([ + 'service_status' => 'suspended_expired', + 'network_status' => 'blocked_expired', + 'spec_json' => json_encode($poweredSpec, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), + ]); +$activationJobId = 9307; +Capsule::table('cloud_instance_migrations') + ->where('id', $poweredOffCase['migrationId'])->update([ + 'activate_job_id' => $activationJobId, + 'cleanup_mode' => 'retain', + 'state' => 'activation_running', + 'source_was_running' => 0, + 'source_autostart_enabled' => 1, + 'source_service_status' => 'suspended_expired', + 'source_network_status' => 'blocked_expired', + ]); $activationJob = new CloudJob(); -$activationJob->id = 300; -$activationJob->instance_id = 13; +$activationJob->id = $activationJobId; +$activationJob->instance_id = $poweredOffCase['instanceId']; $activationJob->host_id = 2; $activationJob->user_id = 9; $finishActivation = new ReflectionMethod( @@ -469,33 +821,459 @@ $finishActivation = new ReflectionMethod( ); $finishActivation->invoke( null, - CloudInstanceMigration::findOrFail(3), + CloudInstanceMigration::findOrFail($poweredOffCase['migrationId']), $activationJob, 'succeeded', [ - 'uuid' => $targetUuid, - 'state' => 'shutoff', + 'uuid' => $poweredOffCase['targetUuid'], 'running' => false, 'autostart_enabled' => true, ], [] ); -$poweredOff = CloudInstance::findOrFail(13); -$activated = CloudInstanceMigration::findOrFail(3); +$poweredOff = CloudInstance::findOrFail($poweredOffCase['instanceId']); +$activated = CloudInstanceMigration::findOrFail( + $poweredOffCase['migrationId'] +); +$activatedDebt = CloudInstanceMigrationSourceDebt::findOrFail( + (int) $poweredOffDebt->id +); $assert((string) $activated->state === 'source_retained' + && (string) $activatedDebt->state === 'retained' && (string) $poweredOff->service_status === 'suspended_expired' && (string) $poweredOff->network_status === 'blocked_expired' - && str_contains((string) $poweredOff->spec_json, - 'retained.example'), - 'Maintenance migration did not retain powered-off, disconnected, or hostname state.'); + && str_contains((string) $poweredOff->spec_json, 'retained.example'), + 'Activation changed powered-off, expired-network, or hostname state.'); + +// Automatic cleanup: cutover creates prepared debt, then the exact cleanup +// operation releases both source families without moving either claim set. +$automatic = $seed(1); +$historicalSourceOperation = md5('source-history-automatic'); +$activeSourceClaims = Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', $automatic['sourceReservationId']) + ->where('active_claim', 1)->orderBy('address_family')->get()->all(); +foreach ($activeSourceClaims as $activeSourceClaim) { + Capsule::table('cloud_resource_reservation_ips')->insert([ + 'reservation_id' => $automatic['sourceReservationId'], + 'address_family' => (int) $activeSourceClaim->address_family, + 'claim_generation' => 1, + 'active_claim' => null, + 'network_attachment_id' => + (int) $activeSourceClaim->network_attachment_id, + 'public_ip_pool_id' => (int) $activeSourceClaim->public_ip_pool_id, + 'public_ip_id' => (int) $activeSourceClaim->public_ip_id, + 'operation_id' => $historicalSourceOperation, + 'state' => 'released', + 'released_at' => $now - 60, + 'release_reason' => 'network_change_completed', + 'created_at' => $now - 120, + 'updated_at' => $now - 60, + ]); + Capsule::table('cloud_resource_reservation_ips') + ->where('id', (int) $activeSourceClaim->id) + ->update(['claim_generation' => 2]); +} +$automatic['sourceSet'] = CloudPublicNetworkAllocationService::currentClaimSet( + $automatic['sourceReservationId'], + 'committed', + $automatic['instanceId'] +); +$sourceIdentityBefore = $claimIdentity($automatic['sourceReservationId']); +$targetIdentityBefore = $claimIdentity($automatic['targetReservationId']); +$debt = $cutover($automatic); +$migration = CloudInstanceMigration::findOrFail($automatic['migrationId']); +$sameDebt = CloudInstanceMigrationSettlement::settleCutoverLocked( + $migration, + $automatic['sourceSet'] +); +$assert((int) $sameDebt->id === (int) $debt->id, + 'Cutover retry created a second source debt.'); +$cleanup = $job($automatic, 9001, md5('cleanup-automatic')); +$migration->cleanup_job_id = (int) $cleanup->id; +$migration->save(); +CloudInstanceMigrationSettlement::bindCleanupLocked($migration, $cleanup); +CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup); +CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup); +$wrongCleanup = $job($automatic, 9010, md5('cleanup-wrong')); +$rejects( + static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked( + $migration, + $wrongCleanup + ), + 'Released debt accepted a different cleanup generation.' +); + +$instance = Capsule::table('cloud_instances') + ->find($automatic['instanceId']); +$order = Capsule::table('cloud_orders')->find($automatic['orderId']); +$source = Capsule::table('cloud_resource_reservations') + ->find($automatic['sourceReservationId']); +$target = Capsule::table('cloud_resource_reservations') + ->find($automatic['targetReservationId']); +$releasedDebt = Capsule::table('cloud_instance_migration_source_debts') + ->find((int) $debt->id); +$assert((int) $instance->reservation_id === $automatic['targetReservationId'] + && (int) $order->reservation_id === $automatic['targetReservationId'] + && (int) $target->order_id === $automatic['orderId'] + && $target->backup_restore_id === null + && (string) $target->state === 'committed' + && (int) $source->migration_source_debt_id === (int) $debt->id + && $source->order_id === null + && (string) $source->state === 'released' + && (string) $releasedDebt->state === 'released' + && $releasedDebt->active_debt === null + && $releasedDebt->resolved_at !== null, + 'Migration did not settle target ownership and source cleanup debt.'); +$assert($sourceIdentityBefore + === $claimIdentity($automatic['sourceReservationId']) + && $targetIdentityBefore + === $claimIdentity($automatic['targetReservationId']) + && (int) CloudInstanceMigration::findOrFail( + $automatic['migrationId'] + )->source_reservation_id === $automatic['sourceReservationId'] + && (int) CloudInstanceMigration::findOrFail( + $automatic['migrationId'] + )->target_reservation_id === $automatic['targetReservationId'], + 'Migration moved claims or swapped reservation identity.'); +$assert(Capsule::table('cloud_public_ips') + ->where('reservation_id', $automatic['sourceReservationId']) + ->count() === 0 + && Capsule::table('cloud_public_ips') + ->whereIn('id', array_column($sourceIdentityBefore, 2)) + ->where('state', 'available')->count() === 2 + && Capsule::table('cloud_public_ips') + ->where('reservation_id', $automatic['targetReservationId']) + ->where('state', 'assigned')->count() === 2, + 'Source cleanup did not release exactly its own dual-stack bundle.'); + +// A retained source may be cleaned later; an unknown cleanup keeps the exact +// original pair fenced and cannot create a second debt or release one family. +$retained = $seed(2); +$retainedDebt = $cutover($retained); +$retainedMigration = CloudInstanceMigration::findOrFail( + $retained['migrationId'] +); +CloudInstanceMigrationSettlement::markRetainedLocked($retainedMigration); +$retainedJob = $job($retained, 9002, md5('cleanup-retained')); +$retainedMigration->cleanup_job_id = (int) $retainedJob->id; +$retainedMigration->save(); +CloudInstanceMigrationSettlement::bindCleanupLocked( + $retainedMigration, + $retainedJob +); +CloudInstanceMigrationSettlement::markCleanupAttentionLocked( + $retainedMigration, + $retainedJob, + 'operation_effect_unknown' +); +$attention = CloudInstanceMigrationSourceDebt::findOrFail( + (int) $retainedDebt->id +); +$assert((string) $attention->state === 'attention' + && (int) $attention->active_debt === 1 + && (int) $attention->cleanup_job_id === (int) $retainedJob->id + && Capsule::table('cloud_public_ips') + ->where('reservation_id', $retained['sourceReservationId']) + ->where('state', 'reserved')->count() === 2, + 'Unknown cleanup released or detached the retained source bundle.'); +CloudInstanceMigrationSettlement::releaseSourceLocked( + $retainedMigration, + $retainedJob +); +$lateSuccess = CloudInstanceMigrationSourceDebt::findOrFail( + (int) $retainedDebt->id +); +$assert((string) $lateSuccess->state === 'released' + && $lateSuccess->active_debt === null + && Capsule::table('cloud_public_ips') + ->where('reservation_id', $retained['sourceReservationId']) + ->count() === 0, + 'Late confirmed cleanup success did not resolve attention debt.'); + +// A confirmed not-applied cleanup clears only its job claim, leaving the same +// debt and bundle eligible for an explicit successor operation. +$retryable = $seed(3); +$retryableDebt = $cutover($retryable); +$retryableMigration = CloudInstanceMigration::findOrFail( + $retryable['migrationId'] +); +$firstCleanup = $job($retryable, 9003, md5('cleanup-failed')); +$retryableMigration->cleanup_job_id = (int) $firstCleanup->id; +$retryableMigration->save(); +CloudInstanceMigrationSettlement::bindCleanupLocked( + $retryableMigration, + $firstCleanup +); +CloudInstanceMigrationSettlement::markCleanupFailedLocked( + $retryableMigration, + $firstCleanup, + 'migration_cleanup_failed' +); +$afterFailure = CloudInstanceMigrationSourceDebt::findOrFail( + (int) $retryableDebt->id +); +$assert((string) $afterFailure->state === 'retained' + && $afterFailure->cleanup_job_id === null + && $afterFailure->cleanup_operation_id === null + && Capsule::table('cloud_public_ips') + ->where('reservation_id', $retryable['sourceReservationId']) + ->where('state', 'reserved')->count() === 2, + 'Confirmed-not-applied cleanup did not retain a retryable source debt.'); +$secondCleanup = $job($retryable, 9004, md5('cleanup-retry')); +$retryableMigration->cleanup_job_id = (int) $secondCleanup->id; +$retryableMigration->save(); +CloudInstanceMigrationSettlement::bindCleanupLocked( + $retryableMigration, + $secondCleanup +); -// Bundle retry rotation is exercised by -// CloudPublicNetworkAllocationServiceTest with two address families, retained -// claim generations, and a forced second-family rollback. Keeping the former -// singular-IP probe here would model a target shape that production rejects. +// Corrupt frozen evidence must fail closed without changing either family. +Capsule::table('cloud_instance_migration_source_debts') + ->where('id', (int) $retryableDebt->id) + ->update(['source_identity_sha256' => str_repeat('0', 64)]); +$beforeDrift = Capsule::table('cloud_public_ips') + ->whereIn('reservation_id', [ + $retryable['sourceReservationId'], + $retryable['targetReservationId'], + ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(); +$rejects( + static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked( + $retryableMigration, + $secondCleanup + ), + 'Drifted source identity was released.' +); +$afterDrift = Capsule::table('cloud_public_ips') + ->whereIn('reservation_id', [ + $retryable['sourceReservationId'], + $retryable['targetReservationId'], + ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(); +$assert($beforeDrift === $afterDrift, + 'Failed identity validation changed public-network ownership.'); + +// A second-family write failure rolls back the complete handoff, including +// the already-updated IPv4 rows. +$rollback = $seed(4); +$beforeRollback = [ + 'reservations' => Capsule::table('cloud_resource_reservations') + ->whereIn('id', [ + $rollback['sourceReservationId'], + $rollback['targetReservationId'], + ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), + 'claims' => Capsule::table('cloud_resource_reservation_ips') + ->whereIn('reservation_id', [ + $rollback['sourceReservationId'], + $rollback['targetReservationId'], + ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), + 'ips' => Capsule::table('cloud_public_ips') + ->whereIn('reservation_id', [ + $rollback['sourceReservationId'], + $rollback['targetReservationId'], + ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), +]; +Capsule::statement( + "CREATE TRIGGER fail_migration_ipv6_handoff\n" + . "BEFORE UPDATE ON cloud_public_ips\n" + . "WHEN OLD.address_family = 6 AND NEW.state <> OLD.state\n" + . "BEGIN SELECT RAISE(ABORT, 'forced ipv6 failure'); END" +); +$rejects( + static fn () => CloudPublicNetworkAllocationService:: + handoffMigrationBundles( + $rollback['sourceReservationId'], + $rollback['targetReservationId'], + $rollback['instanceId'], + $rollback['sourceOperation'], + $rollback['targetOperation'] + ), + 'Second-family fault did not abort migration handoff.' +); +Capsule::statement('DROP TRIGGER fail_migration_ipv6_handoff'); +$afterRollback = [ + 'reservations' => Capsule::table('cloud_resource_reservations') + ->whereIn('id', [ + $rollback['sourceReservationId'], + $rollback['targetReservationId'], + ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), + 'claims' => Capsule::table('cloud_resource_reservation_ips') + ->whereIn('reservation_id', [ + $rollback['sourceReservationId'], + $rollback['targetReservationId'], + ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), + 'ips' => Capsule::table('cloud_public_ips') + ->whereIn('reservation_id', [ + $rollback['sourceReservationId'], + $rollback['targetReservationId'], + ])->orderBy('id')->get()->map(static fn ($row) => (array) $row)->all(), +]; +$assert($beforeRollback === $afterRollback, + 'Second-family failure left a partial migration handoff.'); + +// A released replay is still a ledger verification, not an unconditional +// success. Tampering either the paid target authority or the frozen source +// claim history must fail closed even though no allocator mutation remains. +Capsule::table('cloud_resource_reservations') + ->where('id', $automatic['targetReservationId']) + ->update(['order_id' => null]); +try { + $rejects( + static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked( + $migration, + $cleanup + ), + 'Released replay accepted drifted target commercial authority.' + ); +} finally { + Capsule::table('cloud_resource_reservations') + ->where('id', $automatic['targetReservationId']) + ->update(['order_id' => $automatic['orderId']]); +} +$releasedSourceClaim = Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', $automatic['sourceReservationId']) + ->where('operation_id', $automatic['sourceOperation']) + ->orderBy('address_family')->first(); +$assert($releasedSourceClaim !== null, + 'Released source claim history disappeared.'); +Capsule::table('cloud_resource_reservation_ips') + ->where('id', (int) $releasedSourceClaim->id) + ->update(['operation_id' => md5('tampered-released-claim')]); +try { + $rejects( + static fn () => CloudInstanceMigrationSettlement::releaseSourceLocked( + $migration, + $cleanup + ), + 'Released replay accepted drifted frozen source claim history.' + ); +} finally { + Capsule::table('cloud_resource_reservation_ips') + ->where('id', (int) $releasedSourceClaim->id) + ->update(['operation_id' => $automatic['sourceOperation']]); +} -// The dual-stack prepare-abort release path is guarded by the migration receive -// contract and the allocator's forced second-family rollback tests. This file -// retains only the legacy settlement probes until Stage 4 replaces that module. +// Historical source generations remain immutable audit rows. Once cleanup is +// released, the concrete addresses may be allocated to a new reservation and +// claim generation; replay must validate the frozen source generation without +// mistaking the new owner for stale source authority. +$reallocatedReservationId = 9901; +$reallocatedOperation = md5('reallocated-source-addresses'); +Capsule::table('cloud_resource_reservations')->insert([ + 'id' => $reallocatedReservationId, + 'reservation_key' => md5('reallocated-reservation'), + 'operation_id' => $reallocatedOperation, + 'order_id' => null, + 'backup_restore_id' => null, + 'backup_restore_attempt_key' => null, + 'migration_source_debt_id' => null, + 'offer_id' => null, + 'resource_pool_id' => 10, + 'host_id' => 1, + 'user_id' => 9, + 'instance_id' => null, + 'network_attachment_id' => null, + 'public_ip_pool_id' => null, + 'public_ip_id' => null, + 'cpu_cores' => 2, + 'memory_mib' => 2048, + 'disk_gib' => 40, + 'state' => 'held', + 'expires_at' => $now + 600, + 'released_at' => null, + 'release_reason' => '', + 'created_at' => $now, + 'updated_at' => $now, +]); +$releasedSourceClaims = Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', $automatic['sourceReservationId']) + ->where('operation_id', $automatic['sourceOperation']) + ->orderBy('address_family')->get()->keyBy('address_family'); +$reallocatedClaims = []; +foreach ([4, 6] as $family) { + $releasedClaim = $releasedSourceClaims->get($family); + $assert(is_object($releasedClaim), + 'Released source generation is incomplete before reallocation.'); + $newClaimId = Capsule::table('cloud_resource_reservation_ips') + ->insertGetId([ + 'reservation_id' => $reallocatedReservationId, + 'address_family' => $family, + 'claim_generation' => 1, + 'active_claim' => 1, + 'network_attachment_id' => 11, + 'public_ip_pool_id' => (int) $releasedClaim->public_ip_pool_id, + 'public_ip_id' => (int) $releasedClaim->public_ip_id, + 'operation_id' => $reallocatedOperation, + 'state' => 'reserved', + 'released_at' => null, + 'release_reason' => '', + 'created_at' => $now, + 'updated_at' => $now, + ]); + $updated = Capsule::table('cloud_public_ips') + ->where('id', (int) $releasedClaim->public_ip_id) + ->where('state', 'available') + ->whereNull('reservation_id') + ->whereNull('reservation_ip_claim_id') + ->whereNull('operation_id') + ->update([ + 'state' => 'reserved', + 'reservation_id' => $reallocatedReservationId, + 'reservation_ip_claim_id' => $newClaimId, + 'instance_id' => null, + 'operation_id' => $reallocatedOperation, + 'updated_at' => $now, + ]); + $assert($updated === 1, + 'Released source address was not legally reallocated.'); + $reallocatedClaims[$family] = [ + 'public_ip_id' => (int) $releasedClaim->public_ip_id, + 'source_claim_id' => (int) $releasedClaim->id, + 'new_claim_id' => $newClaimId, + ]; +} +$reallocatedSet = CloudPublicNetworkAllocationService::currentClaimSet( + $reallocatedReservationId, + 'held' +); +$assert($reallocatedSet->reservationId() === $reallocatedReservationId, + 'Reallocated addresses do not form a valid held dual-stack bundle.'); +CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup); +$assert(Capsule::table('cloud_public_ips') + ->where('reservation_id', $reallocatedReservationId) + ->where('state', 'reserved')->count() === 2 + && Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', $reallocatedReservationId) + ->where('active_claim', 1)->count() === 2, + 'Released replay rejected or changed a legal address reallocation.'); + +$reallocatedIpv4 = $reallocatedClaims[4]; +foreach ([ + 'reservation_id' => $automatic['sourceReservationId'], + 'reservation_ip_claim_id' => $reallocatedIpv4['source_claim_id'], + 'operation_id' => $automatic['sourceOperation'], +] as $field => $staleValue) { + Capsule::table('cloud_public_ips') + ->where('id', $reallocatedIpv4['public_ip_id']) + ->update([$field => $staleValue]); + try { + $rejects( + static fn () => + CloudInstanceMigrationSettlement::releaseSourceLocked( + $migration, + $cleanup + ), + 'Released replay accepted a stale source ' . $field . ' pointer.' + ); + } finally { + $replacement = match ($field) { + 'reservation_id' => $reallocatedReservationId, + 'reservation_ip_claim_id' => $reallocatedIpv4['new_claim_id'], + 'operation_id' => $reallocatedOperation, + }; + Capsule::table('cloud_public_ips') + ->where('id', $reallocatedIpv4['public_ip_id']) + ->update([$field => $replacement]); + } +} +CloudInstanceMigrationSettlement::releaseSourceLocked($migration, $cleanup); -echo "Maintenance migration logical-service and placement finalizer contracts passed.\n"; +echo "Maintenance migration dual-stack source-debt settlement passed.\n"; diff --git a/web/tests/CloudMaintenanceMigrationReceiveContractTest.php b/web/tests/CloudMaintenanceMigrationReceiveContractTest.php index 14f28f91..652f07a1 100644 --- a/web/tests/CloudMaintenanceMigrationReceiveContractTest.php +++ b/web/tests/CloudMaintenanceMigrationReceiveContractTest.php @@ -140,7 +140,9 @@ $assert($transferOutcome !== '' && str_contains($transferOutcome, 'CloudPublicNetworkAllocationService::fenceBundle(') && str_contains($transferOutcome, - 'CloudPublicNetworkAllocationService::commitBundle(') + 'handoffMigrationBundles(') + && str_contains($transferOutcome, + 'CloudInstanceMigrationSettlement::settleCutoverLocked(') && str_contains($transferOutcome, 'verifiedTargetBundle(') && !str_contains($transferOutcome, '$requestedIp'), 'Transfer outcomes are not committed or fenced as one dual-stack bundle.'); diff --git a/web/tests/CloudOperationRecoveryMatrixTest.php b/web/tests/CloudOperationRecoveryMatrixTest.php index 70f7b389..f1cbca68 100644 --- a/web/tests/CloudOperationRecoveryMatrixTest.php +++ b/web/tests/CloudOperationRecoveryMatrixTest.php @@ -39,11 +39,14 @@ namespace { use App\Services\Cloud\CloudAdminOperationRecoveryService; use App\Services\Cloud\CloudAgentError; use App\Services\Cloud\CloudError; + use App\Services\Cloud\CloudFrozenPublicNetworkIdentity; use App\Services\Cloud\CloudInstanceMigrationService; use App\Services\Cloud\CloudOperationCompletionService; use App\Services\Cloud\CloudOperationRecoveryContract; use App\Services\Cloud\CloudOperationRecoveryIdentity; use App\Services\Cloud\CloudOperationRecoveryPolicy; + use App\Services\Cloud\CloudPortableRestoreCompatibility; + use App\Services\Cloud\CloudPublicNetworkAllocationService; use App\Services\Cloud\CloudRecoveryInstanceStateConflict; use App\Services\Cloud\CloudSecrets; use App\Services\Cloud\CloudVpcProjection; @@ -103,6 +106,9 @@ namespace { $capsule->bootEloquent(); $schema = $capsule->schema('default'); + $schema->create('cloud_hosts', static function (Blueprint $table): void { + $table->increments('id'); + }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); @@ -385,8 +391,11 @@ namespace { $schema->create('cloud_resource_reservations', static function (Blueprint $table): void { $table->increments('id'); + $table->string('reservation_key')->nullable(); $table->integer('order_id')->nullable(); $table->integer('backup_restore_id')->nullable(); + $table->string('backup_restore_attempt_key')->nullable(); + $table->integer('migration_source_debt_id')->nullable(); $table->integer('offer_id')->nullable(); $table->integer('host_id'); $table->integer('user_id'); @@ -421,15 +430,36 @@ namespace { $table->increments('id'); $table->integer('host_id'); $table->integer('public_ip_pool_id')->nullable(); + $table->integer('address_family')->nullable(); $table->string('address'); + $table->binary('address_binary')->nullable(); $table->integer('prefix_length'); $table->string('state'); $table->integer('reservation_id')->nullable(); + $table->integer('reservation_ip_claim_id')->nullable(); $table->integer('instance_id')->nullable(); $table->string('operation_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); + $schema->create('cloud_resource_reservation_ips', static function ( + Blueprint $table + ): void { + $table->increments('id'); + $table->integer('reservation_id'); + $table->integer('address_family'); + $table->integer('claim_generation')->default(1); + $table->integer('active_claim')->nullable()->default(1); + $table->integer('network_attachment_id'); + $table->integer('public_ip_pool_id'); + $table->integer('public_ip_id'); + $table->string('operation_id'); + $table->string('state'); + $table->integer('released_at')->nullable(); + $table->string('release_reason')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + }); $schema->create('cloud_resource_pools', static function (Blueprint $table): void { $table->increments('id'); @@ -448,10 +478,42 @@ namespace { $table->integer('host_id'); $table->integer('network_attachment_id'); $table->string('pool_key'); + $table->integer('address_family')->nullable(); $table->string('cidr'); $table->string('gateway'); + $table->boolean('gateway_on_link')->default(0); + $table->integer('guest_prefix_length')->nullable(); $table->text('dns_servers_json'); + $table->text('reserved_ips_json')->default('[]'); + $table->string('allocation_start')->nullable(); + $table->string('allocation_end')->nullable(); + $table->string('allocation_cursor')->nullable(); + $table->integer('max_allocations')->default(0); }); + $schema->create('cloud_instance_migration_source_debts', static function ( + Blueprint $table + ): void { + $table->increments('id'); + $table->string('debt_key'); + $table->integer('migration_id'); + $table->integer('source_reservation_id'); + $table->integer('source_host_id'); + $table->string('source_uuid'); + $table->integer('source_network_attachment_id'); + $table->text('source_identity_json'); + $table->string('source_identity_sha256'); + $table->string('source_claim_set_sha256'); + $table->text('source_public_network_json'); + $table->string('source_public_network_sha256'); + $table->string('cleanup_operation_id')->nullable(); + $table->integer('cleanup_job_id')->nullable(); + $table->string('state'); + $table->integer('active_debt')->nullable(); + $table->string('last_error_code')->default(''); + $table->integer('resolved_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); $schema->create('cloud_instance_firewall_versions', static function (Blueprint $table): void { $table->increments('id'); @@ -574,6 +636,7 @@ namespace { $instanceId = 1; $instanceUuid = '123e4567-e89b-42d3-a456-426614174000'; Capsule::table('user')->insert(['id' => $ownerId]); + Capsule::table('cloud_hosts')->insert(['id' => $hostId]); Capsule::table('cloud_instances')->insert([ 'id' => $instanceId, 'uuid' => $instanceUuid, @@ -882,7 +945,9 @@ namespace { $migrationLineId = 7101; $migrationAttachmentId = 7201; $migrationPublicPoolId = 7301; + $migrationIpv6PublicPoolId = 7306; $migrationNetworkSelector = 'bridge:migration-wan'; + $migrationSourceOperationId = str_repeat('7', 32); Capsule::table('cloud_resource_pools')->insert([ 'id' => $migrationResourcePoolId, 'line_id' => $migrationLineId, @@ -894,13 +959,40 @@ namespace { 'network_selector' => $migrationNetworkSelector, ]); Capsule::table('cloud_public_ip_pools')->insert([ - 'id' => $migrationPublicPoolId, - 'host_id' => $hostId, - 'network_attachment_id' => $migrationAttachmentId, - 'pool_key' => str_repeat('b', 32), - 'cidr' => '203.0.113.0/24', - 'gateway' => '203.0.113.1', - 'dns_servers_json' => '["1.1.1.1"]', + [ + 'id' => $migrationPublicPoolId, + 'host_id' => $hostId, + 'network_attachment_id' => $migrationAttachmentId, + 'pool_key' => str_repeat('b', 32), + 'address_family' => 4, + 'cidr' => '8.8.30.0/24', + 'gateway' => '8.8.30.1', + 'gateway_on_link' => 0, + 'guest_prefix_length' => 24, + 'dns_servers_json' => '["1.1.1.1"]', + 'reserved_ips_json' => '[]', + 'allocation_start' => null, + 'allocation_end' => null, + 'allocation_cursor' => null, + 'max_allocations' => 0, + ], + [ + 'id' => $migrationIpv6PublicPoolId, + 'host_id' => $hostId, + 'network_attachment_id' => $migrationAttachmentId, + 'pool_key' => str_repeat('c', 32), + 'address_family' => 6, + 'cidr' => '2404:c140:156:30::/64', + 'gateway' => '2404:c140:156:30::1', + 'gateway_on_link' => 0, + 'guest_prefix_length' => 64, + 'dns_servers_json' => '["2606:4700:4700::1111"]', + 'reserved_ips_json' => '[]', + 'allocation_start' => '2404:c140:156:30::100', + 'allocation_end' => '2404:c140:156:30::1ff', + 'allocation_cursor' => null, + 'max_allocations' => 100, + ], ]); $migrationArtifact = [ 'id' => $migrationArtifactId, @@ -958,27 +1050,21 @@ namespace { 'memory_mib' => 2048, 'disk_gib' => 20, 'bandwidth_mbps' => 100, - 'network' => $migrationNetworkSelector, + 'public_network' => [], 'template_id' => '', 'template_revision' => 0, 'template_generation_token' => '', 'startup_script' => '', - 'hostname' => 'matrix-vm', + 'hostname' => '', 'guest_ssh_public_key' => '', 'root_password' => '', - 'public_pool_key' => str_repeat('b', 32), - 'public_pool_cidr' => '203.0.113.0/24', - 'public_ip_cidr' => '203.0.113.30/24', - 'gateway' => '203.0.113.1', - 'dns_servers' => ['1.1.1.1'], - 'platform_network_policy_snapshot' => [ - 'schema_version' => 1, - ], + 'platform_network_policy_snapshot' => [], ], 'owner_id' => $ownerId, 'can_manage' => true, 'reservation_id' => 9004, 'maintenance_migration_id' => 1, + 'migration_target' => [], ], $instanceId, null, @@ -1142,36 +1228,207 @@ namespace { 'updated_at' => $now - 20, ]); Capsule::table('cloud_resource_reservations')->insert([ - 'id' => 9004, + [ + 'id' => 9003, + 'reservation_key' => str_repeat('3', 32), + 'backup_restore_id' => null, + 'backup_restore_attempt_key' => null, + 'migration_source_debt_id' => 1, + 'host_id' => $hostId, + 'user_id' => $ownerId, + 'resource_pool_id' => $migrationResourcePoolId, + 'network_attachment_id' => null, + 'public_ip_pool_id' => null, + 'operation_id' => $migrationSourceOperationId, + 'public_ip_id' => null, + 'instance_id' => null, + 'cpu_cores' => 2, + 'memory_mib' => 2048, + 'disk_gib' => 20, + 'state' => 'held', + 'expires_at' => null, + 'released_at' => null, + 'release_reason' => '', + 'created_at' => $now - 300, + 'updated_at' => $now - 20, + ], + [ + 'id' => 9004, + 'reservation_key' => str_repeat('4', 32), + 'backup_restore_id' => 9002, + 'backup_restore_attempt_key' => null, + 'migration_source_debt_id' => null, + 'host_id' => $hostId, + 'user_id' => $ownerId, + 'resource_pool_id' => $migrationResourcePoolId, + 'network_attachment_id' => null, + 'public_ip_pool_id' => null, + 'operation_id' => $migrationTransfer['operation_id'], + 'public_ip_id' => null, + 'instance_id' => null, + 'cpu_cores' => 2, + 'memory_mib' => 2048, + 'disk_gib' => 20, + 'state' => 'uncertain', + 'expires_at' => null, + 'released_at' => null, + 'release_reason' => '', + 'created_at' => $now - 300, + 'updated_at' => $now - 20, + ], + ]); + $insertMigrationClaim = static function ( + int $claimId, + int $ipId, + int $reservationId, + int $family, + int $poolId, + string $address, + string $operationId, + string $state + ) use ($hostId, $migrationAttachmentId, $now): void { + Capsule::table('cloud_public_ips')->insert([ + 'id' => $ipId, + 'host_id' => $hostId, + 'public_ip_pool_id' => $poolId, + 'address_family' => $family, + 'address' => $address, + 'address_binary' => inet_pton($address), + 'prefix_length' => $family === 4 ? 24 : 64, + 'state' => $state === 'reserved' ? 'reserved' : 'uncertain', + 'reservation_id' => $reservationId, + 'reservation_ip_claim_id' => $claimId, + 'instance_id' => null, + 'operation_id' => $operationId, + 'created_at' => $now - 300, + 'updated_at' => $now - 20, + ]); + Capsule::table('cloud_resource_reservation_ips')->insert([ + 'id' => $claimId, + 'reservation_id' => $reservationId, + 'address_family' => $family, + 'claim_generation' => 1, + 'active_claim' => 1, + 'network_attachment_id' => $migrationAttachmentId, + 'public_ip_pool_id' => $poolId, + 'public_ip_id' => $ipId, + 'operation_id' => $operationId, + 'state' => $state, + 'released_at' => null, + 'release_reason' => '', + 'created_at' => $now - 300, + 'updated_at' => $now - 20, + ]); + }; + $insertMigrationClaim( + 9201, + 9007, + 9004, + 4, + $migrationPublicPoolId, + '8.8.30.30', + $migrationTransfer['operation_id'], + 'uncertain' + ); + $insertMigrationClaim( + 9202, + 9008, + 9004, + 6, + $migrationIpv6PublicPoolId, + '2404:c140:156:30::130', + $migrationTransfer['operation_id'], + 'uncertain' + ); + $insertMigrationClaim( + 9203, + 9009, + 9003, + 4, + $migrationPublicPoolId, + '8.8.30.29', + $migrationSourceOperationId, + 'reserved' + ); + $insertMigrationClaim( + 9204, + 9010, + 9003, + 6, + $migrationIpv6PublicPoolId, + '2404:c140:156:30::129', + $migrationSourceOperationId, + 'reserved' + ); + $migrationTargetClaims = + CloudPublicNetworkAllocationService::currentClaimSet( + 9004, + 'uncertain' + ); + $migrationTargetIdentity = CloudFrozenPublicNetworkIdentity::fromClaimSet( + $migrationTargetUuid, + $migrationTargetClaims + ); + $migrationSourceClaims = + CloudPublicNetworkAllocationService::currentClaimSet(9003, 'held'); + $migrationSourceIdentity = CloudFrozenPublicNetworkIdentity::fromClaimSet( + $instanceUuid, + $migrationSourceClaims + ); + $migrationTransfer['payload']['body']['public_network'] = + $migrationTargetIdentity->publicNetwork()->toArray(); + $migrationTransfer['payload']['migration_target'] = [ + 'contract' => 1, 'backup_restore_id' => 9002, + 'reservation_id' => 9004, 'host_id' => $hostId, - 'user_id' => $ownerId, 'resource_pool_id' => $migrationResourcePoolId, - 'network_attachment_id' => $migrationAttachmentId, - 'public_ip_pool_id' => $migrationPublicPoolId, - 'operation_id' => $migrationTransfer['operation_id'], - 'public_ip_id' => 9007, - 'instance_id' => $instanceId, - 'cpu_cores' => 2, - 'memory_mib' => 2048, - 'disk_gib' => 20, - 'state' => 'uncertain', - 'expires_at' => null, - 'released_at' => null, - 'release_reason' => '', - 'created_at' => $now - 300, - 'updated_at' => $now - 20, - ]); - Capsule::table('cloud_public_ips')->insert([ - 'id' => 9007, - 'host_id' => $hostId, - 'public_ip_pool_id' => $migrationPublicPoolId, - 'address' => '203.0.113.30', - 'prefix_length' => 24, - 'state' => 'uncertain', - 'reservation_id' => 9004, - 'instance_id' => null, 'operation_id' => $migrationTransfer['operation_id'], + 'target_uuid' => strtolower($migrationTargetUuid), + 'target_agent_revision' => 3, + 'target_agent_contract_sha256' => + CloudPortableRestoreCompatibility::supported()->sha256(), + 'target_identity_json' => $migrationTargetIdentity->identityJson(), + 'target_identity_sha256' => + $migrationTargetIdentity->identitySha256(), + 'target_claim_set_sha256' => + $migrationTargetIdentity->claimSetSha256(), + 'target_public_network_json' => + $migrationTargetIdentity->publicNetworkJson(), + 'target_public_network_sha256' => + $migrationTargetIdentity->publicNetworkSha256(), + 'vpc_port' => null, + ]; + $migrationTransferPayload = CloudSecrets::encrypt( + $migrationTransfer['payload'], + 'job-payload' + ); + Capsule::table('cloud_jobs')->where('id', $migrationTransfer['id']) + ->update(['sealed_payload' => $migrationTransferPayload]); + $migrationTransfer['job']->sealed_payload = $migrationTransferPayload; + Capsule::table('cloud_instance_migration_source_debts')->insert([ + 'id' => 1, + 'debt_key' => str_repeat('5', 32), + 'migration_id' => 1, + 'source_reservation_id' => 9003, + 'source_host_id' => $hostId, + 'source_uuid' => strtolower($instanceUuid), + 'source_network_attachment_id' => $migrationAttachmentId, + 'source_identity_json' => $migrationSourceIdentity->identityJson(), + 'source_identity_sha256' => + $migrationSourceIdentity->identitySha256(), + 'source_claim_set_sha256' => + $migrationSourceIdentity->claimSetSha256(), + 'source_public_network_json' => + $migrationSourceIdentity->publicNetworkJson(), + 'source_public_network_sha256' => + $migrationSourceIdentity->publicNetworkSha256(), + 'cleanup_operation_id' => $migrationCleanup['operation_id'], + 'cleanup_job_id' => $migrationCleanup['id'], + 'state' => 'attention', + 'active_debt' => 1, + 'last_error_code' => 'operation_effect_unknown', + 'resolved_at' => null, 'created_at' => $now - 300, 'updated_at' => $now - 20, ]); @@ -3325,6 +3582,24 @@ namespace { 'uuid' => $instanceUuid, 'reservation_id' => 9003, ]); + Capsule::table('cloud_instance_migration_source_debts') + ->where('migration_id', 1)->delete(); + Capsule::table('cloud_resource_reservations')->where('id', 9003)->update([ + 'migration_source_debt_id' => null, + 'instance_id' => $instanceId, + 'state' => 'committed', + 'updated_at' => $now, + ]); + Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', 9003)->update([ + 'state' => 'committed', + 'updated_at' => $now, + ]); + Capsule::table('cloud_public_ips')->where('reservation_id', 9003)->update([ + 'state' => 'assigned', + 'instance_id' => $instanceId, + 'updated_at' => $now, + ]); Capsule::table('cloud_instance_migrations')->where('id', 1)->update([ 'target_receive_job_id' => null, 'firewall_job_id' => null, @@ -3352,15 +3627,20 @@ namespace { ]); Capsule::table('cloud_resource_reservations')->where('id', 9004)->update([ 'operation_id' => $abortTargetOperation, - 'instance_id' => $instanceId, + 'instance_id' => null, 'state' => 'held', 'released_at' => null, 'release_reason' => '', 'updated_at' => $now, ]); - Capsule::table('cloud_public_ips')->where('id', 9007)->update([ + Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', 9004)->update([ + 'state' => 'reserved', + 'operation_id' => $abortTargetOperation, + 'updated_at' => $now, + ]); + Capsule::table('cloud_public_ips')->where('reservation_id', 9004)->update([ 'state' => 'reserved', - 'reservation_id' => 9004, 'instance_id' => null, 'operation_id' => $abortTargetOperation, 'updated_at' => $now, @@ -3565,7 +3845,8 @@ namespace { . '/peer-migration/receive', 'body' => [ 'operation_id' => $operationId, - 'public_ip_cidr' => '203.0.113.30/24', + 'public_network' => + $migrationTargetIdentity->publicNetwork()->toArray(), ], 'owner_id' => $ownerId, 'can_manage' => true, @@ -3635,11 +3916,17 @@ namespace { ->update([ 'backup_restore_id' => 9012, 'operation_id' => $residueTransfer['operation_id'], - 'instance_id' => $instanceId, + 'instance_id' => null, + 'state' => 'uncertain', + 'updated_at' => $now, + ]); + Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', 9004)->update([ 'state' => 'uncertain', + 'operation_id' => $residueTransfer['operation_id'], 'updated_at' => $now, ]); - Capsule::table('cloud_public_ips')->where('id', 9007)->update([ + Capsule::table('cloud_public_ips')->where('reservation_id', 9004)->update([ 'state' => 'uncertain', 'instance_id' => null, 'operation_id' => $residueTransfer['operation_id'], diff --git a/web/tests/CloudPortableBackupAdminUiContractTest.php b/web/tests/CloudPortableBackupAdminUiContractTest.php index b34d5e55..3b1110b1 100644 --- a/web/tests/CloudPortableBackupAdminUiContractTest.php +++ b/web/tests/CloudPortableBackupAdminUiContractTest.php @@ -73,14 +73,16 @@ $assert(str_contains($runtime, 'export function createAdminInstanceMigration(') && str_contains($runtime, '原账单、到期时间和流量周期') && str_contains( $migrationSettlement, - "'maintenance_migration_transferred'" + "'maintenance_migration_source_removed'" ) + && str_contains($migrationSettlement, 'settleCutoverLocked(') + && str_contains($migrationSettlement, 'migration_source_debt_id') && str_contains($migration, "'traffic_counter_generation'") && str_contains($migration, "'reconcile_cleanup'") && str_contains($migration, "'managed_artifacts_absent'") - && str_contains($migration, 'reserveBackupRestore(') + && str_contains($migration, 'reserveMaintenanceMigrationTarget(') && str_contains($migration, "'target_resource_pool_id'") - && str_contains($migration, 'preparedBackupRestorePlacement(') + && str_contains($migration, 'preparedMaintenanceMigrationTarget(') && str_contains($migration, '$restore->target_instance_id = null;') && str_contains($jobService, 'CloudInstanceMigrationService::assertInstanceMutationAvailable(') @@ -110,7 +112,7 @@ $assert(str_contains($peerArtifacts, && str_contains($migration, 'queueActivationLocked(') && strpos($migration, 'queueMaintenanceMigrationReapplyLocked(') < strpos($migration, 'queueActivationLocked(') - && str_contains($migration, "\$spec['guest_hostname'] ?? \$instance->name") + && str_contains($migration, "'hostname' => ''") && str_contains($migrationWorkflow, "'label' => '恢复目标电源状态'") && str_contains($migrationWorkflow, "'label' => '重试目标状态恢复'") && str_contains($migrationWorkflow, diff --git a/web/tests/CloudPortableBackupRecoveryEnvelopeTest.php b/web/tests/CloudPortableBackupRecoveryEnvelopeTest.php index 6742b7c0..30c07ec8 100644 --- a/web/tests/CloudPortableBackupRecoveryEnvelopeTest.php +++ b/web/tests/CloudPortableBackupRecoveryEnvelopeTest.php @@ -102,4 +102,75 @@ $assert(($restoreCompact['body'] ?? null) === [ && !str_contains($encodedRestore, 'X-Amz-Signature'), 'Prepared restore evidence retained an expired capability or secret.'); +$publicNetwork = [ + 'attachment_key' => str_repeat('3', 32), + 'network' => 'bridge:br0', + 'addresses' => [ + [ + 'address_family' => 4, + 'address_cidr' => '203.0.113.29/24', + 'gateway' => '203.0.113.1', + 'pool_key' => str_repeat('4', 32), + 'dns_servers' => ['9.9.9.9'], + ], + [ + 'address_family' => 6, + 'address_cidr' => '2001:db8:100::29/64', + 'gateway' => '2001:db8:100::1', + 'pool_key' => str_repeat('5', 32), + 'dns_servers' => ['2620:fe::fe'], + ], + ], +]; +$vpcPort = [ + 'vpc_key' => str_repeat('6', 32), + 'address_cidr' => '10.0.0.29/24', + 'mac_address' => '02:00:00:00:00:29', +]; +$migrationPayload = [ + $identityKey => array_replace( + $identity, + ['kind' => 'instance_migration_transfer'] + ), + 'owner_id' => 42, + 'can_manage' => true, + 'reservation_id' => 92, + 'maintenance_migration_id' => 93, + 'body' => [ + 'vcpu' => 8, + 'memory_mib' => 16384, + 'disk_gib' => 160, + 'bandwidth_mbps' => 1000, + 'public_network' => $publicNetwork, + 'vpc_port' => $vpcPort, + 'public_ip_cidr' => '203.0.113.29/24', + 'gateway' => '203.0.113.1', + 'source_url' => $largeUrls[0], + 'root_password' => 'must-not-survive', + ], +]; +$migrationCompact = $compact->invoke( + null, + 'instance_migration_transfer', + $migrationPayload +); +$encodedMigration = json_encode( + $migrationCompact, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES +); +$assert(($migrationCompact['body'] ?? null) === [ + 'vcpu' => 8, + 'memory_mib' => 16384, + 'disk_gib' => 160, + 'bandwidth_mbps' => 1000, + 'public_network' => $publicNetwork, + 'vpc_port' => $vpcPort, + ] + && !str_contains($encodedMigration, 'public_ip_cidr') + && !str_contains($encodedMigration, 'source_url') + && !str_contains($encodedMigration, 'root_password') + && !str_contains($encodedMigration, 'X-Amz-Signature'), + 'Prepared migration evidence lost canonical dual-stack state or retained' + . ' a retired scalar field, capability, or secret.'); + echo "Portable-backup prepared recovery envelope contract passed.\n"; diff --git a/web/tests/CloudPublicNetworkLegacyWriterGateContractTest.php b/web/tests/CloudPublicNetworkLegacyWriterGateContractTest.php index 0648073d..94428d90 100644 --- a/web/tests/CloudPublicNetworkLegacyWriterGateContractTest.php +++ b/web/tests/CloudPublicNetworkLegacyWriterGateContractTest.php @@ -100,7 +100,8 @@ $deletedInstance = $method( $assert(str_contains($gate, 'assertWriterReady') && str_contains($gate, 'assertPortableRestoreReady') - && str_contains($gate, 'assertMigrationReady') + && str_contains($gate, 'assertLegacyMigrationWriterRetired') + && !str_contains($gate, 'assertMigrationReady') && str_contains($gate, 'assertInstanceDeleteReady'), 'Legacy-writer gate does not cover every pre-ledger writer.'); $enqueueStart = strpos($jobs, 'public static function enqueue('); @@ -128,7 +129,9 @@ $assert($before($worker, && $before($worker, "(string) \$job->kind === 'backup_restore'", "'phase' => 'dispatching'") - && $before($worker, "'instance_migration_'", "'phase' => 'dispatching'"), + && $before($worker, + 'CloudOperationRecoveryIdentity::expectedFromPayload(', + "'phase' => 'dispatching'"), 'Worker defense runs after an old operation may be dispatched.'); $restoreDispatchGate = strpos( $worker, @@ -165,27 +168,27 @@ $assert(!str_contains($backupRestore, && !str_contains($backupRestore, 'new CloudBackupRestore()') && !str_contains($backupRestore, 'reserveBackupRestore('), 'Portable-restore request seam retained the retired writer or closed gate.'); -$assert($before($migration, - 'CloudPublicNetworkTransitionGate::assertMigrationReady()', - 'new CloudBackupRestore()'), - 'Migration gate runs after target rows may be created.'); +$assert(!str_contains($migration, + 'CloudPublicNetworkTransitionGate::') + && str_contains($migration, 'reserveMaintenanceMigrationTarget('), + 'Canonical dual-stack migration start is still behind a legacy gate.'); $assert(str_contains($placement, 'self::assertBackupRestoreTransitionReady($restore)') && str_contains($placementGate, - 'CloudPublicNetworkTransitionGate::assertMigrationReady()') + 'assertLegacyMigrationWriterRetired()') && str_contains($placementGate, 'CloudPublicNetworkTransitionGate::assertPortableRestoreReady()'), 'Placement lacks a defense-in-depth gate for old restore writers.'); -$assert($before($migrationEnsureCleanup, - 'CloudPublicNetworkTransitionGate::assertMigrationReady()', - 'getConnection()->transaction') - && $before($migrationRecover, - 'CloudPublicNetworkTransitionGate::assertMigrationReady()', - 'getConnection()->transaction') - && $before($migrationCleanup, - 'CloudPublicNetworkTransitionGate::assertMigrationReady()', - 'getConnection()->transaction'), - 'A migration recovery entry can create retry intent before its gate.'); +$assert(!str_contains($migrationEnsureCleanup, + 'CloudPublicNetworkTransitionGate::') + && !str_contains($migrationRecover, + 'CloudPublicNetworkTransitionGate::') + && !str_contains($migrationCleanup, + 'CloudPublicNetworkTransitionGate::') + && str_contains($migrationEnsureCleanup, 'getConnection()->transaction') + && str_contains($migrationRecover, 'getConnection()->transaction') + && str_contains($migrationCleanup, 'getConnection()->transaction'), + 'Canonical migration recovery is still gated or lost its transaction.'); $assert($before($networkFinalizer, 'CloudPublicNetworkTransitionGate::assertWriterReady()', 'lockForUpdate()'), diff --git a/web/tests/CloudRestoreMigrationLedgerContractTest.php b/web/tests/CloudRestoreMigrationLedgerContractTest.php index 4ab4094a..4ff947e3 100644 --- a/web/tests/CloudRestoreMigrationLedgerContractTest.php +++ b/web/tests/CloudRestoreMigrationLedgerContractTest.php @@ -167,7 +167,7 @@ $assert(str_contains($debtShape, && str_contains($debtShape, "state = 'cleanup_running' AND active_debt IS NOT NULL AND active_debt = 1 AND cleanup_operation_id IS NOT NULL AND cleanup_job_id IS NOT NULL AND resolved_at IS NULL") && str_contains($debtShape, - "state = 'attention' AND active_debt IS NOT NULL AND active_debt = 1 AND resolved_at IS NULL") + "state = 'attention' AND active_debt IS NOT NULL AND active_debt = 1 AND cleanup_operation_id IS NOT NULL AND cleanup_job_id IS NOT NULL AND resolved_at IS NULL") && str_contains($debtShape, "state = 'released' AND active_debt IS NULL AND cleanup_operation_id IS NOT NULL AND cleanup_job_id IS NOT NULL AND resolved_at IS NOT NULL") && str_contains($debtShape,