diff --git a/docs/INTEGRATION_PLAN.md b/docs/INTEGRATION_PLAN.md index 5bca657d..e36be8a0 100644 --- a/docs/INTEGRATION_PLAN.md +++ b/docs/INTEGRATION_PLAN.md @@ -189,21 +189,41 @@ Debian 13 ifupdown、nft 语法、重启、防火墙重载、跨宿主机握手 保留 PHP tombstone,直到对应 VM `private0` 移除和宿主机资源回读完成。单台 VM 首版只允许加入一个 VPC,与当前单一私网角色和防止跨租户混网的边界一致。 -本阶段没有 VPC 根删除接口。CIDR reservation 在租户私网数据面、引擎删除 -tombstone 和宿主机回读闭环之前不会释放;这会保留地址以避免未知状态下复用, -不能把 allocator 的 `active` 误认为已经可以回收。后续删除流程必须先阻断新成员 -变更、确认所有成员/路由/ACL 已收敛,再以独立 revision 和 CAS 释放 reservation。 -删除中的宿主机、私网 NIC、地址、子网或成员 tombstone 仍会关闭 `usable`;这些 -对象没有专门的删除回读证据时,不能在只剩其余宿主机 active 时提前放行。Route/ACL -tombstone 属于已纳入每台宿主机精确路由、nft 和 revision/digest 回读的对象,只有所有 -仍存活的宿主机逐一匹配后才会收敛为 `deleted`。空 VPC 也不会进入 active,必须先有 -至少一个成员宿主机通过 Fabric 门禁。 +VPC 根删除使用独立的 `DELETE /api/v1/vpcs/{vpc_key}` 状态机。PHP 先在 VPC 行锁内 +阻断全部新变更,把根和仍存活的子资源推进到 `deleting`,再建立不可被新操作替换的 +`vpc_delete` revision。删除目标除空的子网、成员、route 和 ACL 集合外,还包含摘要 +覆盖的 `retiring_members` 闭集(member ID、实例 UUID、宿主机、私网 IP 和 MAC); +该清单覆盖“宿主机/libvirt 已变更、但旧 apply 尚未写回引擎持久记录”的中断窗口。 +引擎先持久化删除 marker,再逐成员移除并回读持久化 `private0` XML 与 cloud-init seed, +逐宿主机移除 bridge、route、WireGuard tenant AllowedIPs、nft 和 sysctl 状态。只有所有 +成员与宿主机都得到精确不存在性证明后,引擎才先写 durable receipt、再清理 desired +记录;丢失 HTTP 响应时只能用原 operation/revision/digest 重取同一回执。终态回执只 +保留操作身份、摘要、宿主机闭集和完成时间,不永久保留成员 UUID、私网 IP 或 MAC; +旧格式回执在首次读取或引擎恢复时会压缩为同一最小 tombstone。 + +PHP 只接受与当前删除操作和摘要一致、且覆盖删除 payload 全部待清宿主机的回执;回执 +不得包含 PHP 从未部署过的宿主机,已由历史精确证明为不存在的宿主机则允许重复出现或 +省略。PHP 在同一数据库事务中核对 CIDR interval 与全部 /24 claim 后,以 revision CAS +释放 reservation;随后才把 +根置为 `deleted` 并清除已完成的子资源账本。任何宿主机、成员 XML、回执、CIDR 或 claim +证据缺失都会保留删除 tombstone,不会提前复用地址。运行中的 VM、managed-save 或原生 +快照会让 private0 移除 fail-closed,处理对应实例状态后通过显式 reconcile 重试原删除 +操作。根处于 `deleting` 时,VM 生命周期回读只能补充证据,不能越过根 finalizer 单独 +终结成员账本。删除完成后该根不再阻止 Fabric identity/revision 变更;账户删除只会清除 +`deleted` 根及已 `released` 且无残留 claim 的网络历史。 + +普通成员/route/ACL 删除仍使用各自 tombstone。删除中的宿主机、私网 NIC、地址、子网或 +成员会关闭 `usable`;这些对象没有专门的删除回读证据时,不能在只剩其余宿主机 active +时提前放行。Route/ACL tombstone 属于已纳入每台宿主机精确路由、nft 和 +revision/digest 回读的对象,只有所有仍存活的宿主机逐一匹配后才会收敛为 `deleted`。 +空 VPC 也不会进入 active,必须先有至少一个成员宿主机通过 Fabric 门禁。 VM 删除与 VPC 成员是两个独立状态机:仍有成员(包括 `deleting` tombstone)时, 手动删除和到期永久删除都会 fail-closed;到期实例进入 `delete_failed/purge_failed`, 等待管理员先完成成员及其引用 route 的 VPC 清理和后续数据面回读。删除 worker 不会在 -引擎已返回成功后把 tombstone 伪造为 `deleted`。账户删除同样拒绝遗留 VPC 或私网 -CIDR reservation,避免独立 VPC worker 继续消费孤儿租户计划。 +引擎已返回成功后把 tombstone 伪造为 `deleted`。账户删除同样拒绝活动或删除未完成的 +VPC,以及未释放或仍有 claim 的私网 reservation,避免独立 VPC worker 继续消费孤儿 +租户计划。 AlmaLinux 10 与 Debian 13 的 WireGuard/NetworkManager/ifupdown 能力、真实宿主机 路由、SELinux/firewalld、MTU 和跨机房连通性尚未在本地环境宣称通过;必须完成 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 6c3b1996..14cf8ad5 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,38 @@ # 本版验证记录 +## 2026-10-08:根 VPC 可恢复删除状态机 + +- 租户 API 新增根 VPC 删除:精确名称确认后,PHP 在事务中把根和子资源固定为 + `deleting`,签署空 desired target、待清宿主机闭集及 `retiring_members` 身份清单。 + 普通变更不能覆盖该 revision;失败或响应丢失只能用原 operation/revision/digest + 显式核对。 +- 引擎在任何 SSH/libvirt 变更前持久化删除 marker,逐成员核对并移除持久化 + `private0`、私网 metadata 与 cloud-init seed,再清理宿主机 VPC 数据面。部分宿主机 + 已确认时,重试只联系尚无精确不存在性证明的宿主机;完整 receipt 必须先持久化, + 随后才能清理引擎 desired 记录。终态 receipt 只保留重放所需的操作身份、摘要、 + 宿主机闭集和时间戳,不永久保存成员 UUID、私网 IP 或 MAC;旧格式会自动压缩。 +- PHP finalizer 严格核对 payload、receipt、历史/待清宿主机集合和子账本,然后验证 + CIDR interval 与全部 /24 claim,最后用 revision CAS 释放地址。live allocation 的 + 幂等重放也会重新证明 claim 闭集;畸形 VPC 核心/子表、残留 claim 或未完成根删除 + 均会阻止账户删除。只有完成删除的子账本会被清理,以释放全局 member 唯一键;根、 + 宿主机和 operation 保留为审计 tombstone。 +- 前端只有详情接口明确返回 `state=deleted` 才结束删除提示;分页列表缺失或 404 不会被 + 当作成功。`deleting` 期间锁定所有普通编辑,仅保留“核对 / 重试”。 +- 直接相关 Python 模块回归:**429 passed、56 subtests passed**;覆盖 VPC 协调器、 + 数据面脚本、内部 API 和 libvirt 持久 XML/metadata/seed 清理。完整 engine 回归为 + **1007 passed、10 skipped、80 subtests passed、5 failed**;5 个失败在未修改的 + `7f9fb9c` 干净快照逐项复现,分别属于既有云模板 spec、重装错误结构和公网 IP 池 + fixture,不是本次删除状态机引入。 +- 隔离 PHP 8.5/WASM 下 `CloudVpcRootDeletionTest`、 + `CloudPrivateCidrAllocatorTest`、`CloudVpcControlPlaneContractTest`、 + `CloudVpcUserPageContractTest`、`ManualAccountDeletionSqliteTest` 和 + `ManualAccountDeletionTest` 全部通过;本轮 11 个 PHP 文件语法、274 条路由目标、 + `vpc.js` Node 语法、Python `compileall` 与 `git diff --check` 通过。 +- 当前没有连接真实 MariaDB、SSH/libvirt、nft、NetworkManager 或 ifupdown 宿主机。 + 生产验收必须覆盖多宿主机部分失败/重启续跑、HTTP 回执丢失、真实 seed 重写和数据面 + 移除回读。含私网成员的根删除要求相关 VM 已停机;managed-save 或原生快照存在时会 + 保留 tombstone 并拒绝继续,处理阻断后再显式 reconcile。 + ## 2026-10-07:在线更新后部署页 SQL 回归修复 - 在线更新到 `521d40a` 后,`/user/cloud/deploy` 在读取销售目录时会因 `cloud_hosts` 不存在 `host` 列而返回 500;同一错误也影响管理员资源汇总。根因已在 `e7230ff` 修复,应用查询现在只使用面板资源策略、reservation 和已物化的公网 IP ledger。 diff --git a/engine/tests/fixtures/vpc_delete_digest.json b/engine/tests/fixtures/vpc_delete_digest.json new file mode 100644 index 00000000..9f679c90 --- /dev/null +++ b/engine/tests/fixtures/vpc_delete_digest.json @@ -0,0 +1,36 @@ +{ + "document": { + "schema_version": 1, + "vpc_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "region": "test-region", + "fabric_id": "test-fabric", + "fabric_revision": 3, + "cidr": "10.20.0.0/24", + "desired_revision": 10, + "operation_id": "vpc-delete-digest-fixture-op", + "subnets": [], + "hosts": [ + { + "host_id": "11", + "members": [] + }, + { + "host_id": "2", + "members": [] + } + ], + "members": [], + "routes": [], + "acls": [], + "retiring_members": [ + { + "member_id": "11111111111111111111111111111111", + "instance_uuid": "11111111-2222-4333-8444-555555555555", + "host_id": "2", + "private_ip_cidr": "10.20.0.10/32", + "private_mac": "12:e8:3c:ae:d0:18" + } + ] + }, + "expected_digest": "2f83e2fe639c82702a21c200461cd4c59a57030ec7ba9603e949c5175de73314" +} diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index 33d7baac..9d0644f4 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -37,6 +37,7 @@ from virsh_panel.libvirt_service import ( ) from virsh_panel.metrics import host_scope from virsh_panel.user_isos import user_iso_scope +from virsh_panel.vpc import parse_vpc_delete_plan SECRET = b"s" * 32 @@ -384,6 +385,55 @@ def test_vpc_control_plane_is_global_and_does_not_use_host_network_change(bounda assert not store.list("operation") +def test_vpc_root_delete_endpoint_returns_exact_durable_receipt(boundary): + client, _engine, store, _logs, _services = boundary + target = { + "schema_version": 1, + "vpc_id": "vpc-empty-delete", + "region": "hk", + "fabric_id": "fabric-boundary", + "fabric_revision": 1, + "cidr": "10.31.0.0/24", + "desired_revision": 3, + "operation_id": "vpc-empty-delete-op-0001", + "subnets": [], + "hosts": [], + "members": [], + "routes": [], + "acls": [], + "retiring_members": [], + } + target["desired_digest"] = parse_vpc_delete_plan(target)[0].desired_digest + path = "/internal/v1/vpcs/vpc-empty-delete/delete" + deleted = call( + client, "POST", path, {**USER, "can_manage": False}, target) + assert deleted.status_code == 200, deleted.text + receipt = deleted.json() + assert receipt == { + "vpc_id": "vpc-empty-delete", + "operation_id": "vpc-empty-delete-op-0001", + "desired_revision": 3, + "desired_digest": target["desired_digest"], + "state": "deleted", + "deleted": True, + "readback": "confirmed", + "removed_host_ids": [], + "completed_at": receipt["completed_at"], + } + assert receipt["completed_at"] > 0 + assert call(client, "POST", path, SUPER, target).json() == receipt + assert store.get("vpc_deleted", "vpc-empty-delete") is not None + assert store.get("vpc_desired", "vpc-empty-delete") is None + + missing_manifest = dict(target) + missing_manifest.pop("retiring_members") + invalid = call( + client, "POST", "/internal/v1/vpcs/missing-manifest/delete", + SUPER, {**missing_manifest, "vpc_id": "missing-manifest"}) + assert invalid.status_code == 422 + assert invalid.json()["error_code"] == "vpc_delete_manifest_invalid" + + def test_host_public_network_pools_are_revisioned_idempotent_and_aggregated( boundary): diff --git a/engine/tests/test_libvirt_service.py b/engine/tests/test_libvirt_service.py index afdbda64..0d77d6a7 100644 --- a/engine/tests/test_libvirt_service.py +++ b/engine/tests/test_libvirt_service.py @@ -2699,6 +2699,303 @@ class VMServiceTests(unittest.TestCase): seed.capacity = len(seed_data) return service, created, spec, domain, seed_name + def cloud_vm_with_private_artifacts(self, *, include_private_nic): + from virsh_panel.cloud_init import replace_seed_private_network + + service, created, spec, domain, seed_name = ( + self.cloud_vm_with_real_seed()) + private_mac = "52:54:00:12:34:57" + private_ip_cidr = "10.20.0.10/32" + bridge = "vcbr-private" + managed = domain.inactive_root.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed") + managed.attrib.update({ + "private-vpc-id": "vpc-delete-test", + "private-member-id": "member-delete-test", + "private-ip-cidr": private_ip_cidr, + "private-network-bridge": bridge, + "private-network-verified": "true", + "private-dataplane-owner": "virt-cloud-vpc-dataplane-v1", + "private-network-revision": "1", + "private-network-digest": "a" * 64, + }) + if include_private_nic: + domain.inactive_root.find("./devices").append(ET.fromstring( + "" + f"" + f"" + "" + "" + )) + seed = self.backend.pools["images"].volumes[seed_name] + replacement = replace_seed_private_network( + bytes(seed.contents), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=["10.20.0.11/32"], + ) + seed.contents = bytearray(replacement) + seed.capacity = len(replacement) + domain.running = False + domain.live_root = None + resources = types.SimpleNamespace(bridge=bridge) + return (service, created, domain, seed, resources, + private_mac, private_ip_cidr) + + def test_remove_private_interface_repairs_absent_nic_stale_metadata_and_seed(self): + from virsh_panel.cloud_init import read_seed_network + + (service, created, domain, seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=False) + + result = service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + + self.assertTrue(result["removed"]) + self.assertFalse(result["already_absent"]) + self.assertEqual(result["readback"], "confirmed") + self.assertIsNone(self.service._private_interface_from_root( + domain.inactive_root)) + managed = domain.inactive_root.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed") + self.assertFalse(any( + key.startswith("private-") for key in managed.attrib)) + observed_seed = read_seed_network(bytes(seed.contents)) + self.assertNotIn("private_ip_cidr", observed_seed) + self.assertNotIn("private_mac", observed_seed) + self.assertNotIn("private_routes", observed_seed) + + def test_remove_private_interface_retries_define_committed_seed_stale(self): + from virsh_panel.cloud_init import read_seed_network + + (service, created, domain, seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + original_define = FakeConnection.defineXML + + def commit_then_lose_response(connection, xml): + original_define(connection, xml) + raise FakeLibvirtError("define response lost") + + with patch.object( + FakeConnection, "defineXML", commit_then_lose_response): + with self.assertRaises(self.error_type) as uncertain: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + self.assertEqual( + uncertain.exception.error_code, "vpc_private_remove_uncertain") + self.assertIsNone(self.service._private_interface_from_root( + domain.inactive_root)) + self.assertIn( + "private_ip_cidr", read_seed_network(bytes(seed.contents))) + + result = service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + self.assertTrue(result["removed"]) + self.assertFalse(result["already_absent"]) + self.assertNotIn( + "private_ip_cidr", read_seed_network(bytes(seed.contents))) + + def test_remove_private_interface_rejects_unowned_stale_seed_identity(self): + from virsh_panel.cloud_init import read_seed_network + + (service, created, domain, seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=False) + managed = domain.inactive_root.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed") + for key in tuple(managed.attrib): + if key.startswith("private-"): + managed.attrib.pop(key) + + with self.assertRaises(self.error_type) as conflict: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr="10.20.0.99/32", + ) + self.assertEqual( + conflict.exception.error_code, "vpc_private_interface_conflict") + self.assertEqual( + read_seed_network(bytes(seed.contents))["private_ip_cidr"], + private_ip_cidr, + ) + + def test_remove_private_interface_rejects_nic_owner_or_mac_mismatch(self): + (service, created, domain, seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + managed = domain.inactive_root.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed") + private = self.service._private_interface_from_root( + domain.inactive_root) + old_seed = bytes(seed.contents) + + managed.set("private-vpc-id", "vpc-foreign") + with self.assertRaises(self.error_type) as wrong_owner: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + self.assertEqual( + wrong_owner.exception.error_code, + "vpc_private_interface_conflict", + ) + managed.set("private-vpc-id", "vpc-delete-test") + + private.find("mac").set("address", "52:54:00:12:34:59") + with self.assertRaises(self.error_type) as wrong_mac: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + self.assertEqual( + wrong_mac.exception.error_code, + "vpc_private_interface_conflict", + ) + self.assertEqual(bytes(seed.contents), old_seed) + + def test_remove_private_interface_confirms_all_layers_already_absent(self): + service, created, _spec, domain, _seed_name = ( + self.cloud_vm_with_real_seed()) + domain.running = False + domain.live_root = None + resources = types.SimpleNamespace(bridge="vcbr-private") + + with patch.object( + service, "_download_seed_volume", + wraps=service._download_seed_volume) as download: + result = service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac="52:54:00:12:34:57", + expected_private_ip_cidr="10.20.0.10/32", + ) + + self.assertTrue(result["removed"]) + self.assertTrue(result["already_absent"]) + self.assertEqual(result["readback"], "confirmed") + # One preflight read plus one post-operation exact absence readback. + self.assertEqual(download.call_count, 2) + + def test_remove_private_interface_accepts_missing_owned_seed_only_when_xml_clean(self): + service = self.cloud_service() + created = self.create_cloud_vm(service, self.cloud_spec()) + domain = self.backend.domains[0] + domain.running = False + domain.live_root = None + managed = domain.inactive_root.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed") + managed.attrib.pop("seed-volume") + devices = domain.inactive_root.find("./devices") + for cdrom in tuple( + devices.findall("./disk[@device='cdrom']")): + devices.remove(cdrom) + + result = service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=types.SimpleNamespace(bridge="vcbr-private"), + desired_revision=2, desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac="52:54:00:12:34:57", + expected_private_ip_cidr="10.20.0.10/32", + ) + + self.assertTrue(result["removed"]) + self.assertTrue(result["already_absent"]) + self.assertEqual(result["readback"], "confirmed") + + def test_remove_private_interface_rejects_unowned_cdrom_as_absence_proof(self): + service = self.cloud_service() + created = self.create_cloud_vm(service, self.cloud_spec()) + domain = self.backend.domains[0] + domain.running = False + domain.live_root = None + managed = domain.inactive_root.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed") + managed.attrib.pop("seed-volume") + + with self.assertRaises(self.error_type) as missing: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=types.SimpleNamespace(bridge="vcbr-private"), + desired_revision=2, desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac="52:54:00:12:34:57", + expected_private_ip_cidr="10.20.0.10/32", + ) + + self.assertEqual(missing.exception.error_code, "vpc_seed_missing") + + def test_remove_private_interface_rolls_back_define_when_seed_upload_fails(self): + (service, created, domain, seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + old_xml = domain.XMLDesc(1) + old_seed = bytes(seed.contents) + original_upload = service._upload_volume + attempts = 0 + + def fail_first_upload(connection, volume, chunks, size): + nonlocal attempts + attempts += 1 + if attempts == 1: + volume.contents = bytearray(b"partial-seed-write") + raise FakeLibvirtError("seed upload failed") + return original_upload(connection, volume, chunks, size) + + with patch.object( + service, "_upload_volume", + side_effect=fail_first_upload): + with self.assertRaises(self.error_type) as uncertain: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + + self.assertEqual( + uncertain.exception.error_code, + "vpc_private_remove_uncertain", + ) + self.assertEqual(attempts, 2) + self.assertEqual(domain.XMLDesc(1), old_xml) + self.assertEqual(bytes(seed.contents), old_seed) + def test_create_list_resources_and_console(self): resources = self.service.resources() self.assertEqual(resources["networks"], ["default"]) diff --git a/engine/tests/test_vpc.py b/engine/tests/test_vpc.py index ce60a3f4..6e770a09 100644 --- a/engine/tests/test_vpc.py +++ b/engine/tests/test_vpc.py @@ -5,16 +5,19 @@ import asyncio import json import threading from contextlib import asynccontextmanager +from pathlib import Path from types import SimpleNamespace import pytest -from virsh_panel.fabric import FabricError +from virsh_panel.fabric import FabricError, FabricNode, FabricPlan from virsh_panel.vpc import ( + VPC_DELETE_RECEIPT_KIND, VPC_SCHEMA_VERSION, VpcCoordinator, VpcError, _plan_from_record, + parse_vpc_delete_plan, parse_vpc_plan, ) from virsh_panel.vpc_dataplane import derive_private_mac @@ -90,6 +93,62 @@ def payload(**overrides): return value +def delete_payload(*, vpc_id="vpc-hk-1", desired_revision=2, + operation_id="vpc-delete-00000001", + hosts=("host-a",), retiring_members=(), **overrides): + value = payload( + vpc_id=vpc_id, + desired_revision=desired_revision, + operation_id=operation_id, + subnets=[], members=[], routes=[], acls=[], + hosts=[{"host_id": host_id, "members": []} for host_id in hosts], + ) + value["retiring_members"] = list(retiring_members) + value.update(overrides) + value["desired_digest"] = parse_vpc_delete_plan(value)[0].desired_digest + return value + + +def confirm_delete_hosts(plan, record): + """Fill the exact bounded absence proof required for a delete receipt.""" + for host in record.get("hosts", []): + host.update({ + "state": "active", + "removed": True, + "transport_verified": True, + "private_dataplane_verified": True, + "desired_revision": plan.desired_revision, + "observed_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "observed_digest": plan.desired_digest, + "member_ids": [], + "retiring_members": [], + "member_nics": [], + "routes": [], + "wg_allowed_ips": [], + "proxy_ips": [], + "bridge": None, + "bridge_owner": None, + "nft_table": None, + "anti_spoof": False, + "rp_filter": None, + "proxy_arp": None, + "proxy_arp_pvlan": None, + "readback": "confirmed", + "rollback": "none", + }) + + +def retiring_member_a(*, host_id="host-a"): + return { + "member_id": "member-a", + "instance_uuid": INSTANCE_A, + "host_id": host_id, + "private_ip_cidr": "10.20.0.10/32", + "private_mac": derive_private_mac("vpc-hk-1", "member-a"), + } + + def test_plan_canonicalizes_members_and_rejects_public_or_overlapping_state(): first = parse_vpc_plan(payload()) second = parse_vpc_plan(payload( @@ -799,3 +858,590 @@ def test_lifecycle_conflict_requires_reinstall_commit_for_same_operation(): })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") is None + + +def test_root_delete_accepts_revision_jump_and_superseded_unknown_host(): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a", "host-b"))) + asyncio.run(coordinator.apply(payload())) + calls = [] + + async def confirm(plan, record, gate, previous_plan=None, **kwargs): + calls.append({ + "host_ids": sorted(host["host_id"] for host in record["hosts"]), + "retiring": all(host.get("retiring") is True + for host in record["hosts"]), + "operation_id": kwargs.get("operation_id"), + }) + confirm_delete_hosts(plan, record) + + coordinator._reconcile_dataplane = confirm + target = delete_payload( + desired_revision=4, + operation_id="vpc-delete-revision-jump-0001", + hosts=("host-a", "host-b"), + retiring_members=(retiring_member_a(),), + ) + result = asyncio.run(coordinator.delete("vpc-hk-1", target)) + + assert result == { + "vpc_id": "vpc-hk-1", + "operation_id": "vpc-delete-revision-jump-0001", + "desired_revision": 4, + "desired_digest": target["desired_digest"], + "state": "deleted", + "deleted": True, + "readback": "confirmed", + "removed_host_ids": ["host-a", "host-b"], + "completed_at": result["completed_at"], + } + assert result["completed_at"] > 0 + assert calls == [{ + "host_ids": ["host-a", "host-b"], + "retiring": True, + "operation_id": "vpc-delete-revision-jump-0001", + }] + assert store.get("vpc_desired", "vpc-hk-1") is None + assert store.list("vpc_host") == {} + durable_receipt = json.loads(store.get( + VPC_DELETE_RECEIPT_KIND, "vpc-hk-1")) + assert durable_receipt["readback"] == "confirmed" + assert durable_receipt["completed_at"] == result["completed_at"] + assert "retiring_members" not in durable_receipt + + # A receipt written by an earlier engine build is compacted on replay; + # the digest retains the manifest commitment without permanently keeping + # member UUID/private IP/MAC identifiers after successful deletion. + legacy_receipt = dict(durable_receipt) + legacy_receipt["retiring_members"] = target["retiring_members"] + store.set(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1", json.dumps(legacy_receipt)) + + replay = asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert replay == result + compacted_raw = store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") + compacted = json.loads(compacted_raw) + assert "retiring_members" not in compacted + assert INSTANCE_A not in compacted_raw + assert "10.20.0.10" not in compacted_raw + assert retiring_member_a()["private_mac"] not in compacted_raw + conflict = dict(target, operation_id="vpc-delete-revision-jump-0002") + with pytest.raises(VpcError) as changed: + asyncio.run(coordinator.delete("vpc-hk-1", conflict)) + assert changed.value.status == 409 + assert changed.value.error_code == "vpc_delete_receipt_conflict" + with pytest.raises(VpcError) as recreated: + asyncio.run(coordinator.apply(payload( + desired_revision=5, operation_id="vpc-after-delete-0001"))) + assert recreated.value.error_code == "vpc_deleted" + + +def test_delete_manifest_accepts_unicast_with_odd_last_octet_only(): + derived = derive_private_mac("never", "m") + assert derived.endswith(":49") + valid = delete_payload( + vpc_id="never", cidr="10.24.0.0/24", desired_revision=1, + operation_id="vpc-delete-mac-boundary-0001", hosts=("host-a",), + retiring_members=({ + "member_id": "m", "instance_uuid": INSTANCE_A, + "host_id": "host-a", "private_ip_cidr": "10.24.0.10/32", + "private_mac": derived, + },), + ) + assert parse_vpc_delete_plan(valid)[1][0].private_mac == derived + invalid = dict(valid) + invalid.pop("desired_digest") + invalid["retiring_members"] = [{ + **valid["retiring_members"][0], + "private_mac": "13:00:00:00:00:48", + }] + with pytest.raises(VpcError) as multicast: + parse_vpc_delete_plan(invalid) + assert multicast.value.error_code == "vpc_delete_manifest_invalid" + invalid["retiring_members"][0]["private_mac"] = "00:00:00:00:00:00" + with pytest.raises(VpcError) as all_zero: + parse_vpc_delete_plan(invalid) + assert all_zero.value.error_code == "vpc_delete_manifest_invalid" + + +def test_delete_digest_matches_shared_php_fixture(): + fixture = json.loads(( + Path(__file__).with_name("fixtures") / "vpc_delete_digest.json" + ).read_text(encoding="utf-8")) + plan, retiring = parse_vpc_delete_plan(fixture["document"]) + assert plan.desired_digest == fixture["expected_digest"] + assert plan.desired_digest == ( + "2f83e2fe639c82702a21c200461cd4c59a57030ec7ba9603e949c5175de73314" + ) + assert [host.host_id for host in plan.hosts] == ["11", "2"] + assert [member.member_id for member in retiring] == ["1" * 32] + + +def test_root_delete_recovers_required_host_after_restart(): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a",))) + asyncio.run(coordinator.apply(payload())) + target = delete_payload( + operation_id="vpc-delete-recover-0001", + desired_revision=2, + retiring_members=(retiring_member_a(),), + ) + + async def interrupted(plan, record, gate, previous_plan=None, **kwargs): + # Simulate a process loss after retiring-member cleanup consumed its + # tombstone but before a host absence proof became durable. + for host in record["hosts"]: + host["retiring_members"] = [] + + coordinator._reconcile_dataplane = interrupted + with pytest.raises(VpcError) as pending: + asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert pending.value.status == 503 + assert pending.value.error_code == "vpc_delete_incomplete" + durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) + assert durable["delete_target"]["required_host_ids"] == ["host-a"] + assert durable["hosts"][0]["retiring_members"] == [] + + coordinator.recover() + calls = [] + + async def confirm(plan, record, gate, previous_plan=None, **kwargs): + calls.append({ + "retiring": record["hosts"][0].get("retiring"), + "retiring_members": list( + record["hosts"][0].get("retiring_members", [])), + }) + confirm_delete_hosts(plan, record) + + coordinator._reconcile_dataplane = confirm + result = asyncio.run(coordinator.reconcile("vpc-hk-1")) + assert result["deleted"] is True + assert result["readback"] == "confirmed" + assert calls == [{ + "retiring": True, + "retiring_members": [target["retiring_members"][0]], + }] + + +def test_root_delete_restart_retries_only_hosts_without_exact_removal_proof(): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a", "host-b"))) + asyncio.run(coordinator.apply(payload())) + target = delete_payload( + operation_id="vpc-delete-partial-restart-0001", + desired_revision=2, + hosts=("host-a", "host-b"), + retiring_members=(retiring_member_a(),), + ) + calls = [] + + async def partial_then_confirm( + plan, record, gate, previous_plan=None, **kwargs): + host_ids = [host["host_id"] for host in record["hosts"]] + calls.append(host_ids) + if len(calls) == 1: + assert host_ids == ["host-a", "host-b"] + host_a = next( + host for host in record["hosts"] + if host["host_id"] == "host-a") + confirm_delete_hosts(plan, {"hosts": [host_a]}) + host_b = next( + host for host in record["hosts"] + if host["host_id"] == "host-b") + host_b.update({ + "state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": "vpc_host_unavailable", + }) + return + assert host_ids == ["host-b"] + confirm_delete_hosts(plan, record) + + coordinator._reconcile_dataplane = partial_then_confirm + with pytest.raises(VpcError) as incomplete: + asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert incomplete.value.error_code == "vpc_delete_incomplete" + durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) + durable_hosts = {host["host_id"]: host for host in durable["hosts"]} + assert coordinator._confirmed_removal( + durable_hosts["host-a"], 2, target["desired_digest"]) + assert not coordinator._confirmed_removal( + durable_hosts["host-b"], 2, target["desired_digest"]) + + coordinator.recover() + recovered = json.loads(store.get("vpc_desired", "vpc-hk-1")) + recovered_hosts = {host["host_id"]: host for host in recovered["hosts"]} + assert coordinator._confirmed_removal( + recovered_hosts["host-a"], 2, target["desired_digest"]) + assert recovered_hosts["host-b"]["state"] == "uncertain" + + result = asyncio.run(coordinator.reconcile("vpc-hk-1")) + assert calls == [["host-a", "host-b"], ["host-b"]] + assert result["deleted"] is True + assert result["removed_host_ids"] == ["host-a", "host-b"] + assert result["readback"] == "confirmed" + + +def test_delete_pending_projection_does_not_probe_confirmed_offline_host(): + target = delete_payload( + operation_id="vpc-delete-pending-probe-0001", + desired_revision=2, + hosts=("host-a", "host-b"), + ) + plan, _manifest = parse_vpc_delete_plan(target) + nodes = ( + FabricNode("host-a", "100.64.0.1", "192.0.2.1:51820", 51820), + FabricNode("host-b", "100.64.0.2", "192.0.2.2:51820", 51820), + ) + fabric_plan = FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, nodes, + "f" * 64, + ) + probed = [] + + class RetryFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, retry_plan): + for node in retry_plan.nodes: + probed.append(node.host_id) + if node.host_id == "host-a": + raise FabricError( + "confirmed host is now offline", 502, + "fabric_host_capability_probe_failed", + ) + return { + "host-b": ( + SimpleNamespace(manager=object()), + {"host_id": "host-b"}, + ), + } + + coordinator = VpcCoordinator( + MemoryStore(), RetryFabric(hosts=("host-a", "host-b")), + registry=object(), + ) + coordinator._validate_dataplane_resources = lambda _plan: None + coordinator._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, + } + coordinator._fabric_peers_for_plan = lambda _plan, _record: [] + coordinator._run_host_dataplane = lambda *args, **kwargs: { + "state": "active", "removed": True, + "transport_verified": True, + "private_dataplane_verified": True, + "member_nics": [], "routes": [], "wg_allowed_ips": [], + "proxy_ips": [], "bridge": None, "bridge_owner": None, + "nft_table": None, "anti_spoof": False, "rp_filter": None, + "proxy_arp": None, "proxy_arp_pvlan": None, + "readback": "confirmed", "rollback": "none", + } + record = { + "vpc_id": plan.vpc_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "hosts": [{ + "host_id": "host-b", "member_ids": [], "retiring": True, + "retiring_members": [], + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + }], + } + + asyncio.run(coordinator._reconcile_dataplane_unlocked( + plan, record, + {"active": False, "available": True}, + previous_plan=plan, + operation_id=plan.operation_id, + )) + + assert probed == ["host-b"] + assert coordinator._confirmed_removal( + record["hosts"][0], plan.desired_revision, plan.desired_digest) + + +def test_root_delete_surfaces_only_allowlisted_deterministic_host_errors(): + deterministic_codes = ( + "vpc_private_remove_requires_stopped", + "vpc_private_remove_unsafe", + "vpc_private_interface_conflict", + "vpc_seed_missing", + "vpc_seed_readback_invalid", + "vpc_private_xml_invalid", + ) + for index, error_code in enumerate(deterministic_codes, start=1): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a",))) + asyncio.run(coordinator.apply(payload())) + target = delete_payload( + operation_id=f"vpc-delete-deterministic-{index:04d}", + retiring_members=(retiring_member_a(),), + ) + + async def blocked(plan, record, gate, previous_plan=None, **kwargs): + record["hosts"][0].update({ + "state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": error_code, + }) + + coordinator._reconcile_dataplane = blocked + with pytest.raises(VpcError) as blocked_delete: + asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert blocked_delete.value.status == 409 + assert blocked_delete.value.error_code == error_code + assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None + + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a",))) + asyncio.run(coordinator.apply(payload())) + target = delete_payload( + operation_id="vpc-delete-transient-error-0001", + retiring_members=(retiring_member_a(),), + ) + + async def transient(plan, record, gate, previous_plan=None, **kwargs): + record["hosts"][0].update({ + "state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": "vpc_private_remove_uncertain", + }) + + coordinator._reconcile_dataplane = transient + with pytest.raises(VpcError) as transient_delete: + asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert transient_delete.value.status == 503 + assert transient_delete.value.error_code == "vpc_delete_incomplete" + + +@pytest.mark.parametrize("include_proven_host", [False, True]) +def test_root_delete_carries_forward_exact_removed_host_without_recontact( + include_proven_host): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a",))) + asyncio.run(coordinator.apply(payload())) + detached = payload( + desired_revision=2, + operation_id="vpc-member-detach-0001", + subnets=[], members=[], routes=[], acls=[], hosts=[], + ) + # Ordinary apply models the earlier member/deployment detach. Its host + # tombstone is then completed with the exact revision/digest proof that a + # lost PHP response may leave behind. + asyncio.run(coordinator.apply(detached)) + durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) + assert durable["hosts"][0]["retiring"] is True + confirm_delete_hosts(parse_vpc_plan(detached), durable) + store.set("vpc_desired", "vpc-hk-1", json.dumps(durable)) + + async def must_not_recontact(*args, **kwargs): + raise AssertionError("exact removed host must not be contacted again") + + coordinator._reconcile_dataplane = must_not_recontact + target = delete_payload( + desired_revision=3, + operation_id=("vpc-delete-proven-included-0001" + if include_proven_host + else "vpc-delete-proven-omitted-0001"), + hosts=(("host-a",) if include_proven_host else ()), + ) + result = asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert result["removed_host_ids"] == ["host-a"] + assert result["readback"] == "confirmed" + + +def test_root_delete_requires_every_unproven_old_host_in_target(): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a",))) + asyncio.run(coordinator.apply(payload())) + target = delete_payload( + desired_revision=2, + operation_id="vpc-delete-missing-host-0001", + hosts=(), + ) + with pytest.raises(VpcError) as missing: + asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert missing.value.status == 409 + assert missing.value.error_code == "vpc_delete_host_set_mismatch" + assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None + + +def test_root_delete_without_desired_record_proves_requested_hosts_or_empty(): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a",))) + calls = [] + + async def confirm(plan, record, gate, previous_plan=None, **kwargs): + calls.append(( + [host["host_id"] for host in record["hosts"]], + [host.get("retiring") for host in record["hosts"]], + [row["instance_uuid"] for host in record["hosts"] + for row in host.get("retiring_members", [])], + )) + confirm_delete_hosts(plan, record) + + coordinator._reconcile_dataplane = confirm + target = delete_payload( + vpc_id="vpc-never-applied", + operation_id="vpc-delete-never-applied-0001", + desired_revision=3, + hosts=("host-a",), + cidr="10.21.0.0/24", + retiring_members=({ + "member_id": "member-never-applied", + "instance_uuid": INSTANCE_B, + "host_id": "host-a", + "private_ip_cidr": "10.21.0.10/32", + "private_mac": derive_private_mac( + "vpc-never-applied", "member-never-applied"), + },), + ) + result = asyncio.run(coordinator.delete("vpc-never-applied", target)) + assert result["removed_host_ids"] == ["host-a"] + assert calls == [(["host-a"], [True], [INSTANCE_B])] + + empty = delete_payload( + vpc_id="vpc-empty-never-applied", + operation_id="vpc-delete-empty-never-applied-0001", + desired_revision=1, + hosts=(), + cidr="10.22.0.0/24", + ) + result = asyncio.run(coordinator.delete( + "vpc-empty-never-applied", empty)) + assert result["removed_host_ids"] == [] + assert len(calls) == 1 + + +def test_delete_manifest_removes_unknown_member_nic_before_host_resources( + monkeypatch): + import sys + import types + + try: + import libvirt # noqa: F401 + except ModuleNotFoundError: + stub = types.ModuleType("libvirt") + stub.libvirtError = type("libvirtError", (Exception,), {}) + monkeypatch.setitem(sys.modules, "libvirt", stub) + from virsh_panel.libvirt_service import VMService + from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter + + target = delete_payload( + vpc_id="vpc-remote-only", + operation_id="vpc-delete-remote-only-0001", + desired_revision=5, + hosts=("host-a",), + cidr="10.23.0.0/24", + retiring_members=({ + "member_id": "member-remote-only", + "instance_uuid": INSTANCE_B, + "host_id": "host-a", + "private_ip_cidr": "10.23.0.10/32", + "private_mac": derive_private_mac( + "vpc-remote-only", "member-remote-only"), + },), + ) + plan, manifest = parse_vpc_delete_plan(target) + rows = [{ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "host_id": member.host_id, + "private_ip_cidr": f"{member.private_ip}/32", + "private_mac": member.private_mac, + } for member in manifest] + events = [] + + monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None) + + def remove_nic(self, instance_uuid, **kwargs): + events.append(("nic", instance_uuid, kwargs["vpc_id"])) + return {"removed": True, "readback": "confirmed"} + + monkeypatch.setattr(VMService, "remove_private_interface", remove_nic) + monkeypatch.setattr( + VpcPrivateHostAdapter, "__init__", + lambda self, manager, profile: None, + ) + + def remove_host(self, dataplane): + events.append(("host", dataplane.host_id, dataplane.vpc.vpc_id)) + return { + "state": "active", "removed": True, + "bridge": None, "bridge_owner": None, "nft_table": None, + "anti_spoof": False, "rp_filter": None, + "proxy_arp": None, "proxy_arp_pvlan": None, + "proxy_ips": [], "member_nics": [], "routes": [], + "wg_allowed_ips": [], "readback": "confirmed", + } + + monkeypatch.setattr(VpcPrivateHostAdapter, "remove", remove_host) + coordinator = VpcCoordinator(MemoryStore(), FakeFabric()) + observed = coordinator._run_host_dataplane( + plan, "host-a", SimpleNamespace(manager=object()), + {"libvirt_uri": "qemu:///system", "storage_pool": "default", + "iso_pool": "iso", "default_network": "default"}, + [], None, retiring_members=rows, fabric_public_keys={}, + ) + assert events == [ + ("nic", INSTANCE_B, "vpc-remote-only"), + ("host", "host-a", "vpc-remote-only"), + ] + assert observed["removed"] is True + assert observed["readback"] == "confirmed" + + +def test_root_delete_receipt_survives_purge_failure_and_keeps_timestamp(): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a",))) + asyncio.run(coordinator.apply(payload())) + + async def confirm(plan, record, gate, previous_plan=None, **kwargs): + confirm_delete_hosts(plan, record) + + coordinator._reconcile_dataplane = confirm + target = delete_payload( + operation_id="vpc-delete-purge-crash-0001", + retiring_members=(retiring_member_a(),), + ) + original_delete = store.delete + failed = False + + def fail_once(kind, key): + nonlocal failed + if kind == "vpc_desired" and not failed: + failed = True + raise RuntimeError("crash after receipt commit") + return original_delete(kind, key) + + store.delete = fail_once + with pytest.raises(RuntimeError, match="after receipt commit"): + asyncio.run(coordinator.delete("vpc-hk-1", target)) + durable_receipt = json.loads(store.get( + VPC_DELETE_RECEIPT_KIND, "vpc-hk-1")) + first_completed_at = durable_receipt["completed_at"] + assert first_completed_at > 0 + + result = asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert result["completed_at"] == first_completed_at + assert result["readback"] == "confirmed" + assert store.get("vpc_desired", "vpc-hk-1") is None diff --git a/engine/tests/test_vpc_dataplane.py b/engine/tests/test_vpc_dataplane.py index eea5d181..32dbde96 100644 --- a/engine/tests/test_vpc_dataplane.py +++ b/engine/tests/test_vpc_dataplane.py @@ -487,6 +487,46 @@ def test_remove_script_checks_ownership_before_destructive_cleanup(): assert "managed 71 72" in script assert "empty 1 1" in script assert "empty 71 72" in script + identity_check = 'vc_vpc_artifact_identity_check "$nft_identity"' + assert script.index(identity_check) < script.index( + 'mkdir -m 700 "$TXN"') + assert script.index(identity_check) < script.index( + '"$IP" route del "$destination"') + assert script.index(identity_check) < script.index( + '"$NFT" delete table "$family" "$name"') + + +def test_remove_artifact_identity_allows_older_revision_only(): + plan = _plan() + delete_plan = replace( + plan, + vpc=replace( + plan.vpc, + desired_revision=2, + desired_digest="b" * 64, + ), + members=tuple(member for member in plan.members + if member.host_id != plan.host_id), + ) + script = render_remove_script(delete_plan) + function = re.search( + r"(?ms)^vc_vpc_artifact_identity_check\(\) \{.*?^\}\n", + script, + ) + assert function is not None + contract = function.group(0) + r''' +vc_vpc_artifact_identity_check "vpc=vpc-hk-dp revision=1 digest=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || exit 81 +vc_vpc_artifact_identity_check "vpc=vpc-hk-dp revision=2 digest=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" || exit 82 +if vc_vpc_artifact_identity_check "vpc=vpc-hk-dp revision=2 digest=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; then exit 83; fi +if vc_vpc_artifact_identity_check "vpc=vpc-hk-dp revision=3 digest=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; then exit 84; fi +if vc_vpc_artifact_identity_check "vpc=foreign revision=1 digest=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; then exit 85; fi +exit 0 +''' + result = subprocess.run( + ["/bin/sh"], input=contract, text=True, + capture_output=True, check=False, + ) + assert result.returncode == 0, result.stderr def test_proxy_markers_are_bounded_and_typed(): diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index 62f4243e..69ae6f12 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -145,6 +145,7 @@ WRITE_PATHS = { "POST": (r"host/(?:scan|unlock|test)", r"fabric/apply", r"fabric/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}/reconcile", r"vpcs/apply", r"vpcs/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}/reconcile", + r"vpcs/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}/delete", r"vm-templates/(?:import|upload)", r"rescue-templates", r"user-isos/import", r"vms", rf"vms/{VM_ID}/(?:actions|credential|reinstall|network|settings|rescue|console-ticket)", @@ -1085,6 +1086,26 @@ class EngineApplication: result = await self.vpc.apply(value) await json_response(send, 200, result) return + delete_match = re.fullmatch( + r"vpcs/([A-Za-z0-9][A-Za-z0-9_.-]{0,63})/delete", + resource, + ) + if delete_match is not None: + try: + value = json.loads(body) + except (TypeError, ValueError) as exc: + raise VpcError("VPC 删除计划 JSON 无效", 422) from exc + if not isinstance(value, dict): + raise VpcError("VPC 删除计划必须是对象", 422) + if not isinstance(value.get("operation_id"), str): + raise VpcError("VPC 删除计划缺少 operation_id", 400, + "vpc_operation_required") + if not isinstance(value.get("desired_digest"), str): + raise VpcError("VPC 删除计划缺少 desired_digest", 400, + "vpc_digest_required") + result = await self.vpc.delete(delete_match[1], value) + await json_response(send, 200, result) + return match = re.fullmatch( r"vpcs/([A-Za-z0-9][A-Za-z0-9_.-]{0,63})/reconcile", resource, diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index badec08a..678d13b8 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -1778,9 +1778,48 @@ class VMService: def remove_private_interface( self, vm_id: str, *, vpc_id: str, resources: Any, desired_revision: int, desired_digest: str, + expected_member_id: str, expected_private_mac: str, + expected_private_ip_cidr: str, ) -> dict[str, Any]: """Detach a private0 NIC only when its panel VPC ownership matches.""" vm_uuid = self._uuid(vm_id) + if not isinstance(vpc_id, str) or not vpc_id: + raise VMError("VPC 标识无效", 422, error_code="vpc_invalid") + if (isinstance(desired_revision, bool) + or not isinstance(desired_revision, int) + or desired_revision < 1): + raise VMError("VPC revision 无效", 422, + error_code="vpc_invalid") + if (not isinstance(desired_digest, str) + or re.fullmatch(r"[0-9a-f]{64}", desired_digest) is None): + raise VMError("VPC 摘要无效", 422, error_code="vpc_invalid") + if (not isinstance(expected_member_id, str) + or re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}", + expected_member_id) is None): + raise VMError("VPC 成员标识无效", 422, + error_code="vpc_invalid") + if (not isinstance(expected_private_mac, str) + or re.fullmatch( + r"[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}", + expected_private_mac) is None + or int(expected_private_mac.replace(":", ""), 16) == 0 + or int(expected_private_mac[:2], 16) & 1): + raise VMError("VPC 成员 MAC 地址无效", 422, + error_code="vpc_invalid") + expected_private_mac = expected_private_mac.lower() + expected_bridge = getattr(resources, "bridge", None) + if (not isinstance(expected_bridge, str) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,14}", + expected_bridge)): + raise VMError("VPC bridge 标识无效", 422, + error_code="vpc_invalid") + try: + expected_private_ip_cidr = validate_private_network( + expected_private_ip_cidr)["private_ip_cidr"] + except (KeyError, TypeError, ValueError) as exc: + raise VMError("VPC 成员私网 IP 无效", 422, + error_code="vpc_invalid") from exc with _CREATE_LOCK, self._connect() as conn: domain = self._domain(conn, vm_uuid) if not domain.isPersistent() or getattr(domain, "hasManagedSaveImage", lambda _flags: 0)(0): @@ -1796,74 +1835,152 @@ class VMService: root = ET.fromstring(old_xml) private = self._private_interface_from_root(root) managed = root.find(f"./metadata/{{{PANEL_NS}}}managed") - if private is None: - return {"removed": True, "already_absent": True, - "xml": old_xml, "readback": "confirmed"} - if managed is None or managed.get("private-vpc-id") != vpc_id: + private_fields = [ + key for key in managed.attrib + if key.startswith("private-") + ] if managed is not None else [] + owner_vpc = (managed.get("private-vpc-id") + if managed is not None else None) + owner_member = (managed.get("private-member-id") + if managed is not None else None) + owner_bridge = (managed.get("private-network-bridge") + if managed is not None else None) + if (owner_vpc is not None and owner_vpc != vpc_id) or ( + owner_member is not None + and owner_member != expected_member_id) or ( + owner_bridge is not None + and owner_bridge != expected_bridge): raise VMError("private0 不属于目标 VPC", 409, error_code="vpc_private_interface_conflict") - devices = root.find("./devices") - if devices is None: - raise VMError("虚拟机 XML 缺少 devices", 409, - error_code="vpc_private_xml_invalid") - devices.remove(private) - # Clear only private ownership fields; public metadata remains. - for key in ("private-vpc-id", "private-member-id", "private-ip-cidr", - "private-network-bridge", "private-network-verified", - "private-dataplane-owner", "private-network-revision", - "private-network-digest"): - managed.attrib.pop(key, None) + # A remaining private NIC needs the VPC owner marker. Conversely, + # partial private metadata without its owner is not enough to + # authorize deletion of an unknown interface/seed. + if ((private is not None and owner_vpc != vpc_id) + or (private_fields and owner_vpc is None)): + raise VMError("private0 缺少可验证的 VPC 所有权", 409, + error_code="vpc_private_interface_conflict") + if private is not None: + private_mac = private.find("mac") + private_source = private.find("source") + if (private_mac is None + or private_mac.get("address", "").lower() + != expected_private_mac + or private_source is None + or private_source.get("bridge") != expected_bridge): + raise VMError("private0 与删除成员身份不一致", 409, + error_code="vpc_private_interface_conflict") seed = self._owned_seed_volume(conn, domain) if seed is None: + cdroms = root.findall("./devices/disk[@device='cdrom']") + if private is None and not private_fields and not cdroms: + # No panel-owned seed exists to carry a stale private0 + # role, there is no unowned CD-ROM that could still carry + # one, and both persistent XML layers are already clean. + # This is a positive absence proof. An unowned CD-ROM is + # deliberately not inspected or rewritten and therefore + # cannot be treated as proof of seed convergence. + return {"removed": True, "already_absent": True, + "xml": old_xml, "readback": "confirmed"} raise VMError("找不到 cloud-init 配置盘,无法移除私网", 409, error_code="vpc_seed_missing") old_seed = self._download_seed_volume(conn, seed) - replacement = replace_seed_private_network( - old_seed, private_mac=None, private_ip_cidr=None) + try: + old_seed_network = read_seed_network(old_seed) + except (KeyError, TypeError, ValueError) as exc: + raise VMError("无法核对 cloud-init 私网配置", 502, + error_code="vpc_seed_readback_invalid") from exc + seed_has_private = ( + "private_ip_cidr" in old_seed_network + or "private_mac" in old_seed_network + or "private_routes" in old_seed_network + or "private0" in old_seed_network.get("roles", []) + ) + if seed_has_private and ( + old_seed_network.get("private_ip_cidr") + != expected_private_ip_cidr + or str(old_seed_network.get("private_mac", "")).lower() + != expected_private_mac): + raise VMError("cloud-init 私网与删除成员身份不一致", 409, + error_code="vpc_private_interface_conflict") + + xml_changed = private is not None or bool(private_fields) + if private is not None: + devices = root.find("./devices") + if devices is None: + raise VMError("虚拟机 XML 缺少 devices", 409, + error_code="vpc_private_xml_invalid") + devices.remove(private) + if managed is not None: + # Every ``private-*`` attribute is owned by this feature; the + # public VM metadata and seed-volume identity remain intact. + for key in tuple(managed.attrib): + if key.startswith("private-"): + managed.attrib.pop(key, None) + replacement = ( + replace_seed_private_network( + old_seed, private_mac=None, private_ip_cidr=None) + if seed_has_private else old_seed + ) define_attempted = False defined = False - seed_touched = False + seed_attempted = False committed = False + already_absent = not xml_changed and not seed_has_private try: - define_attempted = True - defined_domain = conn.defineXML(ET.tostring(root, encoding="unicode")) - if defined_domain is None: - raise VMError("宿主机未能移除 private0", 502, - error_code="vpc_private_remove_failed") - defined = True - if len(replacement) > int(seed.info()[1]): - seed.resize(len(replacement), 0) - seed_touched = True - self._upload_volume(conn, seed, iter((replacement,)), len(replacement)) + defined_domain = domain + if xml_changed: + define_attempted = True + defined_domain = conn.defineXML( + ET.tostring(root, encoding="unicode")) + if defined_domain is None: + raise VMError("宿主机未能移除 private0", 502, + error_code="vpc_private_remove_failed") + defined = True + if seed_has_private: + seed_attempted = True + if len(replacement) > int(seed.info()[1]): + seed.resize(len(replacement), 0) + self._upload_volume( + conn, seed, iter((replacement,)), len(replacement)) observed_xml = defined_domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE) observed_root = ET.fromstring(observed_xml) if self._private_interface_from_root(observed_root) is not None: raise VMError("private0 移除回读不一致", 502, error_code="vpc_private_remove_readback_mismatch") + observed_managed = observed_root.find( + f"./metadata/{{{PANEL_NS}}}managed") + if (observed_managed is not None + and any(key.startswith("private-") + for key in observed_managed.attrib)): + raise VMError("private0 所有权元数据移除回读不一致", 502, + error_code="vpc_private_remove_readback_mismatch") observed_seed = read_seed_network(self._download_seed_volume(conn, seed)) - if "private_ip_cidr" in observed_seed or "private_mac" in observed_seed: + if ("private_ip_cidr" in observed_seed + or "private_mac" in observed_seed + or "private_routes" in observed_seed + or "private0" in observed_seed.get("roles", [])): raise VMError("私网配置盘移除回读不一致", 502, error_code="vpc_seed_remove_readback_mismatch") committed = True - return {"removed": True, "already_absent": False, + return {"removed": True, "already_absent": already_absent, "xml": observed_xml, "readback": "confirmed"} except VMError: raise except Exception as exc: # A lost define/upload response must be reconciled from the # host. Do not blindly re-attach a live or unknown device. - if define_attempted: + if define_attempted or seed_attempted: raise VMError("私网移除结果不确定,请等待宿主机回读", 502, error_code="vpc_private_remove_uncertain") from exc raise VMError("宿主机未能移除 private0", 502, error_code="vpc_private_remove_failed") from exc finally: - if not committed and defined: + if not committed and (defined or seed_attempted): try: # The domain is stopped, so restoring the persistent # XML and seed is a confirmed, safe inverse operation. conn.defineXML(old_xml) - if seed_touched: + if seed_attempted: if len(old_seed) > int(seed.info()[1]): seed.resize(len(old_seed), 0) self._upload_volume(conn, seed, iter((old_seed,)), len(old_seed)) diff --git a/engine/virsh_panel/vpc.py b/engine/virsh_panel/vpc.py index 0e0a7f21..0675fd22 100644 --- a/engine/virsh_panel/vpc.py +++ b/engine/virsh_panel/vpc.py @@ -32,6 +32,15 @@ VPC_MAX_ACLS = 2048 VPC_MAX_READBACK = 4096 VPC_MAX_READBACK_VALUE = 512 VPC_STATES = frozenset({"applying", "active", "degraded", "uncertain"}) +VPC_DELETE_RECEIPT_KIND = "vpc_deleted" +VPC_DELETE_DETERMINISTIC_HOST_ERRORS = frozenset({ + "vpc_private_remove_requires_stopped", + "vpc_private_remove_unsafe", + "vpc_private_interface_conflict", + "vpc_seed_missing", + "vpc_seed_readback_invalid", + "vpc_private_xml_invalid", +}) _IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z") _HOST_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z") @@ -128,7 +137,7 @@ def _mac(value: Any) -> str | None: raise VpcError("VPC 成员 MAC 地址无效") parsed = int(value.replace(":", ""), 16) # Multicast and all-zero addresses cannot identify a guest interface. - if parsed == 0 or (parsed & 1): + if parsed == 0 or (int(value[:2], 16) & 1): raise VpcError("VPC 成员 MAC 地址无效") return value.lower() @@ -233,6 +242,17 @@ class VpcMember: private_mac: str | None = None +@dataclass(frozen=True) +class VpcDeleteMember: + """Signed VM identity whose persistent private NIC must be absent.""" + + member_id: str + instance_uuid: str + host_id: str + private_ip: str + private_mac: str + + @dataclass(frozen=True) class VpcRoute: route_id: str @@ -719,6 +739,95 @@ def parse_vpc_plan(value: Mapping[str, Any]) -> VpcPlan: provisional.acls, computed_digest, provisional.operation_id) +def _canonical_delete_members( + members: Sequence[VpcDeleteMember], +) -> list[dict[str, str]]: + return [{ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "host_id": member.host_id, + "private_ip_cidr": f"{member.private_ip}/32", + "private_mac": member.private_mac, + } for member in members] + + +def _parse_delete_members( + value: Any, plan: VpcPlan, +) -> tuple[VpcDeleteMember, ...]: + if not isinstance(value, list) or len(value) > VPC_MAX_MEMBERS: + raise VpcError("VPC 删除成员清单必须是数组且不超过限制", 422, + "vpc_delete_manifest_invalid") + network = ipaddress.ip_network(plan.cidr) + members: list[VpcDeleteMember] = [] + member_ids: set[str] = set() + instance_uuids: set[str] = set() + private_ips: set[str] = set() + private_macs: set[str] = set() + try: + for raw in value: + if not isinstance(raw, Mapping): + raise VpcError("VPC 删除成员必须是对象") + _field_set(raw, { + "member_id", "instance_uuid", "host_id", + "private_ip_cidr", "private_mac", + }, "VPC 删除成员") + member_id = _identifier(raw.get("member_id"), "VPC member") + instance_uuid = _uuid(raw.get("instance_uuid")) + host_id = _host_id(raw.get("host_id")) + if host_id not in plan.host_ids: + raise VpcError("VPC 删除成员宿主机不在删除目标中") + private_ip = _host_address(raw.get("private_ip_cidr")) + address = ipaddress.IPv4Address(private_ip) + if address not in network or address in { + network.network_address, network.broadcast_address}: + raise VpcError("VPC 删除成员私网 IP 不在 VPC 可用范围") + private_mac = _mac(raw.get("private_mac")) + if private_mac is None: + raise VpcError("VPC 删除成员 private_mac 缺失") + if (member_id in member_ids or instance_uuid in instance_uuids + or private_ip in private_ips + or private_mac in private_macs): + raise VpcError("VPC 删除成员清单包含重复身份") + member_ids.add(member_id) + instance_uuids.add(instance_uuid) + private_ips.add(private_ip) + private_macs.add(private_mac) + members.append(VpcDeleteMember( + member_id, instance_uuid, host_id, private_ip, private_mac)) + except VpcError as exc: + if exc.error_code == "vpc_delete_manifest_invalid": + raise + raise VpcError(str(exc), 422, + "vpc_delete_manifest_invalid") from exc + return tuple(sorted(members, key=lambda item: item.member_id)) + + +def parse_vpc_delete_plan( + value: Mapping[str, Any], +) -> tuple[VpcPlan, tuple[VpcDeleteMember, ...]]: + """Parse the empty target plus its digest-covered VM cleanup manifest.""" + if not isinstance(value, Mapping): + raise VpcError("VPC 删除计划必须是对象", 422, + "vpc_delete_manifest_invalid") + if "retiring_members" not in value: + raise VpcError("VPC 删除计划缺少 retiring_members", 422, + "vpc_delete_manifest_invalid") + base = dict(value) + raw_members = base.pop("retiring_members") + supplied_digest = base.pop("desired_digest", None) + plan = parse_vpc_plan(base) + members = _parse_delete_members(raw_members, plan) + canonical = plan.canonical() + canonical["retiring_members"] = _canonical_delete_members(members) + computed_digest = _canonical_digest(canonical) + if supplied_digest is not None: + supplied_digest = _digest(supplied_digest) + if supplied_digest != computed_digest: + raise VpcError("desired_digest 与 VPC 删除计划不一致", 409, + "vpc_digest_conflict") + return replace(plan, desired_digest=computed_digest), members + + def _plan_from_record(value: Mapping[str, Any]) -> VpcPlan: if not isinstance(value, Mapping): raise VpcError("VPC 持久化记录损坏", 500, "vpc_record_corrupt") @@ -766,7 +875,29 @@ def _plan_from_record(value: Mapping[str, Any]) -> VpcPlan: plan = parse_vpc_plan(payload) except VpcError as exc: raise VpcError("VPC 持久化记录无效", 500, "vpc_record_corrupt") from exc - if value.get("desired_digest") not in (None, plan.desired_digest): + stored_digest = value.get("desired_digest") + delete_target = value.get("delete_target") + if delete_target is not None: + if (not isinstance(delete_target, Mapping) + or "retiring_members" not in delete_target): + raise VpcError("VPC 删除目标记录无效", 500, + "vpc_record_corrupt") + try: + retiring_members = _parse_delete_members( + delete_target["retiring_members"], plan) + except VpcError as exc: + raise VpcError("VPC 删除成员清单记录无效", 500, + "vpc_record_corrupt") from exc + canonical = plan.canonical() + canonical["retiring_members"] = _canonical_delete_members( + retiring_members) + delete_digest = _canonical_digest(canonical) + if (stored_digest != delete_digest + or delete_target.get("desired_digest") != delete_digest): + raise VpcError("VPC 删除持久化摘要不一致", 500, + "vpc_record_corrupt") + return replace(plan, desired_digest=delete_digest) + if stored_digest not in (None, plan.desired_digest): raise VpcError("VPC 持久化摘要不一致", 500, "vpc_record_corrupt") return plan @@ -817,6 +948,215 @@ class VpcCoordinator: self.store.set(kind, key, json.dumps(value, ensure_ascii=True, separators=(",", ":"))) + @staticmethod + def _confirmed_removal( + host: Mapping[str, Any], revision: int, digest: str, + ) -> bool: + """Whether a durable host row is an exact absence proof. + + A boolean ``removed`` marker alone is not enough to retire a host from + a later root-delete target. The proof is useful only when it belongs + to the exact durable revision/digest and every bounded tenant resource + is absent. This also accepts the historical non-``retiring`` shape + produced when PHP kept a deleting deployment in the desired host list. + """ + readback = host.get("readback") + if isinstance(readback, Mapping): + readback_state = str( + readback.get("state", readback.get("status", ""))) + else: + readback_state = str(readback or "") + return bool( + host.get("removed") is True + and host.get("state") == "active" + and host.get("transport_verified") is True + and host.get("private_dataplane_verified") is True + and host.get("desired_revision") == revision + and host.get("observed_revision") == revision + and host.get("desired_digest") == digest + and host.get("observed_digest") == digest + and host.get("member_ids", []) == [] + and host.get("retiring_members", []) == [] + and host.get("member_nics") == [] + and host.get("routes") == [] + and host.get("wg_allowed_ips") == [] + and host.get("proxy_ips") == [] + and host.get("bridge") is None + and host.get("bridge_owner") is None + and host.get("nft_table") is None + and host.get("anti_spoof") is False + and host.get("rp_filter") is None + and host.get("proxy_arp") is None + and host.get("proxy_arp_pvlan") is None + and readback_state.lower() == "confirmed" + and host.get("rollback") in {"none", "ok"} + ) + + @staticmethod + def _delete_plan(plan: VpcPlan) -> None: + """Require the signed root-delete target to contain no tenant intent.""" + if (plan.subnets or plan.members or plan.routes or plan.acls + or any(host.member_ids for host in plan.hosts)): + raise VpcError( + "VPC 删除目标必须清空全部子资源和宿主机成员", + 409, "vpc_delete_target_not_empty") + + @staticmethod + def _host_closure(record: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: + hosts = record.get("hosts", []) + if not isinstance(hosts, list) or len(hosts) > VPC_MAX_HOSTS: + raise VpcError("VPC 删除宿主机闭集无效", 500, + "vpc_record_corrupt") + result: dict[str, Mapping[str, Any]] = {} + for host in hosts: + if not isinstance(host, Mapping): + raise VpcError("VPC 删除宿主机闭集无效", 500, + "vpc_record_corrupt") + host_id = _host_id(host.get("host_id")) + if host_id in result: + raise VpcError("VPC 删除宿主机闭集重复", 500, + "vpc_record_corrupt") + result[host_id] = host + return result + + @staticmethod + def _delete_marker(record: Mapping[str, Any]) -> dict[str, Any] | None: + marker = record.get("delete_target") + if marker is None: + return None + if not isinstance(marker, dict): + raise VpcError("VPC 删除目标记录损坏", 500, + "vpc_record_corrupt") + for field in ("operation_id", "desired_revision", "desired_digest", + "all_host_ids", "required_host_ids", + "preverified_host_ids", "retiring_members"): + if field not in marker: + raise VpcError("VPC 删除目标记录不完整", 500, + "vpc_record_corrupt") + if (not isinstance(marker["operation_id"], str) + or _OPERATION_ID.fullmatch(marker["operation_id"]) is None + or isinstance(marker["desired_revision"], bool) + or not isinstance(marker["desired_revision"], int) + or marker["desired_revision"] < 1 + or not isinstance(marker["desired_digest"], str) + or _DIGEST.fullmatch(marker["desired_digest"]) is None): + raise VpcError("VPC 删除目标记录无效", 500, + "vpc_record_corrupt") + parsed: dict[str, list[str]] = {} + for field in ("all_host_ids", "required_host_ids", + "preverified_host_ids"): + values = marker[field] + if (not isinstance(values, list) or len(values) > VPC_MAX_HOSTS + or values != sorted(set(values))): + raise VpcError("VPC 删除宿主机闭集无效", 500, + "vpc_record_corrupt") + try: + parsed[field] = [_host_id(value) for value in values] + except VpcError as exc: + raise VpcError("VPC 删除宿主机闭集无效", 500, + "vpc_record_corrupt") from exc + if ((set(parsed["required_host_ids"]) + | set(parsed["preverified_host_ids"])) + != set(parsed["all_host_ids"]) + or set(parsed["required_host_ids"]) + & set(parsed["preverified_host_ids"])): + raise VpcError("VPC 删除宿主机闭集不一致", 500, + "vpc_record_corrupt") + if (not isinstance(marker["retiring_members"], list) + or len(marker["retiring_members"]) > VPC_MAX_MEMBERS): + raise VpcError("VPC 删除成员清单记录无效", 500, + "vpc_record_corrupt") + return marker + + def _load_delete_receipt(self, vpc_id: str) -> dict[str, Any] | None: + receipt = self._load(VPC_DELETE_RECEIPT_KIND, vpc_id) + if receipt is None: + return None + required = { + "vpc_id", "operation_id", "desired_revision", "desired_digest", + "state", "deleted", "readback", "removed_host_ids", + "completed_at", + } + if (not required.issubset(receipt) + or receipt.get("schema_version") != VPC_SCHEMA_VERSION + or receipt.get("vpc_id") != vpc_id + or not isinstance(receipt.get("operation_id"), str) + or _OPERATION_ID.fullmatch(receipt["operation_id"]) is None + or isinstance(receipt.get("desired_revision"), bool) + or not isinstance(receipt.get("desired_revision"), int) + or receipt["desired_revision"] < 1 + or not isinstance(receipt.get("desired_digest"), str) + or _DIGEST.fullmatch(receipt["desired_digest"]) is None + or receipt.get("state") != "deleted" + or receipt.get("deleted") is not True + or receipt.get("readback") != "confirmed" + or isinstance(receipt.get("completed_at"), bool) + or not isinstance(receipt.get("completed_at"), int) + or receipt["completed_at"] < 1): + raise VpcError("VPC 删除回执损坏", 500, + "vpc_delete_receipt_corrupt") + host_ids = receipt.get("removed_host_ids") + if (not isinstance(host_ids, list) or len(host_ids) > VPC_MAX_HOSTS + or host_ids != sorted(set(host_ids))): + raise VpcError("VPC 删除回执宿主机闭集损坏", 500, + "vpc_delete_receipt_corrupt") + try: + for host_id in host_ids: + _host_id(host_id) + except VpcError as exc: + raise VpcError("VPC 删除回执宿主机闭集损坏", 500, + "vpc_delete_receipt_corrupt") from exc + # The signed desired digest already commits to the retiring-member + # closure. Once absence is confirmed, replay only needs the operation + # identity, digest, host closure and completion time. Compact an + # older receipt on first read so account deletion does not leave VM + # UUID, private IP and MAC data in the engine indefinitely. + compact = {key: receipt[key] for key in ( + "schema_version", "vpc_id", "operation_id", "desired_revision", + "desired_digest", "state", "deleted", "readback", + "removed_host_ids", "completed_at", + )} + if receipt != compact: + self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, compact) + return compact + + @staticmethod + def _public_delete_receipt(receipt: Mapping[str, Any]) -> dict[str, Any]: + return {key: receipt[key] for key in ( + "vpc_id", "operation_id", "desired_revision", "desired_digest", + "state", "deleted", "readback", "removed_host_ids", + "completed_at", + )} + + def _delete_payload_from_record( + self, record: Mapping[str, Any], + ) -> dict[str, Any]: + """Rebuild the exact signed delete envelope for durable recovery.""" + marker = self._delete_marker(record) + if marker is None: + raise VpcError("VPC 删除目标记录缺失", 500, + "vpc_record_corrupt") + plan = _plan_from_record(record) + if (plan.operation_id != marker["operation_id"] + or plan.desired_revision != marker["desired_revision"] + or plan.desired_digest != marker["desired_digest"]): + raise VpcError("VPC 删除目标记录不一致", 500, + "vpc_record_corrupt") + value = plan.canonical() + value["operation_id"] = marker["operation_id"] + value["desired_digest"] = marker["desired_digest"] + value["retiring_members"] = list(marker["retiring_members"]) + return value + + def _purge_deleted_vpc(self, vpc_id: str) -> None: + prefix = vpc_id + "." + for kind in ("vpc_subnet", "vpc_member", "vpc_route", "vpc_acl", + "vpc_host"): + for key in tuple(self.store.list(kind)): + if key.startswith(prefix): + self.store.delete(kind, key) + self.store.delete("vpc_desired", vpc_id) + def _fabric_gate(self, plan: VpcPlan) -> dict[str, Any]: try: status = self.fabric.get_status(plan.fabric_id) @@ -1078,6 +1418,11 @@ class VpcCoordinator: payload, host_id, trusted_tenant_allowed_ips=trusted_peer_tenant_ips, fabric_peer_keys=authoritative_peer_keys) + # A root-delete plan extends the canonical digest with its signed + # retiring-member manifest. The host projection intentionally + # omits those members from live desired state, so restore the + # authoritative envelope identity after parsing that projection. + dataplane = replace(dataplane, vpc=plan) else: # A retiring host is intentionally absent from the new desired # plan. Construct a typed empty-local plan directly so the @@ -1128,6 +1473,11 @@ class VpcCoordinator: resources=resource_names(cleanup_plan.vpc_id), desired_revision=cleanup_plan.desired_revision, desired_digest=cleanup_plan.desired_digest, + expected_member_id=old_member.member_id, + expected_private_mac=_effective_private_mac( + cleanup_plan.vpc_id, old_member.member_id, + old_member.private_mac), + expected_private_ip_cidr=f"{old_member.private_ip}/32", ) # A host with no local members is a removal target. Run the # manager-aware teardown and require its positive absence marker; @@ -1140,6 +1490,8 @@ class VpcCoordinator: "host_id": host_id, "desired_revision": plan.desired_revision, "desired_digest": plan.desired_digest, + "observed_revision": plan.desired_revision, + "observed_digest": plan.desired_digest, "transport_verified": True, "private_dataplane_verified": True, "rollback": removed.get("rollback", "none"), @@ -1486,7 +1838,19 @@ class VpcCoordinator: raise VpcError("Fabric 公钥记录不完整", 500, "vpc_fabric_record_corrupt") fabric_plan = full_fabric_plan - wanted = set(plan.host_ids) | retiring_ids + # ``record`` is the execution closure for this pass. Root-delete + # retries deliberately project it down to hosts that do not yet + # have an exact durable absence proof; deriving this set from the + # immutable submitted plan would still reprobe an already removed + # host and could block every remaining cleanup when that host is + # now offline. Normal apply records already contain both desired + # and retiring hosts, so the same rule keeps their host closure + # exact as well. + wanted = { + item.get("host_id") for item in record.get("hosts", []) + if isinstance(item, Mapping) + and isinstance(item.get("host_id"), str) + } fabric_plan = replace( fabric_plan, nodes=tuple(node for node in fabric_plan.nodes @@ -1533,7 +1897,8 @@ class VpcCoordinator: host["last_error_code"] = observed.get("reason_code") if host.get("retiring") is True: host["removed"] = observed.get("removed") is True - elif host.get("retiring_members") and observed.get("state") == "active": + if (host.get("retiring_members") + and observed.get("state") == "active"): # The host adapter has positively removed every member # listed in this tombstone. Drop only the consumed rows; # a failed/uncertain pass keeps them retryable. @@ -2001,13 +2366,313 @@ class VpcCoordinator: record["private_dataplane_verified"] = False record["updated_at"] = int(time.time()) + async def delete(self, vpc_id: str, value: Mapping[str, Any]) -> dict[str, Any]: + """Remove every host resource and persist an idempotent root receipt. + + The receipt is written before the desired/child records are purged. + Consequently a lost HTTP response can be retried with the exact same + operation/revision/digest without turning store absence into guessed + success. A signed no-host target is the sole exception: when neither + a desired record nor a receipt exists, PHP's independently locked + deployment ledger supplies the other half of that absence proof. + """ + vpc_id = _identifier(vpc_id, "VPC") + plan, retiring_members = parse_vpc_delete_plan(value) + if plan.vpc_id != vpc_id: + raise VpcError("VPC 删除路径与目标不一致", 409, + "vpc_delete_target_mismatch") + self._delete_plan(plan) + + async with self._lock(vpc_id): + receipt = self._load_delete_receipt(vpc_id) + if receipt is not None: + if (receipt["operation_id"] != plan.operation_id + or receipt["desired_revision"] != plan.desired_revision + or receipt["desired_digest"] != plan.desired_digest): + raise VpcError("VPC 已由另一删除操作完成", 409, + "vpc_delete_receipt_conflict") + self._purge_deleted_vpc(vpc_id) + return self._public_delete_receipt(receipt) + + previous = self._load("vpc_desired", vpc_id) + if previous is None and not plan.host_ids: + receipt = { + "schema_version": VPC_SCHEMA_VERSION, + "vpc_id": vpc_id, + "operation_id": plan.operation_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "state": "deleted", + "deleted": True, + "readback": "confirmed", + "removed_host_ids": [], + "completed_at": int(time.time()), + } + self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, receipt) + self._purge_deleted_vpc(vpc_id) + return self._public_delete_receipt(receipt) + + marker = (self._delete_marker(previous) + if previous is not None else None) + transition_previous: VpcPlan | None = None + if marker is None: + if previous is None: + previous_plan = None + old_hosts: dict[str, Mapping[str, Any]] = {} + previous_observed_revision = 0 + previous_observed_digest = None + else: + previous_plan = _plan_from_record(previous) + old_revision = previous.get("desired_revision") + old_digest = previous.get("desired_digest") + if (old_revision != previous_plan.desired_revision + or old_digest != previous_plan.desired_digest): + raise VpcError("VPC 持久化目标摘要不一致", 500, + "vpc_record_corrupt") + if plan.desired_revision <= previous_plan.desired_revision: + raise VpcError("VPC 删除版本必须高于当前目标", 409, + "vpc_revision_conflict") + if plan.operation_id == previous_plan.operation_id: + raise VpcError("VPC 删除必须使用新的操作编号", 409, + "vpc_delete_receipt_conflict") + old_hosts = self._host_closure(previous) + previous_observed_revision = previous.get( + "observed_revision", 0) + previous_observed_digest = previous.get("observed_digest") + old_host_ids = set(old_hosts) + requested_host_ids = set(plan.host_ids) + member_hosts = { + member.host_id for member in (previous_plan.members + if previous_plan else ()) + } + manifest_hosts = { + member.host_id for member in retiring_members + } + preverified = set() + if previous_plan is not None: + preverified = { + host_id for host_id, host in old_hosts.items() + if host_id not in member_hosts + and host_id not in manifest_hosts + and self._confirmed_removal( + host, previous_plan.desired_revision, + previous_plan.desired_digest) + } + missing = old_host_ids - requested_host_ids + if not missing.issubset(preverified): + raise VpcError("VPC 删除遗漏尚未清理的宿主机", 409, + "vpc_delete_host_set_mismatch") + required_host_ids = requested_host_ids - preverified + + retiring_by_id = { + member.member_id: member for member in retiring_members + } + for old_member in (previous_plan.members + if previous_plan else ()): + signed = retiring_by_id.get(old_member.member_id) + if (signed is None + or signed.instance_uuid != old_member.instance_uuid + or signed.host_id != old_member.host_id + or signed.private_ip != old_member.private_ip + or (old_member.private_mac is not None + and signed.private_mac + != old_member.private_mac)): + raise VpcError( + "VPC 删除成员清单未覆盖旧持久化成员", 409, + "vpc_delete_manifest_mismatch") + + # Already-proven, omitted hosts must not be contacted again. + # Excluding them from the transition plan prevents the generic + # desired-record builder from recreating a retiring host row. + if previous_plan is not None: + transition_previous = replace( + previous_plan, + hosts=tuple(host for host in previous_plan.hosts + if host.host_id not in missing), + ) + previous_hosts = { + host_id: host for host_id, host in old_hosts.items() + if host_id in requested_host_ids + } + gate = self._fabric_gate(plan) + record = self._desired_record( + plan, gate, state="deleting", + observed_revision=previous_observed_revision, + observed_digest=previous_observed_digest, + previous_hosts=previous_hosts, + previous_plan=transition_previous, + ) + retiring_rows_by_host: dict[str, list[dict[str, str]]] = {} + for row in _canonical_delete_members(retiring_members): + retiring_rows_by_host.setdefault( + row["host_id"], []).append(row) + for host in record.get("hosts", []): + if (isinstance(host, dict) + and isinstance(host.get("host_id"), str)): + host["retiring_members"] = list( + retiring_rows_by_host.get(host["host_id"], [])) + marker = { + "operation_id": plan.operation_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "all_host_ids": sorted(old_host_ids | requested_host_ids), + "required_host_ids": sorted(required_host_ids), + "preverified_host_ids": sorted(preverified), + "retiring_members": _canonical_delete_members( + retiring_members), + } + record["delete_target"] = marker + record["state"] = "deleting" + record["next_stage_allowed"] = False + record["private_dataplane_verified"] = False + record["updated_at"] = int(time.time()) + # Persist the target before any SSH/libvirt mutation. A crash + # can then resume the exact same operation and host closure. + self._save("vpc_desired", vpc_id, record) + self._save_children(record) + else: + if (marker["operation_id"] != plan.operation_id + or marker["desired_revision"] != plan.desired_revision + or marker["desired_digest"] != plan.desired_digest): + raise VpcError("VPC 删除目标已由另一操作占用", 409, + "vpc_delete_receipt_conflict") + durable_plan = _plan_from_record(previous) + if (durable_plan.operation_id != plan.operation_id + or durable_plan.desired_revision != plan.desired_revision + or durable_plan.desired_digest != plan.desired_digest + or durable_plan.canonical() != plan.canonical()): + raise VpcError("VPC 删除持久化目标不一致", 500, + "vpc_record_corrupt") + if (not set(marker["required_host_ids"]).issubset(plan.host_ids) + or not plan.host_ids.issubset( + set(marker["all_host_ids"]))): + raise VpcError("VPC 删除持久化宿主机闭集不一致", 500, + "vpc_record_corrupt") + if marker["retiring_members"] != _canonical_delete_members( + retiring_members): + raise VpcError("VPC 删除成员清单与持久化目标不一致", 500, + "vpc_record_corrupt") + record = dict(previous) + transition_previous = durable_plan + + required_host_ids = set(marker["required_host_ids"]) + observed_hosts = self._host_closure(record) + already_confirmed = { + host_id for host_id in required_host_ids + if (host_id in observed_hosts + and self._confirmed_removal( + observed_hosts[host_id], plan.desired_revision, + plan.desired_digest)) + } + pending_host_ids = required_host_ids - already_confirmed + if pending_host_ids: + gate = self._fabric_gate(plan) + # The submitted plan may still contain a deployment whose + # prior remove response was lost by PHP. Its exact durable + # absence proof classifies it as preverified; never contact it + # again. Reconcile a transient projection containing only + # genuinely pending host rows, then merge those observations + # back into the durable submitted plan. + execution_record = dict(record) + execution_hosts: list[dict[str, Any]] = [] + retiring_rows_by_host: dict[str, list[dict[str, str]]] = {} + for row in marker["retiring_members"]: + retiring_rows_by_host.setdefault( + row["host_id"], []).append(dict(row)) + for host in record.get("hosts", []): + if (not isinstance(host, Mapping) + or host.get("host_id") not in pending_host_ids): + continue + pending = dict(host) + # Persisted delete targets retain the submitted canonical + # host list, while the transient retiring marker forces + # the removal lane even after recover() cleared verified + # state and retiring_members had already been consumed. + pending["retiring"] = True + pending["retiring_members"] = list( + retiring_rows_by_host.get(pending["host_id"], [])) + execution_hosts.append(pending) + execution_record["hosts"] = execution_hosts + await self._reconcile_dataplane( + plan, execution_record, gate, transition_previous, + operation_id=plan.operation_id) + observations = { + host.get("host_id"): { + key: value for key, value in host.items() + if key != "retiring" + } + for host in execution_record.get("hosts", []) + if (isinstance(host, Mapping) + and isinstance(host.get("host_id"), str)) + } + record["hosts"] = [ + dict(observations[host["host_id"]]) + if (isinstance(host, Mapping) + and host.get("host_id") in observations) + else host + for host in record.get("hosts", []) + ] + # Preserve partial/uncertain evidence before deciding whether + # a receipt may be issued. + self._save("vpc_desired", vpc_id, record) + self._save_children(record) + observed_hosts = self._host_closure(record) + if not required_host_ids.issubset(observed_hosts) or not all( + self._confirmed_removal( + observed_hosts[host_id], plan.desired_revision, + plan.desired_digest) + for host_id in required_host_ids + ): + record["state"] = "uncertain" + record["next_stage_allowed"] = False + record["private_dataplane_verified"] = False + record["updated_at"] = int(time.time()) + self._save("vpc_desired", vpc_id, record) + deterministic = [ + (host_id, observed_hosts[host_id].get("last_error_code")) + for host_id in sorted(required_host_ids) + if (host_id in observed_hosts + and observed_hosts[host_id].get("last_error_code") + in VPC_DELETE_DETERMINISTIC_HOST_ERRORS) + ] + if deterministic: + raise VpcError( + "VPC 宿主机存在需要用户处理的确定性清理阻断", + 409, str(deterministic[0][1])) + raise VpcError("VPC 宿主机清理尚未全部确认", 503, + "vpc_delete_incomplete") + + receipt = { + "schema_version": VPC_SCHEMA_VERSION, + "vpc_id": vpc_id, + "operation_id": plan.operation_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "state": "deleted", + "deleted": True, + "readback": "confirmed", + "removed_host_ids": list(marker["all_host_ids"]), + "completed_at": int(time.time()), + } + # This ordering is the idempotency boundary. Never erase the + # evidence needed by a retry before its exact receipt is durable. + self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, receipt) + self._purge_deleted_vpc(vpc_id) + return self._public_delete_receipt(receipt) + async def apply(self, value: Mapping[str, Any]) -> dict[str, Any]: plan = parse_vpc_plan(value) async with self._lock(plan.vpc_id): + if self._load_delete_receipt(plan.vpc_id) is not None: + raise VpcError("VPC 已删除,不能重新应用旧标识", 409, + "vpc_deleted") previous = self._load("vpc_desired", plan.vpc_id) previous_plan: VpcPlan | None = None old_revision = plan.desired_revision if previous is not None: + if self._delete_marker(previous) is not None: + raise VpcError("VPC 正在删除,不能应用普通目标", 409, + "vpc_deleting") # Validate the durable prior target before accepting a newer # revision. Otherwise a malformed record could be silently # overwritten by a tenant retry, erasing the evidence needed @@ -2581,25 +3246,53 @@ class VpcCoordinator: async def reconcile(self, vpc_id: str) -> dict[str, Any]: _identifier(vpc_id, "VPC") + # Do not run a durable delete target through the ordinary apply + # reconciler: only ``delete`` can issue the exact positive receipt and + # purge the target after every required host reports absence. The + # first read is only an optimization; the row is rechecked under the + # VPC lock below to close the apply/delete race. + record = self._load("vpc_desired", vpc_id) + if record is not None and self._delete_marker(record) is not None: + return await self.delete( + vpc_id, self._delete_payload_from_record(record)) + delete_payload: dict[str, Any] | None = None async with self._lock(vpc_id): record = self._load("vpc_desired", vpc_id) if record is None: raise VpcError("VPC 不存在", 404, "vpc_not_found") - plan = _plan_from_record(record) - if record.get("desired_digest") != plan.desired_digest: - raise VpcError("VPC 计划摘要与持久化记录不一致", 500, - "vpc_record_corrupt") - gate = self._fabric_gate(plan) - # Reconcile is the durable retry path after a host response was - # lost or the engine restarted. Refreshing only the Fabric gate - # would leave private0/routes/nft in an unknown state forever. - await self._reconcile_dataplane(plan, record, gate, plan) - self._save("vpc_desired", vpc_id, record) - self._save_children(record) - return self._public(record) + if self._delete_marker(record) is not None: + delete_payload = self._delete_payload_from_record(record) + else: + plan = _plan_from_record(record) + if record.get("desired_digest") != plan.desired_digest: + raise VpcError("VPC 计划摘要与持久化记录不一致", 500, + "vpc_record_corrupt") + gate = self._fabric_gate(plan) + # Reconcile is the durable retry path after a host response + # was lost or the engine restarted. Refreshing only the + # Fabric gate would leave private0/routes/nft in an unknown + # state forever. + await self._reconcile_dataplane(plan, record, gate, plan) + self._save("vpc_desired", vpc_id, record) + self._save_children(record) + return self._public(record) + # The locked recheck may observe a delete target that was installed + # after the optimistic read. Release the non-reentrant lock before + # entering ``delete``, which acquires the same per-VPC lock itself. + assert delete_payload is not None + return await self.delete(vpc_id, delete_payload) def recover(self) -> None: """Fail closed after an engine restart until readback is supplied.""" + # Successful roots have no desired row left, but old deployments may + # still contain the pre-compaction receipt shape. Validate and reduce + # those tombstones before normal recovery; corrupt identities remain + # fail-closed for operator inspection. + for key in tuple(self.store.list(VPC_DELETE_RECEIPT_KIND)): + try: + self._load_delete_receipt(key) + except VpcError: + continue for key, raw in self.store.list("vpc_desired").items(): try: value = json.loads(raw) @@ -2626,8 +3319,22 @@ class VpcCoordinator: # Preserve the last durable observed evidence for audit and for a # future read-only reconciler. The gate is still closed until a # fresh observation proves that this revision remains converged. + marker = value.get("delete_target") + marker_revision = (marker.get("desired_revision") + if isinstance(marker, Mapping) else None) + marker_digest = (marker.get("desired_digest") + if isinstance(marker, Mapping) else None) for host in value.get("hosts", []): if isinstance(host, dict): + # An exact root-delete absence proof is terminal evidence, + # not a live readiness claim. Preserve it across an + # engine restart so retry only contacts hosts whose + # removal has not yet been confirmed. + if (isinstance(marker_revision, int) + and isinstance(marker_digest, str) + and self._confirmed_removal( + host, marker_revision, marker_digest)): + continue host.update({"state": "uncertain", "transport_verified": False, "private_dataplane_verified": False}) diff --git a/engine/virsh_panel/vpc_dataplane.py b/engine/virsh_panel/vpc_dataplane.py index 03afacef..a3c68eee 100644 --- a/engine/virsh_panel/vpc_dataplane.py +++ b/engine/virsh_panel/vpc_dataplane.py @@ -1737,6 +1737,25 @@ verify_routes_removed() {{ fi return 0 }} +vc_vpc_artifact_identity_check() {{ + identity=$1 + set -- $identity + [ "$#" -eq 3 ] || return 1 + [ "$1" = "vpc={plan.vpc.vpc_id}" ] || return 1 + artifact_revision=${{2#revision=}} + [ "$2" = "revision=$artifact_revision" ] || return 1 + case "$artifact_revision" in ''|*[!0-9]*) return 1;; esac + [ "$artifact_revision" -ge 1 ] || return 1 + artifact_digest=${{3#digest=}} + [ "$3" = "digest=$artifact_digest" ] || return 1 + printf '%s\n' "$artifact_digest" | grep -Eq '^[0-9a-f]{{64}}$' || return 1 + [ "$artifact_revision" -le {plan.desired_revision} ] || return 1 + if [ "$artifact_revision" = {plan.desired_revision} ] && + [ "$artifact_digest" != {_q(plan.desired_digest)} ]; then + return 1 + fi + return 0 +}} case "$PHASE" in preparing|routes_removed|manager_removed|bridge_removed|nft_removed) PRE_FILES=yes ;; files_removed|wg_merged|committed) PRE_FILES=no ;; @@ -1787,10 +1806,14 @@ done if [ -f "$NFT_FILE" ]; then [ "$($STAT -c '%u:%a' "$NFT_FILE" 2>/dev/null)" = 0:600 ] || exit 71 grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71 + nft_identity=$(sed -n '3{{s/^# //;p;}}' "$NFT_FILE") + vc_vpc_artifact_identity_check "$nft_identity" || exit 71 fi if [ -f "$SYSCTL_FILE" ]; then [ "$($STAT -c '%u:%a' "$SYSCTL_FILE" 2>/dev/null)" = 0:600 ] || exit 71 sed -n '1p' "$SYSCTL_FILE" | grep -Fqx '# virt-cloud-vpc-dataplane-v1' || exit 71 + sysctl_identity=$(sed -n '2{{s/^# //;p;}}' "$SYSCTL_FILE") + vc_vpc_artifact_identity_check "$sysctl_identity" || exit 71 fi if [ -f "$PROXY_FILE" ]; then [ "$($STAT -c '%u:%a' "$PROXY_FILE" 2>/dev/null)" = 0:600 ] || exit 71 @@ -1800,6 +1823,8 @@ fi if [ -f "$IF_FRAGMENT" ]; then [ "$($STAT -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] || exit 71 sed -n '1p' "$IF_FRAGMENT" | grep -Fqx '{ifup_marker}' || exit 71 + if_identity=$(sed -n '1{{s/^# {DATAPLANE_OWNER_MARKER} //;p;}}' "$IF_FRAGMENT") + vc_vpc_artifact_identity_check "$if_identity" || exit 71 grep -Fqx 'auto {r.bridge}' "$IF_FRAGMENT" || exit 71 grep -Fqx 'iface {r.bridge} inet manual' "$IF_FRAGMENT" || exit 71 if grep -Eq '^[[:space:]]*bridge_ports([[:space:]]|$)' "$IF_FRAGMENT"; then @@ -2087,11 +2112,14 @@ set_phase nft_removed if [ -e "$NFT_FILE" ] || [ -L "$NFT_FILE" ]; then [ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] || exit 71 grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71 - grep -Fqx '# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$NFT_FILE" || exit 71 + nft_identity=$(sed -n '3{{s/^# //;p;}}' "$NFT_FILE") + vc_vpc_artifact_identity_check "$nft_identity" || exit 71 fi if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then [ -f "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ] || exit 71 sed -n '1p' "$SYSCTL_FILE" | grep -Fqx '# virt-cloud-vpc-dataplane-v1' || exit 71 + sysctl_identity=$(sed -n '2{{s/^# //;p;}}' "$SYSCTL_FILE") + vc_vpc_artifact_identity_check "$sysctl_identity" || exit 71 fi if [ -e "$PROXY_FILE" ] || [ -L "$PROXY_FILE" ]; then [ -f "$PROXY_FILE" ] && [ ! -L "$PROXY_FILE" ] || exit 71 @@ -2100,6 +2128,8 @@ fi if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then [ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] || exit 71 sed -n '1p' "$IF_FRAGMENT" | grep -Fqx '{ifup_marker}' || exit 71 + if_identity=$(sed -n '1{{s/^# {DATAPLANE_OWNER_MARKER} //;p;}}' "$IF_FRAGMENT") + vc_vpc_artifact_identity_check "$if_identity" || exit 71 fi if [ -f "$STATE_FILE" ]; then [ ! -L "$STATE_FILE" ] || exit 71 diff --git a/web/app/Controllers/CloudVpcController.php b/web/app/Controllers/CloudVpcController.php index bf96da04..84282216 100644 --- a/web/app/Controllers/CloudVpcController.php +++ b/web/app/Controllers/CloudVpcController.php @@ -38,6 +38,13 @@ final class CloudVpcController return $this->json($response, $value, 202); } + public function delete($request, $response, $args) + { + $value = CloudVpcService::delete((string) $args['vpc_key'], + $this->actor($request), CloudInput::object($request)); + return $this->json($response, $value, 202); + } + public function addSubnet($request, $response, $args) { $value = CloudVpcService::addSubnet((string) $args['vpc_key'], diff --git a/web/app/Services/Cloud/CloudEngineClient.php b/web/app/Services/Cloud/CloudEngineClient.php index c6e207e5..4a28c2f2 100644 --- a/web/app/Services/Cloud/CloudEngineClient.php +++ b/web/app/Services/Cloud/CloudEngineClient.php @@ -140,7 +140,7 @@ final class CloudEngineClient $route ) === 1; $vpc = preg_match( - '#^/internal/v1/vpcs(?:/(?:apply|[0-9a-f]{32}(?:/reconcile)?))?$#D', + '#^/internal/v1/vpcs(?:/(?:apply|[0-9a-f]{32}(?:/(?:reconcile|delete))?))?$#D', $route ) === 1; $fabric = preg_match( @@ -256,6 +256,19 @@ final class CloudEngineClient 'vpc_private_wireguard_readback_mismatch' => 'VPC WireGuard 私网路由回读不一致,正在核对', 'vpc_private_dataplane_readback_mismatch' => 'VPC 私网数据面回读不一致,正在核对', 'vpc_operation_mismatch' => 'VPC 引擎结果身份不一致,正在核对', + 'vpc_delete_manifest_invalid' => 'VPC 删除成员清单无效,请刷新后重试', + 'vpc_delete_manifest_mismatch' => 'VPC 删除成员清单与当前目标不一致,正在核对', + 'vpc_delete_target_not_empty' => 'VPC 删除目标仍包含活动资源,请刷新后重试', + 'vpc_delete_host_set_mismatch' => 'VPC 删除宿主机集合不一致,正在核对', + 'vpc_delete_receipt_conflict' => 'VPC 删除回执与当前操作冲突,请联系管理员', + 'vpc_private_remove_requires_stopped' => 'VPC 成员虚拟机仍在运行,请先关机后重试删除', + 'vpc_private_remove_unsafe' => 'VPC 成员存在 managed-save 或原生快照,暂不能移除私网', + 'vpc_private_interface_conflict' => 'VPC 私网网卡身份不一致,请联系管理员核对', + 'vpc_seed_missing' => 'VPC 成员 cloud-init 配置盘缺失,无法确认私网清理', + 'vpc_seed_readback_invalid' => 'VPC 成员 cloud-init 配置盘无法核对,请联系管理员', + 'vpc_private_xml_invalid' => 'VPC 成员持久化 XML 无效,无法确认私网清理', + 'vpc_deleted' => 'VPC 已删除,不能重新应用旧目标', + 'vpc_deleting' => 'VPC 正在删除,不能应用普通变更', ][$value] ?? null; } @@ -444,7 +457,7 @@ final class CloudEngineClient ?array $payload = null, ?string $expectedOperationId = null): array { if (!preg_match( - '#^/internal/v1/vpcs(?:/(?:apply|[0-9a-f]{32}(?:/reconcile)?))?$#D', + '#^/internal/v1/vpcs(?:/(?:apply|[0-9a-f]{32}(?:/(?:reconcile|delete))?))?$#D', $resource )) { throw new CloudError('VPC 引擎资源路径无效', 422); diff --git a/web/app/Services/Cloud/CloudPrivateCidrAllocator.php b/web/app/Services/Cloud/CloudPrivateCidrAllocator.php index dd6c4b2c..15e107d7 100644 --- a/web/app/Services/Cloud/CloudPrivateCidrAllocator.php +++ b/web/app/Services/Cloud/CloudPrivateCidrAllocator.php @@ -70,12 +70,37 @@ final class CloudPrivateCidrAllocator 'allocation_key', $allocationKey )->lockForUpdate()->first(); if ($existing !== null) { + if ((string) $existing->state === 'released' + && $existing->active_cidr === null) { + // An allocation key is a durable identity, not a + // resurrection token. Reusing a released row + // would return a CIDR whose /24 claims no longer + // exist and could already belong to another VPC. + throw new CloudError( + 'CIDR 分配请求已经释放,请使用新的请求标识', 409 + ); + } + if ((string) $existing->state === 'released' + || $existing->active_cidr === null) { + throw new CloudError('CIDR 分配账本状态无效', 503); + } + if (!in_array((string) $existing->state, + self::LIVE_STATES, true)) { + throw new CloudError('CIDR 分配账本状态无效', 503); + } self::assertIdempotent( $existing, $providedVpcKey === null ? (string) $existing->vpc_key : $vpcKey, $userId, $regionId, $requestedCidr, $prefixLength, ); + // An idempotent retry is allowed to return only the + // same intact allocation. Without the exact quantum + // closure check, a live row whose interval or claim + // ledger drifted could be handed back to VPC create + // even though its CIDR is no longer exclusively + // reserved. + self::assertClaimLedger($existing); return $existing; } $sameVpc = CloudPrivateNetwork::where('vpc_key', $vpcKey) @@ -184,6 +209,7 @@ final class CloudPrivateCidrAllocator || (int) $network->revision !== $expectedRevision) { throw new CloudError('私网 CIDR 状态已被其他操作更新,请重新读取', 409); } + self::assertClaimLedger($network); self::deleteClaims($allocationKey); return self::markReleased($network, $expectedRevision); }); @@ -216,6 +242,7 @@ final class CloudPrivateCidrAllocator throw new CloudError('私网 CIDR 状态已被其他操作更新,请重新读取', 409); } if ($to === 'released') { + self::assertClaimLedger($network); self::deleteClaims($allocationKey); return self::markReleased($network, $expectedRevision); } @@ -247,6 +274,47 @@ final class CloudPrivateCidrAllocator CloudPrivateCidrClaim::where('allocation_key', $allocationKey)->delete(); } + /** Prove the exact /24 ownership ledger before making a CIDR reusable. */ + private static function assertClaimLedger(CloudPrivateNetwork $network): void + { + $prefix = (int) $network->prefix_length; + if ($prefix < self::DEFAULT_MIN_PREFIX_LENGTH + || $prefix > self::QUANTUM_PREFIX_LENGTH) { + throw new CloudError('私网 CIDR 声明账本不一致,已停止释放', 503); + } + try { + $canonical = self::normalizeCidr((string) $network->cidr, $prefix); + } catch (\Throwable) { + throw new CloudError('私网 CIDR 声明账本不一致,已停止释放', 503); + } + if ((string) $network->cidr !== $canonical['cidr'] + || (string) ($network->active_cidr ?? '') !== $canonical['cidr'] + || (int) $network->network_address !== $canonical['network'] + || (int) $network->broadcast_address !== $canonical['broadcast'] + || !self::isRfc1918($canonical['network'], $canonical['broadcast'])) { + throw new CloudError('私网 CIDR 声明账本不一致,已停止释放', 503); + } + $blockSize = self::blockSize(self::QUANTUM_PREFIX_LENGTH); + $count = 1 << (self::QUANTUM_PREFIX_LENGTH - $prefix); + $expected = []; + for ($index = 0; $index < $count; $index++) { + $expected[] = self::description( + $canonical['network'] + ($index * $blockSize), + self::QUANTUM_PREFIX_LENGTH + )['cidr']; + } + sort($expected, SORT_STRING); + $actual = CloudPrivateCidrClaim::where( + 'allocation_key', (string) $network->allocation_key + )->orderBy('block_cidr')->lockForUpdate()->pluck('block_cidr')->map( + static fn ($cidr): string => (string) $cidr + )->all(); + sort($actual, SORT_STRING); + if ($actual !== $expected) { + throw new CloudError('私网 CIDR 声明账本不一致,已停止释放', 503); + } + } + /** @param array{network:int,broadcast:int,cidr:string,prefix:int} $candidate */ private static function claimCandidate(array $candidate, string $allocationKey): void { diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php index cb08564f..7ef331a2 100644 --- a/web/app/Services/Cloud/CloudVpcService.php +++ b/web/app/Services/Cloud/CloudVpcService.php @@ -4,6 +4,7 @@ namespace App\Services\Cloud; use App\Models\CloudHost; use App\Models\CloudInstance; +use App\Models\CloudPrivateNetwork; use App\Models\CloudRegion; use App\Models\CloudRegionFabric; use App\Models\CloudVpc; @@ -50,7 +51,7 @@ final class CloudVpcService /** Return VPCs visible to an actor without contacting a host. */ public static function listing(array $actor): array { - $query = CloudVpc::query(); + $query = CloudVpc::query()->whereIn('state', self::PUBLIC_STATES); if (!self::isGlobal($actor)) { $query->where('user_id', (int) ($actor['user_id'] ?? 0)); } @@ -298,6 +299,74 @@ final class CloudVpcService } } + /** + * Queue destruction of the complete VPC root and every private child. + * + * The CIDR remains reserved until the dedicated engine delete endpoint + * returns an exact, durable deletion receipt. A retry with the same + * request key is read-only; a second delete identity cannot supersede an + * in-flight teardown. + */ + public static function delete(string $vpcKey, array $actor, array $input): array + { + CloudInput::only($input, ['confirmation_name', 'request_key']); + $confirmationName = $input['confirmation_name'] ?? null; + if (!is_string($confirmationName)) { + throw new CloudError('请输入 VPC 名称确认删除'); + } + $key = self::requestKey($input['request_key'] ?? null); + $requestHash = self::requestHash('vpc_delete', [ + 'confirmation_name' => $confirmationName, + ]); + $vpc = self::findOwned($vpcKey, $actor, true); + return $vpc->getConnection()->transaction(function () use ( + $vpc, $actor, $confirmationName, $key, $requestHash + ): array { + $locked = self::lockOwnedForDelete((string) $vpc->vpc_key, $actor); + if (!hash_equals((string) $locked->name, $confirmationName)) { + throw new CloudError('确认名称与 VPC 名称不一致'); + } + if (self::hasOperation($locked, $key, 'vpc_delete', $requestHash)) { + return self::documentView($locked, $actor); + } + if ((string) $locked->state === 'deleted') { + // DELETE remains idempotent after the original operation has + // completed, even when the caller generated a fresh request + // key after losing the prior HTTP response. + return self::documentView($locked, $actor); + } + if ((string) $locked->state === 'deleting') { + throw new CloudError('VPC 删除正在核对,请勿重复提交', 409); + } + + $now = time(); + foreach ([ + CloudVpcRoute::class, + CloudVpcAclRule::class, + CloudVpcPrivateIp::class, + CloudVpcMember::class, + CloudVpcSubnet::class, + ] as $childModel) { + $rows = $childModel::where('vpc_id', $locked->id) + ->where('state', '<>', 'deleted')->lockForUpdate()->get(); + foreach ($rows as $row) { + $row->state = 'deleting'; + $row->updated_at = $now; + $row->save(); + } + } + $locked->state = 'deleting'; + $locked->sync_state = 'pending'; + $locked->last_error_code = null; + $locked->updated_at = $now; + $locked->save(); + $result = self::touchDesiredLocked( + $locked, 'vpc_delete', $key, $requestHash + ); + return self::documentView($result['vpc'], $actor); + }, 3); + } + public static function addSubnet(string $vpcKey, array $actor, array $input): array { CloudInput::only($input, ['cidr', 'gateway_ip', 'request_key']); @@ -833,9 +902,20 @@ final class CloudVpcService throw new CloudError('VPC 不存在', 404); } $operation = CloudVpcOperation::where('vpc_id', $locked->id) - ->whereIn('status', $allowDeferredRetry - ? ['queued', 'running', 'uncertain', 'waiting_observed'] - : ['queued', 'running', 'uncertain']) + ->where(function ($query) use ($allowDeferredRetry): void { + $query->whereIn('status', $allowDeferredRetry + ? ['queued', 'running', 'uncertain', 'waiting_observed'] + : ['queued', 'running', 'uncertain']); + if ($allowDeferredRetry) { + // A definitive delete rejection stays fail-closed, + // but an explicit reconcile may retry the exact same + // operation after an operator repairs engine state. + $query->orWhere(function ($failed): void { + $failed->where('status', 'failed') + ->where('kind', 'vpc_delete'); + }); + } + }) ->orderByDesc('desired_revision')->lockForUpdate()->first(); if ($operation === null) { throw new CloudError('VPC 没有待核对的目标版本', 409); @@ -873,7 +953,8 @@ final class CloudVpcService $operation->lease_until = 0; $operation->updated_at = $corruptAt; $operation->save(); - $locked->state = 'degraded'; + $locked->state = (string) $operation->kind === 'vpc_delete' + ? 'deleting' : 'degraded'; $locked->sync_state = 'failed'; $locked->last_error_code = 'vpc_operation_corrupt'; $locked->last_failure_at = $corruptAt; @@ -896,6 +977,34 @@ final class CloudVpcService [$operation, $payload] = $claim; $leaseToken = (string) $operation->lease_token; try { + if ((string) $operation->kind === 'vpc_delete') { + $result = (new CloudEngineClient())->vpc( + 'POST', '/internal/v1/vpcs/' . $vpcKey . '/delete', + $actor, $payload, (string) $operation->operation_id + ); + $expectedRevision = (int) ($payload['desired_revision'] ?? 0); + $expectedDigest = (string) ($payload['desired_digest'] ?? ''); + if (!is_array($result) + || (string) ($result['vpc_id'] ?? '') !== $vpcKey + || (string) ($result['operation_id'] ?? '') + !== (string) $operation->operation_id + || (int) ($result['desired_revision'] ?? 0) !== $expectedRevision + || !hash_equals($expectedDigest, + (string) ($result['desired_digest'] ?? '')) + || ($result['deleted'] ?? false) !== true + || (string) ($result['state'] ?? '') !== 'deleted' + || (string) ($result['readback'] ?? '') !== 'confirmed' + || !is_array($result['removed_host_ids'] ?? null) + || !is_int($result['completed_at'] ?? null) + || (int) $result['completed_at'] < 1) { + throw new CloudError('引擎 VPC 删除回执不完整,正在核对', 502); + } + self::recordDeleted( + $vpcKey, $expectedRevision, $expectedDigest, $result, + (string) $operation->operation_id, $leaseToken + ); + return self::show($vpcKey, $actor); + } $result = (new CloudEngineClient())->vpc('POST', '/internal/v1/vpcs/apply', $actor, $payload, (string) $operation->operation_id); $expectedRevision = (int) ($payload['desired_revision'] ?? 0); @@ -976,7 +1085,9 @@ final class CloudVpcService $row->save(); if ((string) $root->last_operation_id === (string) $row->operation_id && (int) $root->desired_revision === (int) $row->desired_revision) { - $root->state = $status === 'uncertain' ? 'uncertain' : 'degraded'; + $root->state = (string) $row->kind === 'vpc_delete' + ? 'deleting' + : ($status === 'uncertain' ? 'uncertain' : 'degraded'); $root->sync_state = $status === 'uncertain' ? 'uncertain' : 'failed'; $root->last_error_code = $row->error_code; $root->last_failure_at = $now; @@ -988,6 +1099,225 @@ final class CloudVpcService } } + /** Finalize a root deletion only from an exact durable engine receipt. */ + public static function recordDeleted(string $vpcKey, int $revision, string $digest, + array $receipt, string $operationId, ?string $leaseToken = null): void + { + self::assertKey($vpcKey, 'VPC'); + if ($revision < 1 || preg_match('/^[a-f0-9]{64}$/D', $digest) !== 1 + || preg_match(self::KEY_PATTERN, $operationId) !== 1) { + throw new CloudError('VPC 删除回执版本无效', 502); + } + $connection = (new CloudVpc())->getConnection(); + $connection->transaction(function () use ( + $vpcKey, $revision, $digest, $receipt, $operationId, $leaseToken + ): void { + $vpc = CloudVpc::where('vpc_key', $vpcKey)->lockForUpdate()->first(); + if ($vpc === null) { + throw new CloudError('VPC 不存在', 404); + } + if ((string) $vpc->state !== 'deleting' + || (int) $vpc->desired_revision !== $revision + || !hash_equals((string) $vpc->desired_digest, $digest) + || (string) $vpc->last_operation_id !== $operationId) { + throw new CloudError('VPC 删除回执不是当前目标版本', 409); + } + $operationQuery = CloudVpcOperation::where('operation_id', $operationId) + ->where('vpc_id', $vpc->id)->where('kind', 'vpc_delete'); + if ($leaseToken !== null) { + $operationQuery->where('lease_token', $leaseToken) + ->where('status', 'running'); + } + $operation = $operationQuery->lockForUpdate()->first(); + if ($operation === null + || (int) $operation->desired_revision !== $revision) { + throw new CloudError('VPC 删除操作记录不存在', 502); + } + $payload = json_decode((string) $operation->payload_json, true); + $currentDeleteDocument = self::deletionDocument($vpc); + $payloadDocument = is_array($payload) ? $payload : []; + unset($payloadDocument['operation_id'], $payloadDocument['desired_digest']); + if (!is_array($payload) + || (string) ($payload['operation_id'] ?? '') !== $operationId + || !hash_equals($digest, + (string) ($payload['desired_digest'] ?? '')) + || !is_array($payload['hosts'] ?? null) + || !is_array($payload['retiring_members'] ?? null) + || self::canonicalJson($payloadDocument) + !== self::canonicalJson($currentDeleteDocument) + || self::canonicalJson([ + 'retiring_members' => $payload['retiring_members'], + ]) !== self::canonicalJson([ + 'retiring_members' => self::retiringMembersForDeletion($vpc), + ]) + || !hash_equals($digest, + hash('sha256', self::canonicalJson($currentDeleteDocument)))) { + throw new CloudError('VPC 删除成员闭集与操作记录不一致', 409); + } + if ((string) ($receipt['vpc_id'] ?? '') !== $vpcKey + || (string) ($receipt['operation_id'] ?? '') !== $operationId + || (int) ($receipt['desired_revision'] ?? 0) !== $revision + || !hash_equals($digest, (string) ($receipt['desired_digest'] ?? '')) + || ($receipt['deleted'] ?? false) !== true + || (string) ($receipt['state'] ?? '') !== 'deleted' + || (string) ($receipt['readback'] ?? '') !== 'confirmed' + || !is_int($receipt['completed_at'] ?? null) + || (int) $receipt['completed_at'] < 1) { + throw new CloudError('VPC 删除回执身份不一致', 502); + } + + $deployments = CloudVpcHostDeployment::where('vpc_id', $vpc->id) + ->orderBy('host_id')->lockForUpdate()->get(); + $historicalHostIds = $deployments->map( + static fn ($row): string => (string) $row->host_id + )->unique()->values()->all(); + sort($historicalHostIds, SORT_STRING); + $pendingHostIds = $deployments->filter( + static fn ($row): bool => (string) $row->state !== 'deleted' + )->map(static fn ($row): string => (string) $row->host_id) + ->unique()->values()->all(); + sort($pendingHostIds, SORT_STRING); + $payloadHostIds = array_map( + static fn (array $host): string => (string) $host['host_id'], + $payload['hosts'] + ); + sort($payloadHostIds, SORT_STRING); + if ($payloadHostIds !== $pendingHostIds) { + throw new CloudError('VPC 删除待清宿主机闭集已变化', 409); + } + $removedHostIds = $receipt['removed_host_ids'] ?? null; + if (!is_array($removedHostIds) || !array_is_list($removedHostIds)) { + throw new CloudError('VPC 删除回执宿主机集合无效', 502); + } + foreach ($removedHostIds as $hostId) { + if (!is_string($hostId) + || preg_match('/^[1-9][0-9]*$/D', $hostId) !== 1) { + throw new CloudError('VPC 删除回执宿主机集合无效', 502); + } + } + if (count($removedHostIds) !== count(array_unique($removedHostIds))) { + throw new CloudError('VPC 删除回执宿主机集合重复', 502); + } + $canonicalRemovedHostIds = $removedHostIds; + sort($canonicalRemovedHostIds, SORT_STRING); + if ($removedHostIds !== $canonicalRemovedHostIds + || array_diff($payloadHostIds, $canonicalRemovedHostIds) !== [] + || array_diff($canonicalRemovedHostIds, $historicalHostIds) !== []) { + throw new CloudError('VPC 删除回执缺少待清宿主机证明或包含未知宿主机', 502); + } + + $now = time(); + foreach ([ + CloudVpcSubnet::class, + CloudVpcPrivateIp::class, + CloudVpcMember::class, + CloudVpcRoute::class, + CloudVpcAclRule::class, + ] as $childModel) { + $rows = $childModel::where('vpc_id', $vpc->id) + ->lockForUpdate()->get(); + foreach ($rows as $row) { + if ((string) $row->state === 'deleted') { + continue; + } + if ((string) $row->state !== 'deleting') { + throw new CloudError('VPC 删除账本仍包含活动资源', 409); + } + } + } + foreach ($deployments as $deployment) { + if ((string) $deployment->state === 'deleted') { + // Preserve the revision/digest of the earlier exact + // absence proof when this engine receipt legitimately + // omits an already-retired historical host. + continue; + } + if ((string) $deployment->state !== 'deleting') { + throw new CloudError('VPC 删除账本仍包含活动宿主机', 409); + } + $deployment->observed_revision = $revision; + $deployment->observed_digest = $digest; + $deployment->state = 'deleted'; + $deployment->last_error_code = null; + $deployment->last_success_at = $now; + $deployment->updated_at = $now; + $deployment->save(); + } + + $network = CloudPrivateNetwork::where( + 'id', (int) $vpc->private_network_id + )->lockForUpdate()->first(); + if ($network === null + || !hash_equals((string) $network->vpc_key, $vpcKey) + || (int) $network->user_id !== (int) $vpc->user_id + || (int) $network->region_id !== (int) $vpc->region_id + || (string) $network->cidr !== (string) $vpc->cidr + || (string) ($network->active_cidr ?? '') !== (string) $vpc->cidr + || !in_array((string) $network->state, + ['active', 'applying', 'degraded', 'uncertain'], true) + || (int) $network->revision < 1 + || preg_match(self::KEY_PATTERN, + (string) $network->allocation_key) !== 1) { + throw new CloudError('VPC 私网 CIDR 删除账本不一致', 409); + } + $released = CloudPrivateCidrAllocator::release( + (string) $network->allocation_key, (int) $network->revision + ); + if ((int) $released->id !== (int) $network->id + || (string) $released->state !== 'released' + || $released->active_cidr !== null + || (int) $released->revision !== (int) $network->revision + 1) { + throw new CloudError('VPC 私网 CIDR 未确认释放', 409); + } + + // The signed operation payload is the durable deletion manifest + // and the engine receipt is the absence proof. Once both and the + // CIDR CAS are exact, remove child ledgers so their global unique + // keys (notably member.instance_id) cannot block a VM from + // joining a new VPC. Root, host deployments and operations remain + // as the idempotent audit tombstone. + foreach ([ + CloudVpcRoute::class, + CloudVpcAclRule::class, + CloudVpcPrivateIp::class, + CloudVpcMember::class, + CloudVpcSubnet::class, + ] as $childModel) { + $childModel::where('vpc_id', $vpc->id)->delete(); + } + + $safeReceipt = [ + 'vpc_id' => $vpcKey, + 'operation_id' => $operationId, + 'desired_revision' => $revision, + 'desired_digest' => $digest, + 'state' => 'deleted', + 'deleted' => true, + 'readback' => 'confirmed', + 'removed_host_ids' => $canonicalRemovedHostIds, + 'completed_at' => (int) $receipt['completed_at'], + ]; + $operation->observed_json = json_encode($safeReceipt, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + $operation->status = 'succeeded'; + $operation->error_code = null; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->finished_at = $now; + $operation->updated_at = $now; + $operation->save(); + + $vpc->observed_revision = $revision; + $vpc->observed_digest = $digest; + $vpc->state = 'deleted'; + $vpc->sync_state = 'applied'; + $vpc->last_error_code = null; + $vpc->last_success_at = $now; + $vpc->updated_at = $now; + $vpc->save(); + }, 3); + } + /** Apply an exact engine observation and enforce the all-host gate. */ public static function recordObserved(string $vpcKey, int $revision, string $digest, array $observed, string $operationId, ?string $leaseToken = null): void @@ -1199,9 +1529,20 @@ final class CloudVpcService } foreach ($required as $hostId => $deployment) { $item = $hosts[(string) $hostId] ?? null; + $deleting = (string) $deployment->state === 'deleting'; + if ($deleting) { + $hasRemovalTarget = true; + } if (!is_array($item)) { $allActive = false; $allLiveHostsExact = false; + if ($deleting) { + // A missing deleting host is failed removal proof, + // not an absent target. Without this assignment a + // second host's valid receipt could incorrectly make + // a partial multi-host teardown look complete. + $allRemovalProof = false; + } // A deleting host remains a tombstone until an exact // desired revision proves that the empty-host target was // accepted. Converting it to uncertain would make a @@ -1223,7 +1564,6 @@ final class CloudVpcService && ($item['removed'] ?? false) !== true && $itemRevision <= $revision && $itemRevision === $revision && hash_equals($digest, $itemDigest); - $deleting = (string) $deployment->state === 'deleting'; $removalActive = $deleting && ($item['removed'] ?? false) === true && ($item['removal_verified'] ?? false) === true @@ -1233,7 +1573,6 @@ final class CloudVpcService && $itemRevision === $revision && hash_equals($digest, $itemDigest); if ($deleting) { - $hasRemovalTarget = true; if (!$removalActive) { $allRemovalProof = false; } @@ -1559,6 +1898,102 @@ final class CloudVpcService ]); } + /** Build the signed terminal target used only by the root delete lane. */ + private static function deletionDocument(CloudVpc $vpc): array + { + $document = self::desiredDocument($vpc); + foreach (['subnets', 'members', 'routes', 'acls'] as $field) { + if (($document[$field] ?? null) !== []) { + throw new CloudError('VPC 删除目标仍包含活动资源', 409); + } + } + $retiringMembers = self::retiringMembersForDeletion($vpc); + // Only unresolved deployments are removal targets in the desired + // plan. A historical `deleted` host has already supplied exact + // absence proof and must not be reintroduced as a normal empty host + // by the engine parser. The final receipt must cover every pending + // host and may repeat, but need not repeat, a prior historical proof; + // recordDeleted() still rejects any host PHP has never deployed. + $hostIds = CloudVpcHostDeployment::where('vpc_id', $vpc->id) + ->where('state', '<>', 'deleted') + ->orderBy('host_id')->pluck('host_id')->map( + static fn ($hostId): string => (string) $hostId + )->unique()->values()->all(); + // Python canonicalizes the typed host_id as a string, so numeric SQL + // ordering (2, 11) would otherwise produce a different signed digest + // from the engine's lexical ordering (11, 2). + sort($hostIds, SORT_STRING); + $hostSet = array_fill_keys($hostIds, true); + foreach ($retiringMembers as $member) { + if (!isset($hostSet[(string) $member['host_id']])) { + throw new CloudError('VPC 删除成员缺少宿主机部署账本', 409); + } + } + $document['hosts'] = array_map(static fn (string $hostId): array => [ + 'host_id' => $hostId, 'members' => [], + ], $hostIds); + $document['retiring_members'] = $retiringMembers; + return self::canonicalize($document); + } + + /** + * Preserve the exact private NIC identities that may have reached a host + * before an earlier engine response became durable. Root deletion signs + * this closed set so the engine can remove/read back every possible NIC. + */ + private static function retiringMembersForDeletion(CloudVpc $vpc): array + { + $members = CloudVpcMember::where('vpc_id', $vpc->id) + ->where('state', '<>', 'deleted')->orderBy('member_key')->get(); + if ($members->isEmpty()) { + return []; + } + $instances = CloudInstance::whereIn( + 'id', $members->pluck('instance_id')->map( + static fn ($id): int => (int) $id + )->unique()->values()->all() + )->get()->keyBy('id'); + $privateIps = CloudVpcPrivateIp::where('vpc_id', $vpc->id) + ->whereIn('member_id', $members->pluck('id')->map( + static fn ($id): int => (int) $id + )->all())->get()->keyBy('member_id'); + $result = []; + $macs = []; + foreach ($members as $member) { + $memberKey = (string) $member->member_key; + $instance = $instances[(int) $member->instance_id] ?? null; + $privateIp = $privateIps[(int) $member->id] ?? null; + $uuid = strtolower((string) ($instance->uuid ?? '')); + $mac = self::privateMacForMember((string) $vpc->vpc_key, $memberKey); + if (preg_match(self::KEY_PATTERN, $memberKey) !== 1 + || preg_match('/^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/D', + $uuid) !== 1 + || $privateIp === null + || (int) $member->private_ip_id !== (int) $privateIp->id + || (int) $privateIp->member_id !== (int) $member->id + || (int) $privateIp->instance_id !== (int) $member->instance_id + || (string) $privateIp->private_ip !== (string) $member->private_ip + || (string) $privateIp->state === 'deleted' + || filter_var((string) $member->private_ip, + FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false + || $mac === null || isset($macs[$mac])) { + throw new CloudError('VPC 删除成员身份账本不一致', 409); + } + $macs[$mac] = true; + $result[] = [ + 'member_id' => $memberKey, + 'instance_uuid' => $uuid, + 'host_id' => (string) $member->host_id, + 'private_ip_cidr' => (string) $member->private_ip . '/32', + 'private_mac' => $mac, + ]; + } + usort($result, static fn (array $left, array $right): int => + strcmp($left['member_id'], $right['member_id']) + ); + return $result; + } + /** * A VM cannot enter destructive deletion while a VPC member row still * exists. A `deleting` row is an explicit tombstone that remains @@ -2168,6 +2603,34 @@ final class CloudVpcService || $memberDigest !== $expectedDigest) { return false; } + if ((string) $vpc->state === 'deleting') { + // Root deletion owns the terminal transition and its signed + // retiring_members closure must remain immutable until the + // dedicated engine receipt is finalized. A concurrent VM + // power/delete worker may still contribute useful exact + // readback evidence, but it cannot shrink that manifest by + // closing the member/lease tombstones independently. + $member->observed_revision = max( + 0, (int) ($state['observed_revision'] ?? 0) + ); + $member->observed_digest = self::safeDigest( + $state['observed_digest'] ?? null + ); + $member->state = 'deleting'; + $member->last_error_code = $ok ? null + : (is_string($validation['error_code'] ?? null) + ? self::safeErrorCode((string) $validation['error_code']) + : 'vpc_private_observation_missing'); + $member->updated_at = time(); + $member->save(); + $private->state = 'deleting'; + $private->observed_revision = $member->observed_revision; + $private->observed_digest = $member->observed_digest; + $private->last_error_code = $member->last_error_code; + $private->updated_at = time(); + $private->save(); + return true; + } // `last_operation_id` on a member is the VPC proof marker. A // VM lifecycle job may refresh the private lease, but must not // replace a successful root operation marker with its unrelated @@ -2234,7 +2697,7 @@ final class CloudVpcService } $revision = (int) $vpc->desired_revision + 1; $vpc->desired_revision = $revision; - $vpc->state = 'applying'; + $vpc->state = $kind === 'vpc_delete' ? 'deleting' : 'applying'; $vpc->sync_state = 'pending'; $vpc->last_error_code = null; $vpc->updated_at = time(); @@ -2303,7 +2766,8 @@ final class CloudVpcService $row->save(); } } - $document = self::desiredDocument($vpc); + $document = $kind === 'vpc_delete' + ? self::deletionDocument($vpc) : self::desiredDocument($vpc); $digest = hash('sha256', self::canonicalJson($document)); $vpc->desired_digest = $digest; $vpc->updated_at = time(); @@ -2570,6 +3034,27 @@ final class CloudVpcService private static function lockOwned(string $key, array $actor): CloudVpc { + self::assertKey($key, 'VPC'); + $query = CloudVpc::where('vpc_key', $key); + if (!self::isGlobal($actor)) { + $query->where('user_id', (int) ($actor['user_id'] ?? 0)); + } + $vpc = $query->lockForUpdate()->first() + ?? throw new CloudError('VPC 不存在或无权访问', 404); + if (!in_array((string) $vpc->state, self::LIVE_STATES, true)) { + throw new CloudError( + (string) $vpc->state === 'deleted' + ? 'VPC 已删除' : 'VPC 正在删除,不能再修改', + 409 + ); + } + return $vpc; + } + + /** Lock a root for the delete state machine without reopening mutation. */ + private static function lockOwnedForDelete(string $key, array $actor): CloudVpc + { + self::assertKey($key, 'VPC'); $query = CloudVpc::where('vpc_key', $key); if (!self::isGlobal($actor)) { $query->where('user_id', (int) ($actor['user_id'] ?? 0)); diff --git a/web/app/Services/UserDataDeletionService.php b/web/app/Services/UserDataDeletionService.php index 6ede312e..7ae6ef0e 100644 --- a/web/app/Services/UserDataDeletionService.php +++ b/web/app/Services/UserDataDeletionService.php @@ -28,6 +28,15 @@ class UserDataDeletionService private const EMAIL_REFERENCES = [ ['checkout_apply', 'email'], ['email_verify', 'email'], ['ss_password_reset', 'email'], ]; + private const VPC_HISTORY_TABLES = [ + 'cloud_vpc_routes', + 'cloud_vpc_acl_rules', + 'cloud_vpc_private_ips', + 'cloud_vpc_members', + 'cloud_vpc_subnets', + 'cloud_vpc_host_deployments', + 'cloud_vpc_operations', + ]; private $schemaCache = []; public function deleteUsers(array $userIds, $cleanupCaches = true) @@ -55,13 +64,12 @@ class UserDataDeletionService ->where('status', '!=', 'deleted')->exists()) { throw new \RuntimeException('用户仍有云服务器,请先删除或转交实例后再删除账户'); } - // VPC roots own the private CIDR, member ledger and durable - // desired/observed operations. Phase 3 has no root-delete or - // data-plane readback endpoint, so deleting an account while any - // VPC/network reservation remains would leave an unreachable - // tenant plan that the dedicated VPC worker could still apply. + // Only a fully observed root tombstone may be removed with the + // account. Any live/deleting root remains owned by the dedicated + // VPC worker and must keep the account reachable. if ($connection->getSchemaBuilder()->hasTable('cloud_vpcs') - && $connection->table('cloud_vpcs')->whereIn('user_id', $ids)->exists()) { + && $connection->table('cloud_vpcs')->whereIn('user_id', $ids) + ->where('state', '<>', 'deleted')->exists()) { throw new \RuntimeException('用户仍有 VPC 私网,请先清理 VPC 后再删除账户'); } if ($this->hasColumns($connection, 'cloud_private_networks', ['user_id', 'state']) @@ -69,6 +77,7 @@ class UserDataDeletionService ->whereNotIn('state', ['released'])->exists()) { throw new \RuntimeException('用户仍有私网地址预留,请先完成 VPC 清理后再删除账户'); } + $this->assertVpcHistoryCleanable($connection, $ids); if ($connection->getSchemaBuilder()->hasTable('cloud_orders') && $connection->table('cloud_orders')->whereIn('user_id', $ids) ->whereIn('status', ['awaiting_payment', 'pending', 'queued', @@ -111,7 +120,8 @@ class UserDataDeletionService // jobs before their parent rows so encrypted owner/VM payloads do // not survive as orphaned administrator audit records. $resourceJobRows = $this->deleteCloudResourceJobs($connection, $ids); - $rows = $deletedMailRows + $resourceJobRows + $vpcRows = $this->deleteVpcHistory($connection, $ids); + $rows = $deletedMailRows + $resourceJobRows + $vpcRows + $this->deleteRelatedRecords($connection, $ids, $emails); $deleted = (int) User::whereIn('id', $ids)->where('is_admin', 0)->delete(); if ($deleted !== count($ids)) { @@ -218,6 +228,111 @@ class UserDataDeletionService })->delete(); } + /** + * Verify that account cleanup cannot hide an unreleased or corrupt VPC + * allocation. Released allocations must have no remaining quantum + * claims before either their audit row or owner is removed. + */ + private function assertVpcHistoryCleanable($connection, array $userIds): void + { + $schema = $connection->getSchemaBuilder(); + $vpcTableExists = $schema->hasTable('cloud_vpcs'); + $networkTableExists = $schema->hasTable('cloud_private_networks'); + if (!$vpcTableExists && !$networkTableExists) { + return; + } + $hasVpcs = $vpcTableExists && $this->hasColumns($connection, 'cloud_vpcs', [ + 'id', 'vpc_key', 'user_id', 'private_network_id', 'state', + ]); + $hasNetworks = $networkTableExists && $this->hasColumns( + $connection, 'cloud_private_networks', [ + 'id', 'vpc_key', 'allocation_key', 'user_id', 'state', 'active_cidr', + ] + ); + if (!$hasVpcs || !$hasNetworks) { + throw new \RuntimeException('VPC 私网删除账本不完整,账户删除已停止'); + } + $vpcs = $connection->table('cloud_vpcs')->whereIn('user_id', $userIds) + ->lockForUpdate()->get(); + $networks = $connection->table('cloud_private_networks') + ->whereIn('user_id', $userIds)->lockForUpdate()->get(); + $networkById = []; + $allocationKeys = []; + foreach ($networks as $network) { + if ((string) $network->state !== 'released' + || $network->active_cidr !== null) { + throw new \RuntimeException('用户仍有私网地址预留,请先完成 VPC 清理后再删除账户'); + } + $networkById[(int) $network->id] = $network; + $allocationKeys[] = (string) $network->allocation_key; + } + foreach ($vpcs as $vpc) { + $network = $networkById[(int) $vpc->private_network_id] ?? null; + if ((string) $vpc->state !== 'deleted' + || $network === null + || (int) $network->user_id !== (int) $vpc->user_id + || (string) $network->vpc_key !== (string) $vpc->vpc_key) { + throw new \RuntimeException('VPC 私网删除账本不一致,账户删除已停止'); + } + } + $vpcIds = $vpcs->map(static fn ($vpc): int => (int) $vpc->id) + ->values()->all(); + if ($vpcIds !== []) { + foreach (self::VPC_HISTORY_TABLES as $table) { + if (!$this->hasColumns($connection, $table, ['vpc_id'])) { + throw new \RuntimeException( + 'VPC 私网子账本不完整,账户删除已停止' + ); + } + } + } + $networkIds = array_keys($networkById); + if ($networkIds !== [] + && $connection->table('cloud_vpcs') + ->whereIn('private_network_id', $networkIds) + ->whereNotIn('user_id', $userIds)->exists()) { + throw new \RuntimeException('VPC 私网删除账本归属不一致,账户删除已停止'); + } + if ($allocationKeys !== []) { + if (!$this->hasColumns($connection, 'cloud_private_cidr_claims', [ + 'allocation_key', + ])) { + throw new \RuntimeException('VPC CIDR 声明账本不完整,账户删除已停止'); + } + $residualClaims = $connection->table('cloud_private_cidr_claims') + ->whereIn('allocation_key', array_values(array_unique($allocationKeys))) + ->orderBy('allocation_key')->lockForUpdate()->get(['allocation_key']); + if (!$residualClaims->isEmpty()) { + throw new \RuntimeException('已释放私网仍有 CIDR 声明,账户删除已停止'); + } + } + } + + /** Delete only already-terminal VPC/account audit history. */ + private function deleteVpcHistory($connection, array $userIds): int + { + if (!$this->hasColumns($connection, 'cloud_vpcs', ['id', 'user_id', 'state']) + || !$this->hasColumns($connection, 'cloud_private_networks', [ + 'user_id', 'state', + ])) { + return 0; + } + $vpcIds = $connection->table('cloud_vpcs')->whereIn('user_id', $userIds) + ->where('state', 'deleted')->pluck('id')->all(); + $deleted = 0; + foreach (self::VPC_HISTORY_TABLES as $table) { + $deleted += $this->deleteWhereIn( + $connection, $table, 'vpc_id', $vpcIds + ); + } + $deleted += $this->deleteWhereIn( + $connection, 'cloud_vpcs', 'id', $vpcIds + ); + $deleted += (int) $connection->table('cloud_private_networks') + ->whereIn('user_id', $userIds)->where('state', 'released')->delete(); + return $deleted; + } + private function deleteLogRecords(array $ids, array $emails) { if (Config::get('enable_log', 'false') !== 'true') { diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index ea2b2163..3ba2ae1e 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -26,6 +26,8 @@ $app->group('/api/v1', function () { $this->get('/vpcs', 'App\Controllers\CloudVpcController:index'); $this->post('/vpcs', 'App\Controllers\CloudVpcController:create'); $this->get('/vpcs/{vpc_key:[0-9a-f]{32}}', 'App\Controllers\CloudVpcController:show'); + $this->delete('/vpcs/{vpc_key:[0-9a-f]{32}}', + 'App\Controllers\CloudVpcController:delete'); $this->post('/vpcs/{vpc_key:[0-9a-f]{32}}/subnets', 'App\Controllers\CloudVpcController:addSubnet'); $this->delete('/vpcs/{vpc_key:[0-9a-f]{32}}/subnets/{subnet_key:[0-9a-f]{32}}', 'App\Controllers\CloudVpcController:removeSubnet'); diff --git a/web/public/cloud/vpc.js b/web/public/cloud/vpc.js index 52968d08..30f2ae72 100644 --- a/web/public/cloud/vpc.js +++ b/web/public/cloud/vpc.js @@ -5,7 +5,7 @@ const state = { csrfToken: '', userId: '', regions: [], instances: [], vpcs: [], selectedKey: '', selected: null, busy: false, stopped: false, detailDirty: false, listEpoch: 0, detailEpoch: 0, listController: null, detailController: null, - pollTimer: 0, + pollTimer: 0, deleteRequestKeys: new Map(), }; const STATUS_LABELS = Object.freeze({ @@ -77,8 +77,10 @@ async function api(path, options = {}) { if (!response.ok || payload?.ret === 0) { const detail = Array.isArray(payload?.detail) ? payload.detail.map(item => item?.msg || '输入错误').join(';') : payload?.detail; - throw new Error(String(detail || payload?.message || payload?.error + const error = new Error(String(detail || payload?.message || payload?.error || `请求失败 (${response.status})`)); + error.status = response.status; + throw error; } return payload; } @@ -104,11 +106,14 @@ function safeText(value, fallback = '—') { function rootStatus(root) { const sync = String(root?.sync_state || '').toLowerCase(); const raw = String(root?.state || '').toLowerCase(); + // Root deletion is a durable tombstone, even when the latest engine call + // is failed or uncertain. Never reopen mutation controls while teardown + // still owns the VPC. + if (raw === 'deleting') return 'deleting'; if (root?.usable === true) return 'active'; if (sync === 'waiting' || sync === 'waiting_observed') return 'waiting'; if (sync === 'uncertain' || raw === 'uncertain') return 'uncertain'; if (raw === 'degraded' || sync === 'failed') return 'degraded'; - if (raw === 'deleting') return 'deleting'; if (raw === 'applying' || ['pending', 'queued', 'running'].includes(sync)) return 'applying'; return raw === 'active' ? 'uncertain' : (raw || 'unknown'); } @@ -198,6 +203,49 @@ function clearDetailDirty() { state.detailDirty = false; } +function selectedRootDeleting() { + return rootStatus(state.selected?.vpc) === 'deleting'; +} + +function allowSelectedMutation() { + if (!selectedRootDeleting()) return true; + setError('VPC 正在删除,完成所有宿主机清理前不能再修改子网、成员、路由或 ACL。'); + return false; +} + +function finishVpcDeletion(message = 'VPC 已完成宿主机清理,私网 CIDR 已释放。') { + const deletedKey = state.selectedKey; + if (deletedKey) { + state.deleteRequestKeys.delete(deletedKey); + // Remove the exact completed root immediately. A best-effort list refresh + // follows, but a transient refresh failure must not leave a clickable + // stale row that the detail endpoint has already proved deleted. + state.vpcs = state.vpcs.filter(item => item?.vpc_key !== deletedKey); + } + state.detailController?.abort(); + state.detailController = null; + state.detailEpoch += 1; + state.selectedKey = ''; + state.selected = null; + clearDetailDirty(); + byId('vpc-detail-panel').hidden = true; + byId('vpc-detail').replaceChildren(); + renderList(); + setError(''); + if (message) { setNotice(message, 'success'); toast(message, 'success'); } +} + +function lockDeletingDetail(container, reconcile) { + if (!selectedRootDeleting()) return; + container.dataset.rootDeleting = 'true'; + for (const control of container.querySelectorAll('button, input, select, textarea')) { + control.disabled = true; + } + // Reconcile does not change desired state; it is the explicit retry path + // for a deletion whose host readback is incomplete or uncertain. + reconcile.disabled = state.busy; +} + function refreshRouteEditorState() { const editor = byId('vpc-route-editor'); if (!editor) return; @@ -310,11 +358,15 @@ function renderDetail() { overviewHeading.append(statusPill(status, root.last_error_code), element('span', 'vpc-readback-note', root.usable === true ? 'WireGuard、私网路由和实例网卡均已回读确认。' - : (rootHint || '配置仍在宿主机应用或核对,期间不要重复创建同一成员。'))); + : (status === 'deleting' + ? '正在清理所有宿主机私网对象;精确回读完成前不会释放 VPC CIDR。' + : (rootHint || '配置仍在宿主机应用或核对,期间不要重复创建同一成员。')))); const actions = element('div', 'vpc-detail-actions'); const reconcile = button('核对 / 重试', 'button button-primary small'); reconcile.dataset.vpcAction = 'reconcile'; reconcile.disabled = state.busy || status === 'active'; - actions.append(reconcile); + const remove = button('删除 VPC', 'button button-outline small danger'); + remove.dataset.vpcAction = 'delete'; remove.disabled = state.busy || status === 'deleting'; + actions.append(reconcile, remove); overviewHeading.append(actions); const facts = element('dl', 'vpc-facts'); facts.append(field('VPC CIDR', root.cidr), field('地区', regionName(root.region_id)), @@ -329,6 +381,7 @@ function renderDetail() { container.append(renderAcl(doc)); container.append(renderOperations(doc)); clearDetailDirty(); + lockDeletingDetail(container, reconcile); } function renderSubnets(doc) { @@ -531,7 +584,11 @@ async function loadList(silent = false) { try { const payload = await api('/api/v1/vpcs', { signal: state.listController.signal }); if (epoch !== state.listEpoch || state.stopped) return; - state.vpcs = Array.isArray(payload?.vpcs) ? payload.vpcs.map(normalizeRoot).filter(Boolean) : []; + state.vpcs = Array.isArray(payload?.vpcs) ? payload.vpcs.map(normalizeRoot) + .filter(item => item && rootStatus(item) !== 'deleted') : []; + // The list is bounded to the newest 200 rows. Absence from it is not a + // deletion receipt, so keep a selected tombstone until its detail read + // explicitly reports state=deleted. renderList(); if (!silent) setError(''); } catch (error) { @@ -549,6 +606,11 @@ async function loadDetail(key, silent = false, allowDirty = false) { const payload = await api(`/api/v1/vpcs/${encodeURIComponent(key)}`, { signal: state.detailController.signal }); if (epoch !== state.detailEpoch || state.stopped || key !== state.selectedKey) return; state.selected = normalizeDocument(payload); + if (rootStatus(state.selected?.vpc) === 'deleted') { + finishVpcDeletion(); + await loadList(true); + return; + } byId('vpc-detail-panel').hidden = !state.selected; renderDetail(); if (!silent) setError(''); @@ -573,6 +635,11 @@ async function selectVpc(key) { async function applyDocument(payload, message = '') { const doc = normalizeDocument(payload); if (doc) { + if (rootStatus(doc.vpc) === 'deleted') { + finishVpcDeletion(); + await loadList(true); + return; + } clearDetailDirty(); state.selected = doc; state.selectedKey = doc.vpc.vpc_key; @@ -641,7 +708,7 @@ async function createVpc(event) { } async function submitSubnet(form) { - if (!confirmDetailMutation()) return; + if (!allowSelectedMutation() || !confirmDetailMutation()) return; const value = form.cidr.value.trim(); if (!value) return; const payload = await mutate(`/api/v1/vpcs/${encodeURIComponent(state.selectedKey)}/subnets`, 'POST', { @@ -651,7 +718,7 @@ async function submitSubnet(form) { } async function submitMember(form) { - if (!confirmDetailMutation()) return; + if (!allowSelectedMutation() || !confirmDetailMutation()) return; const body = { subnet_key: form.subnet_key.value, instance_id: Number(form.instance_id.value), request_key: requestKey('member-add') }; if (form.private_ip.value.trim()) body.private_ip = form.private_ip.value.trim(); await mutate(`/api/v1/vpcs/${encodeURIComponent(state.selectedKey)}/members`, 'POST', body, @@ -682,6 +749,7 @@ function readAcl() { } async function saveEditor(kind) { + if (!allowSelectedMutation()) return; if (kind === 'routes') { refreshRouteEditorState(); if (document.querySelector('#vpc-route-editor [data-vpc-tombstone-state="deleting"]')) { @@ -712,19 +780,42 @@ async function reconcile() { } async function removeSubnet(key) { - if (!state.selectedKey || state.busy || !confirmDetailMutation() + if (!state.selectedKey || state.busy || !allowSelectedMutation() || !confirmDetailMutation() || !confirm('确认删除这个子网?子网必须没有成员和地址租约。')) return; await mutate(`/api/v1/vpcs/${encodeURIComponent(state.selectedKey)}/subnets/${encodeURIComponent(key)}`, 'DELETE', { request_key: requestKey('subnet-remove') }, '子网删除目标已提交。'); } async function removeMember(key) { - if (!state.selectedKey || state.busy || !confirmDetailMutation() + if (!state.selectedKey || state.busy || !allowSelectedMutation() || !confirmDetailMutation() || !confirm('确认让这个实例退出 VPC?宿主机回读确认前,私网 IP 不会重新分配。')) return; await mutate(`/api/v1/vpcs/${encodeURIComponent(state.selectedKey)}/members/${encodeURIComponent(key)}`, 'DELETE', { request_key: requestKey('member-remove') }, '成员退出目标已提交,等待私网网卡移除回读。'); } +async function deleteVpc() { + if (!state.selectedKey || !state.selected || state.busy || selectedRootDeleting()) return; + if (!confirmDetailMutation()) return; + const root = state.selected.vpc; + const confirmation = prompt(`永久删除 VPC“${root.name}”并清理其私网网卡、路由和 ACL。宿主机回读完成后 CIDR 将被释放。请输入完整 VPC 名称确认:`); + if (confirmation === null) return; + if (confirmation !== root.name) { + toast('VPC 名称不匹配,未执行删除。', 'error'); + return; + } + const key = state.deleteRequestKeys.get(root.vpc_key) || requestKey('vpc-delete'); + state.deleteRequestKeys.set(root.vpc_key, key); + // A detail poll started before the destructive mutation must not overwrite + // the returned deleting state with its older active projection. + state.detailController?.abort(); + state.detailController = null; + state.detailEpoch += 1; + await mutate(`/api/v1/vpcs/${encodeURIComponent(root.vpc_key)}`, 'DELETE', { + confirmation_name: root.name, + request_key: key, + }, 'VPC 删除目标已提交;完成宿主机清理前不会释放 CIDR。'); +} + function handleClick(event) { const listRow = event.target.closest('[data-vpc-key]'); if (listRow && listRow.closest('#vpc-list')) { selectVpc(listRow.dataset.vpcKey); return; } @@ -734,8 +825,10 @@ function handleClick(event) { if (removeMemberKey) { removeMember(removeMemberKey); return; } const action = event.target.closest('[data-vpc-action]')?.dataset.vpcAction; if (action === 'reconcile') { reconcile(); return; } + if (action === 'delete') { deleteVpc(); return; } const add = event.target.closest('[data-editor-add]')?.dataset.editorAdd; if (add) { + if (!allowSelectedMutation()) return; const list = byId(add === 'route' ? 'vpc-route-editor' : 'vpc-acl-editor'); if (list.querySelector('[data-vpc-tombstone-state="deleting"]')) return; list.querySelector('.vpc-empty-inline')?.remove(); @@ -746,6 +839,7 @@ function handleClick(event) { } const remove = event.target.closest('[data-editor-remove]')?.dataset.editorRemove; if (remove) { + if (!allowSelectedMutation()) return; event.target.closest('.vpc-editor-row')?.remove(); markDetailDirty(); refreshRouteEditorState(); return; } const save = event.target.closest('[data-vpc-save]')?.dataset.vpcSave; @@ -777,7 +871,8 @@ function readBootstrap() { state.csrfToken = String(user.csrf_token); state.userId = String(user.user_id); state.regions = Array.isArray(payload.regions) ? payload.regions : []; state.instances = Array.isArray(payload.instances) ? payload.instances : []; - state.vpcs = Array.isArray(payload.vpcs) ? payload.vpcs.map(normalizeRoot).filter(Boolean) : []; + state.vpcs = Array.isArray(payload.vpcs) ? payload.vpcs.map(normalizeRoot) + .filter(item => item && rootStatus(item) !== 'deleted') : []; } function bind() { diff --git a/web/tests/CloudPrivateCidrAllocatorTest.php b/web/tests/CloudPrivateCidrAllocatorTest.php index c54b7086..5055ba93 100644 --- a/web/tests/CloudPrivateCidrAllocatorTest.php +++ b/web/tests/CloudPrivateCidrAllocatorTest.php @@ -52,6 +52,13 @@ $assert(is_string($serviceSource) && str_contains($serviceSource, '未知私网 CIDR 状态') && str_contains($serviceSource, 'compareAndSwapState') && str_contains($serviceSource, 'active_cidr') + && str_contains($serviceSource, 'assertClaimLedger') + && str_contains($serviceSource, 'self::normalizeCidr((string) $network->cidr, $prefix)') + && str_contains($serviceSource, + "\$network->broadcast_address !== \$canonical['broadcast']") + && str_contains($serviceSource, + "->orderBy('block_cidr')->lockForUpdate()->pluck('block_cidr')") + && str_contains($serviceSource, 'CIDR 分配请求已经释放') && str_contains($serviceSource, 'isTransientConflict') && !str_contains($serviceSource, 'CloudPrivateCidrAllocatorLock') && !str_contains($serviceSource, "CloudPrivateCidrAllocatorConfig::where(\n 'id', self::CONFIG_ID\n )->lockForUpdate()"), @@ -158,6 +165,9 @@ $assert($released->state === 'released' && $released->active_cidr === null 'Release did not clear the active claim and advance its revision.'); $assert(CloudPrivateCidrClaim::where('allocation_key', $key)->count() === 0, 'Release did not remove the reusable quantum claim.'); +$expectConflict(static fn () => CloudPrivateCidrAllocator::allocate( + $key, 7, 1, null, 24, $vpc +), 'A released allocation key resurrected an unclaimed network row.'); $reused = CloudPrivateCidrAllocator::allocate(str_repeat('1', 32), 10, 1, '10.0.0.0/24', 24, str_repeat('2', 32)); $assert($reused->cidr === '10.0.0.0/24', @@ -165,6 +175,66 @@ $assert($reused->cidr === '10.0.0.0/24', $expectConflict(static fn () => CloudPrivateCidrAllocator::release($key, 1), 'A stale revision released a CIDR a second time.'); +$brokenKey = str_repeat('3', 32); +$broken = CloudPrivateCidrAllocator::allocate( + $brokenKey, 11, 1, null, 24, str_repeat('4', 32) +); +CloudPrivateCidrClaim::where('allocation_key', $brokenKey)->delete(); +try { + CloudPrivateCidrAllocator::allocate( + $brokenKey, 11, 1, null, 24, str_repeat('4', 32) + ); + throw new RuntimeException( + 'An idempotent retry returned a live CIDR with a missing quantum claim.' + ); +} catch (CloudError $error) { + $assert($error->httpStatus === 503, + 'Damaged idempotent CIDR retry returned the wrong status.'); +} +try { + CloudPrivateCidrAllocator::release($brokenKey, (int) $broken->revision); + throw new RuntimeException('A CIDR with a missing quantum claim was released.'); +} catch (CloudError $error) { + $assert($error->httpStatus === 503, + 'A damaged CIDR claim ledger returned the wrong status.'); +} +$assert(CloudPrivateNetwork::where('allocation_key', $brokenKey)->value('state') + === 'active', + 'A damaged CIDR claim ledger was partially released.'); + +$tamperedKey = str_repeat('5', 32); +$tampered = CloudPrivateCidrAllocator::allocate( + $tamperedKey, 12, 1, null, 24, str_repeat('6', 32) +); +CloudPrivateNetwork::where('allocation_key', $tamperedKey)->update([ + // Keep cidr/network_address and the matching /24 claim unchanged. The + // old release check ignored this interval-field drift and would have + // made a corrupt allocation reusable. + 'broadcast_address' => (int) $tampered->broadcast_address - 1, +]); +try { + CloudPrivateCidrAllocator::allocate( + $tamperedKey, 12, 1, null, 24, str_repeat('6', 32) + ); + throw new RuntimeException( + 'An idempotent retry returned a CIDR with tampered interval metadata.' + ); +} catch (CloudError $error) { + $assert($error->httpStatus === 503, + 'Tampered idempotent CIDR retry returned the wrong status.'); +} +try { + CloudPrivateCidrAllocator::release($tamperedKey, (int) $tampered->revision); + throw new RuntimeException('A CIDR with tampered interval metadata was released.'); +} catch (CloudError $error) { + $assert($error->httpStatus === 503, + 'Tampered CIDR interval metadata returned the wrong status.'); +} +$assert(CloudPrivateNetwork::where('allocation_key', $tamperedKey)->value('state') + === 'active' + && CloudPrivateCidrClaim::where('allocation_key', $tamperedKey)->count() === 1, + 'Tampered CIDR metadata partially released its network or exact claim.'); + $normalized = CloudPrivateCidrAllocator::normalize('10.2.3.4/16', 16); $assert($normalized['cidr'] === '10.2.0.0/16' && $normalized['broadcast_address'] === '10.2.255.255', diff --git a/web/tests/CloudVpcControlPlaneContractTest.php b/web/tests/CloudVpcControlPlaneContractTest.php index 145315e5..67f3d0bd 100644 --- a/web/tests/CloudVpcControlPlaneContractTest.php +++ b/web/tests/CloudVpcControlPlaneContractTest.php @@ -78,10 +78,45 @@ $assert(str_contains($service, 'waiting_observed') && str_contains($service, "existing->subnet_id") && str_contains($service, 'hasBlockingMembership'), 'VPC deferred observation, response identity or membership gates are incomplete.'); +$assert(str_contains($service, 'public static function delete(') + && str_contains($service, "'confirmation_name', 'request_key'") + && str_contains($service, "'vpc_delete'") + && str_contains($service, 'deletionDocument') + && str_contains($service, 'recordDeleted') + && str_contains($service, "'/internal/v1/vpcs/' . \$vpcKey . '/delete'") + && str_contains($service, 'removed_host_ids') + && str_contains($service, "\$document['retiring_members'] = \$retiringMembers") + && str_contains($service, 'sort($hostIds, SORT_STRING)') + && str_contains($service, 'canonicalJson($payloadDocument)') + && str_contains($service, 'array_diff($payloadHostIds, $canonicalRemovedHostIds)') + && str_contains($service, 'array_diff($canonicalRemovedHostIds, $historicalHostIds)') + && str_contains($service, "if ((string) \$vpc->state === 'deleting')") + && str_contains($service, 'cannot shrink that manifest') + && str_contains($service, 'CloudPrivateCidrAllocator::release') + && str_contains($service, '$childModel::where(\'vpc_id\', $vpc->id)->delete()') + && str_contains($service, "\$vpc->state = 'deleted'") + && str_contains($service, "\$root->state = (string) \$row->kind === 'vpc_delete'"), + 'VPC root deletion lost confirmation, durable operation, exact receipt or CIDR CAS finalization.'); +$missingRemovalHost = strpos( + $service, '$item = $hosts[(string) $hostId] ?? null;' +); +$missingRemovalEnd = $missingRemovalHost === false ? false + : strpos($service, '// A deleting host remains a tombstone', $missingRemovalHost); +$missingRemovalBlock = ($missingRemovalHost !== false && $missingRemovalEnd !== false) + ? substr($service, $missingRemovalHost, + $missingRemovalEnd - $missingRemovalHost) : ''; +$assert($missingRemovalBlock !== '' + && str_contains($missingRemovalBlock, '$hasRemovalTarget = true') + && str_contains($missingRemovalBlock, '$allRemovalProof = false'), + 'A missing deleting host must make multi-host removal proof incomplete.'); $assert(str_contains($service, 'canViewInfrastructure') && str_contains($service, 'publicRoot($vpc, $includeHostDetails)') && str_contains($service, 'digestMatches($desiredDigest, $observedDigest)'), 'VPC public projection or usable digest gate leaks infrastructure or is incomplete.'); +$assert(str_contains($service, "CloudVpcMember::where('instance_id', \$instanceId)") + && str_contains($service, "['deleted', 'deleting'], true") + && !str_contains($service, "->where('vpc_id', '<>', \$vpc->id)\n ->where('state', '<>', 'deleted')"), + 'A retained member tombstone must not bypass the global instance_id uniqueness guard.'); $assert(str_contains($service, "'observed_revision' => " . '$revision') && str_contains($service, 'CloudVpcAclRule::class') && str_contains($service, 'whereNotIn(\'state\', [\'deleted\'])'), @@ -155,15 +190,33 @@ $assert(str_contains($vpcWorkerService, "'waiting_observed'") $assert(str_contains($routes, "'/vpcs'") && str_contains($routes, "'/vpcs/{vpc_key:[0-9a-f]{32}}/reconcile'") + && str_contains($controller, 'CloudVpcService::delete') && str_contains($controller, 'CloudVpcService::replaceRoutes') && str_contains($controller, 'CloudVpcService::replaceAcl') && preg_match('#/api/v1/vpcs#', $middleware) === 1, 'VPC API routes/controller/middleware are incomplete.'); +$assert(str_contains( + $routes, "\$this->delete('/vpcs/{vpc_key:[0-9a-f]{32}}'," +), 'VPC root DELETE route is missing.'); $assert(str_contains($engineClient, 'function vpc(') - && str_contains($engineClient, '/internal/v1/vpcs'), + && str_contains($engineClient, '/internal/v1/vpcs') + && str_contains($engineClient, '(?:reconcile|delete)') + && str_contains($engineClient, 'vpc_delete_manifest_invalid') + && str_contains($engineClient, 'vpc_delete_manifest_mismatch') + && str_contains($engineClient, 'vpc_private_remove_requires_stopped') + && str_contains($engineClient, 'vpc_private_remove_unsafe') + && str_contains($engineClient, 'vpc_private_interface_conflict') + && str_contains($engineClient, 'vpc_seed_missing') + && str_contains($engineClient, 'vpc_seed_readback_invalid') + && str_contains($engineClient, 'vpc_private_xml_invalid'), 'PHP engine client has no bounded VPC endpoint.'); $assert(str_contains($accountDeletion, "cloud_vpcs") && str_contains($accountDeletion, "cloud_private_networks") + && str_contains($accountDeletion, 'assertVpcHistoryCleanable') + && str_contains($accountDeletion, 'deleteVpcHistory') + && str_contains($accountDeletion, 'VPC_HISTORY_TABLES') + && str_contains($accountDeletion, 'VPC 私网子账本不完整') + && str_contains($accountDeletion, '已释放私网仍有 CIDR 声明') && str_contains($jobService, 'assertInstanceCanBeDeleted') && str_contains($orderService, "private_expectation") && str_contains($workerService, '旧删除任务缺少 VPC 删除门禁'), diff --git a/web/tests/CloudVpcRootDeletionTest.php b/web/tests/CloudVpcRootDeletionTest.php new file mode 100644 index 00000000..db178940 --- /dev/null +++ b/web/tests/CloudVpcRootDeletionTest.php @@ -0,0 +1,427 @@ +where('state', '<>', 'deleted')") + && str_contains($serviceSource, 'removed_host_ids') + && str_contains($serviceSource, 'CloudPrivateCidrAllocator::release'), + 'VPC root deletion lost its tombstone, receipt or CIDR-release boundary.'); +$assert(is_array($digestFixture) + && is_array($digestFixture['document'] ?? null) + && preg_match('/^[0-9a-f]{64}$/D', + (string) ($digestFixture['expected_digest'] ?? '')) === 1, + 'The shared PHP/engine deletion digest fixture is missing or malformed.'); +$missingHost = strpos( + $serviceSource, '$item = $hosts[(string) $hostId] ?? null;' +); +$missingHostEnd = $missingHost === false ? false + : strpos($serviceSource, '// A deleting host remains a tombstone', $missingHost); +$missingHostBlock = ($missingHost !== false && $missingHostEnd !== false) + ? substr($serviceSource, $missingHost, $missingHostEnd - $missingHost) : ''; +$assert($missingHostBlock !== '' + && str_contains($missingHostBlock, '$hasRemovalTarget = true') + && str_contains($missingHostBlock, '$allRemovalProof = false'), + 'A missing deleting host can still be mistaken for complete multi-host removal.'); + +if (!is_file($root . '/vendor/autoload.php')) { + echo "VPC root deletion source contracts passed (runtime skipped).\n"; + return; +} +require $root . '/vendor/autoload.php'; + +$capsule = new Capsule(); +$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); +$schema = $capsule->schema('default'); +$schema->create('cloud_regions', static function (Blueprint $table): void { + $table->increments('id'); +}); +$schema->create('cloud_vpcs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('vpc_key', 32)->unique(); + $table->unsignedInteger('user_id'); + $table->unsignedInteger('region_id'); + $table->unsignedInteger('private_network_id'); + $table->string('name', 63); + $table->string('cidr', 18); + $table->string('fabric_id', 64); + $table->string('fabric_region', 64); + $table->unsignedBigInteger('fabric_revision'); + $table->unsignedInteger('max_members'); + $table->string('state', 16); + $table->unsignedBigInteger('desired_revision'); + $table->unsignedBigInteger('observed_revision'); + $table->string('desired_digest', 64); + $table->string('observed_digest', 64)->nullable(); + $table->string('sync_state', 16); + $table->string('last_operation_id', 32)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('last_success_at')->default(0); + $table->unsignedBigInteger('last_failure_at')->default(0); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +foreach ([ + 'cloud_vpc_subnets' => 'subnet_key', + 'cloud_vpc_routes' => 'route_key', + 'cloud_vpc_acl_rules' => 'rule_key', +] as $tableName => $keyColumn) { + $schema->create($tableName, static function (Blueprint $table) use ($keyColumn): void { + $table->increments('id'); + $table->unsignedInteger('vpc_id'); + $table->string($keyColumn, 32); + $table->string('state', 16); + if ($keyColumn === 'rule_key') { + $table->unsignedInteger('priority')->default(1); + } + }); +} +$schema->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('uuid', 36); +}); +$schema->create('cloud_vpc_members', static function (Blueprint $table): void { + $table->increments('id'); + $table->unsignedInteger('vpc_id'); + $table->string('member_key', 32); + $table->unsignedInteger('subnet_id')->nullable(); + $table->unsignedInteger('private_ip_id')->nullable(); + $table->unsignedInteger('instance_id'); + $table->unsignedInteger('host_id'); + $table->string('private_ip', 15); + $table->string('state', 16); + $table->unsignedBigInteger('desired_revision')->default(0); + $table->string('desired_digest', 64)->nullable(); + $table->unsignedBigInteger('observed_revision')->default(0); + $table->string('observed_digest', 64)->nullable(); + $table->string('last_operation_id', 32)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('updated_at')->default(0); + $table->unique('instance_id'); +}); +$schema->create('cloud_vpc_private_ips', static function (Blueprint $table): void { + $table->increments('id'); + $table->unsignedInteger('vpc_id'); + $table->unsignedInteger('subnet_id')->nullable(); + $table->unsignedInteger('member_id')->nullable(); + $table->unsignedInteger('instance_id')->nullable(); + $table->string('private_ip', 15); + $table->string('state', 16); + $table->unsignedBigInteger('desired_revision')->default(0); + $table->string('desired_digest', 64)->nullable(); + $table->unsignedBigInteger('observed_revision')->default(0); + $table->string('observed_digest', 64)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('updated_at')->default(0); +}); +$schema->create('cloud_vpc_host_deployments', static function (Blueprint $table): void { + $table->increments('id'); + $table->unsignedInteger('vpc_id'); + $table->unsignedInteger('host_id'); + $table->string('state', 16); + $table->unsignedBigInteger('observed_revision')->default(0); + $table->string('observed_digest', 64)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('last_success_at')->default(0); + $table->unsignedBigInteger('updated_at')->default(0); +}); +$schema->create('cloud_vpc_operations', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('operation_id', 32)->unique(); + $table->unsignedInteger('vpc_id'); + $table->unsignedInteger('user_id'); + $table->string('kind', 24); + $table->unsignedBigInteger('desired_revision'); + $table->string('status', 16); + $table->text('payload_json'); + $table->text('observed_json')->nullable(); + $table->string('error_code', 64)->nullable(); + $table->string('lease_token', 32)->nullable(); + $table->unsignedBigInteger('lease_until')->default(0); + $table->unsignedBigInteger('finished_at')->nullable(); + $table->unsignedBigInteger('updated_at'); +}); +$schema->create('cloud_private_networks', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('vpc_key', 32)->unique(); + $table->string('allocation_key', 32)->unique(); + $table->unsignedInteger('user_id'); + $table->unsignedInteger('region_id'); + $table->string('cidr', 18); + $table->string('active_cidr', 18)->nullable(); + $table->unsignedBigInteger('network_address'); + $table->unsignedBigInteger('broadcast_address'); + $table->unsignedTinyInteger('prefix_length'); + $table->string('state', 16); + $table->unsignedBigInteger('revision'); + $table->unsignedBigInteger('released_at')->nullable(); + $table->unsignedBigInteger('updated_at'); +}); +$schema->create('cloud_private_cidr_claims', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('block_cidr', 18)->unique(); + $table->string('allocation_key', 32); +}); +Capsule::table('cloud_regions')->insert(['id' => 1]); + +$vpc = new CloudVpc(); +$vpc->id = 7; +$vpc->vpc_key = str_repeat('a', 32); +$vpc->region_id = 1; +$vpc->fabric_region = 'test-region'; +$vpc->fabric_id = 'test-fabric'; +$vpc->fabric_revision = 3; +$vpc->cidr = '10.20.0.0/24'; +$vpc->desired_revision = 9; + +$method = new ReflectionMethod(CloudVpcService::class, 'deletionDocument'); +$canonicalize = new ReflectionMethod(CloudVpcService::class, 'canonicalize'); + +$historical = new CloudVpcHostDeployment(); +$historical->vpc_id = 7; +$historical->host_id = 11; +$historical->state = 'deleted'; +$historical->save(); +$document = $method->invoke(null, $vpc); +$assert($document['hosts'] === [], + 'A historical deleted host was reintroduced into the engine delete target.'); + +$pending = new CloudVpcHostDeployment(); +$pending->vpc_id = 7; +$pending->host_id = 2; +$pending->state = 'active'; +$pending->save(); +$document = $method->invoke(null, $vpc); +$assert($document['hosts'] === [[ + 'host_id' => '2', 'members' => [], +]], 'The delete target did not contain exactly the unresolved host subset.'); + +$historical->state = 'active'; +$historical->save(); +$document = $method->invoke(null, $vpc); +$assert($document['hosts'] === [ + ['host_id' => '11', 'members' => []], + ['host_id' => '2', 'members' => []], +], 'Delete host IDs were not canonicalized with lexical string ordering.'); +$historical->state = 'deleted'; +$historical->save(); + +$vpcKey = str_repeat('a', 32); +$operationId = str_repeat('b', 32); +$leaseToken = str_repeat('c', 32); +$allocationKey = str_repeat('d', 32); +Capsule::table('cloud_vpcs')->insert([ + 'id' => 7, 'vpc_key' => $vpcKey, 'user_id' => 9, 'region_id' => 1, + 'private_network_id' => 20, 'name' => 'delete-test', + 'cidr' => '10.20.0.0/24', 'fabric_id' => 'test-fabric', + 'fabric_region' => 'test-region', 'fabric_revision' => 3, + 'max_members' => 32, 'state' => 'deleting', 'desired_revision' => 10, + 'observed_revision' => 9, 'desired_digest' => str_repeat('e', 64), + 'observed_digest' => str_repeat('f', 64), 'sync_state' => 'pending', + 'last_operation_id' => $operationId, 'last_error_code' => null, + 'last_success_at' => 0, 'last_failure_at' => 0, + 'created_at' => time(), 'updated_at' => time(), +]); +Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2)->update([ + 'state' => 'deleting', +]); +Capsule::table('cloud_instances')->insert([ + 'id' => 30, 'uuid' => '11111111-2222-4333-8444-555555555555', +]); +$memberKey = str_repeat('1', 32); +Capsule::table('cloud_vpc_members')->insert([ + 'id' => 32, 'vpc_id' => 7, 'member_key' => $memberKey, + 'subnet_id' => null, 'private_ip_id' => 31, 'instance_id' => 30, 'host_id' => 2, + 'private_ip' => '10.20.0.10', 'state' => 'deleting', + 'desired_revision' => 10, 'desired_digest' => str_repeat('e', 64), + 'observed_revision' => 9, 'observed_digest' => str_repeat('f', 64), + 'last_operation_id' => str_repeat('8', 32), + 'last_error_code' => null, 'updated_at' => time(), +]); +Capsule::table('cloud_vpc_private_ips')->insert([ + 'id' => 31, 'vpc_id' => 7, 'subnet_id' => null, + 'member_id' => 32, 'instance_id' => 30, + 'private_ip' => '10.20.0.10', 'state' => 'deleting', + 'desired_revision' => 10, 'desired_digest' => str_repeat('e', 64), + 'observed_revision' => 9, 'observed_digest' => str_repeat('f', 64), + 'last_error_code' => null, 'updated_at' => time(), +]); +$persistedVpc = CloudVpc::findOrFail(7); +Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11)->update([ + 'state' => 'deleting', +]); +$fixtureDocument = $method->invoke(null, $persistedVpc); +$fixtureDigest = hash('sha256', json_encode( + $fixtureDocument, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES +)); +$fixtureTarget = $digestFixture['document']; +unset($fixtureTarget['operation_id']); +$assert($fixtureDocument === $canonicalize->invoke(null, $fixtureTarget) + && $fixtureDigest === $digestFixture['expected_digest'], + 'PHP deletion canonicalization drifted from the shared engine digest fixture.'); +Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11)->update([ + 'state' => 'deleted', +]); +$deleteDocument = $method->invoke(null, $persistedVpc); +$assert($deleteDocument['hosts'] === [[ + 'host_id' => '2', 'members' => [], +]], 'The final delete plan reintroduced a historical deleted host.'); +$assert(count($deleteDocument['retiring_members']) === 1 + && $deleteDocument['retiring_members'][0]['member_id'] === $memberKey + && $deleteDocument['retiring_members'][0]['instance_uuid'] + === '11111111-2222-4333-8444-555555555555' + && $deleteDocument['retiring_members'][0]['host_id'] === '2' + && $deleteDocument['retiring_members'][0]['private_ip_cidr'] + === '10.20.0.10/32' + && preg_match('/^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/D', + $deleteDocument['retiring_members'][0]['private_mac']) === 1, + 'Delete manifest did not preserve the exact retiring private NIC identity.'); +$digest = hash('sha256', json_encode( + $deleteDocument, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES +)); +Capsule::table('cloud_vpcs')->where('id', 7)->update([ + 'desired_digest' => $digest, +]); +Capsule::table('cloud_vpc_members')->where('id', 32)->update([ + 'desired_digest' => $digest, +]); +Capsule::table('cloud_vpc_private_ips')->where('id', 31)->update([ + 'desired_digest' => $digest, +]); +$lifecycleOperationId = str_repeat('9', 32); +$assert(CloudVpcService::recordPrivateObservation(30, [ + 'ok' => true, + 'state' => [ + 'desired_state' => 'removed', + 'desired_revision' => 10, + 'desired_digest' => $digest, + 'observed_revision' => 10, + 'observed_digest' => $digest, + ], + 'error_code' => null, +], $lifecycleOperationId) === true, + 'An exact lifecycle readback was not recorded during root deletion.'); +$assert(Capsule::table('cloud_vpc_members')->where('id', 32)->value('state') + === 'deleting' + && Capsule::table('cloud_vpc_private_ips')->where('id', 31)->value('state') + === 'deleting' + && Capsule::table('cloud_vpc_members')->where('id', 32) + ->value('last_operation_id') === str_repeat('8', 32), + 'A lifecycle readback shrank the immutable root-delete manifest or replaced its proof marker.'); +$operationPayload = $deleteDocument; +$operationPayload['desired_digest'] = $digest; +$operationPayload['operation_id'] = $operationId; +Capsule::table('cloud_vpc_operations')->insert([ + 'operation_id' => $operationId, 'vpc_id' => 7, 'user_id' => 9, + 'kind' => 'vpc_delete', 'desired_revision' => 10, 'status' => 'running', + 'payload_json' => json_encode($operationPayload, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), + 'observed_json' => null, 'error_code' => null, 'lease_token' => $leaseToken, + 'lease_until' => time() + 120, 'finished_at' => null, 'updated_at' => time(), +]); +Capsule::table('cloud_private_networks')->insert([ + 'id' => 20, 'vpc_key' => $vpcKey, 'allocation_key' => $allocationKey, + 'user_id' => 9, 'region_id' => 1, 'cidr' => '10.20.0.0/24', + 'active_cidr' => '10.20.0.0/24', 'network_address' => 169082880, + 'broadcast_address' => 169083135, 'prefix_length' => 24, + 'state' => 'active', 'revision' => 1, 'released_at' => null, + 'updated_at' => time(), +]); +Capsule::table('cloud_private_cidr_claims')->insert([ + 'block_cidr' => '10.20.0.0/24', 'allocation_key' => $allocationKey, +]); +$missingPendingReceipt = [ + 'vpc_id' => $vpcKey, 'operation_id' => $operationId, + 'desired_revision' => 10, 'desired_digest' => $digest, + 'state' => 'deleted', 'deleted' => true, 'readback' => 'confirmed', + 'removed_host_ids' => [], 'completed_at' => time(), +]; +try { + CloudVpcService::recordDeleted( + $vpcKey, 10, $digest, $missingPendingReceipt, $operationId, $leaseToken + ); + throw new RuntimeException('A receipt missing the pending host was accepted.'); +} catch (CloudError $error) { + $assert($error->httpStatus === 502, + 'Missing-pending-host receipt returned the wrong status.'); +} +try { + CloudVpcService::recordDeleted($vpcKey, 10, $digest, [ + 'vpc_id' => $vpcKey, 'operation_id' => $operationId, + 'desired_revision' => 10, 'desired_digest' => $digest, + 'state' => 'deleted', 'deleted' => true, 'readback' => 'confirmed', + 'removed_host_ids' => ['2', '99'], 'completed_at' => time(), + ], $operationId, $leaseToken); + throw new RuntimeException('A receipt containing an unknown host was accepted.'); +} catch (CloudError $error) { + $assert($error->httpStatus === 502, + 'Unknown-host receipt returned the wrong status.'); +} +$assert(Capsule::table('cloud_vpcs')->where('id', 7)->value('state') === 'deleting' + && Capsule::table('cloud_private_networks')->where('id', 20)->value('state') + === 'active', + 'Incomplete host receipt partially finalized root or CIDR state.'); +CloudVpcService::recordDeleted($vpcKey, 10, $digest, [ + 'vpc_id' => $vpcKey, 'operation_id' => $operationId, + 'desired_revision' => 10, 'desired_digest' => $digest, + 'state' => 'deleted', 'deleted' => true, 'readback' => 'confirmed', + // Host 11 was already exactly retired and may no longer exist in the + // engine's latest durable record. Only the signed pending host is + // mandatory in this receipt. + 'removed_host_ids' => ['2'], 'completed_at' => time(), +], $operationId, $leaseToken); +$storedReceipt = json_decode((string) Capsule::table('cloud_vpc_operations') + ->where('operation_id', $operationId)->value('observed_json'), true); +$assert(Capsule::table('cloud_vpcs')->where('id', 7)->value('state') === 'deleted' + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operationId) + ->value('status') === 'succeeded' + && ($storedReceipt['removed_host_ids'] ?? null) === ['2'], + 'Exact delete receipt did not finalize the VPC root and operation atomically.'); +$assert(Capsule::table('cloud_private_networks')->where('id', 20)->value('state') + === 'released' + && Capsule::table('cloud_private_networks')->where('id', 20) + ->value('active_cidr') === null + && Capsule::table('cloud_private_cidr_claims')->count() === 0, + 'Exact delete receipt did not release the CIDR claim.'); +$assert(CloudVpcHostDeployment::where('vpc_id', 7) + ->where('state', '<>', 'deleted')->count() === 0, + 'Exact delete receipt left a host deployment outside deleted state.'); +$assert((int) CloudVpcHostDeployment::where('vpc_id', 7) + ->where('host_id', 11)->value('observed_revision') === 0 + && (int) CloudVpcHostDeployment::where('vpc_id', 7) + ->where('host_id', 2)->value('observed_revision') === 10, + 'Root finalization overwrote an omitted historical proof or missed the pending host.'); +$assert(Capsule::table('cloud_vpc_members')->where('vpc_id', 7)->count() === 0 + && Capsule::table('cloud_vpc_private_ips')->where('vpc_id', 7)->count() === 0, + 'Exact delete receipt did not physically clear the child identity ledgers.'); +Capsule::table('cloud_vpc_members')->insert([ + 'vpc_id' => 8, 'member_key' => str_repeat('2', 32), + 'private_ip_id' => null, 'instance_id' => 30, 'host_id' => 2, + 'private_ip' => '10.21.0.10', 'state' => 'applying', +]); +$assert(Capsule::table('cloud_vpc_members')->where('instance_id', 30)->count() === 1, + 'A completed root delete still occupied the global VM membership key.'); + +echo "VPC root deletion contracts passed.\n"; diff --git a/web/tests/CloudVpcUserPageContractTest.php b/web/tests/CloudVpcUserPageContractTest.php index fae066b7..a1324916 100644 --- a/web/tests/CloudVpcUserPageContractTest.php +++ b/web/tests/CloudVpcUserPageContractTest.php @@ -93,6 +93,31 @@ $assert(str_contains($script, "'/api/v1/vpcs'") && str_contains($script, "'PUT'"), 'VPC module must cover create, subnet, member, route, ACL, reconcile and deletion APIs.'); +$assert(str_contains($script, "remove.dataset.vpcAction = 'delete'") + && str_contains($script, 'async function deleteVpc()') + && str_contains($script, "requestKey('vpc-delete')") + && str_contains($script, 'confirmation_name: root.name') + && str_contains($script, "'DELETE'") + && str_contains($script, 'selectedRootDeleting()') + && str_contains($script, 'lockDeletingDetail(container, reconcile)') + && str_contains($script, 'finishVpcDeletion()') + && str_contains($script, "rootStatus(item) !== 'deleted'") + && str_contains($script, + 'state.vpcs = state.vpcs.filter(item => item?.vpc_key !== deletedKey)') + && str_contains($script, "state.selectedKey = ''"), + 'VPC root deletion must require an exact-name confirmation, lock mutations and clear completed selections.'); + +$assert(str_contains($script, 'Absence from it is not a') + && !str_contains($script, 'const deletionCompleted =') + && !str_contains($script, "error?.status === 404 && key === state.selectedKey"), + 'A bounded list omission or detail 404 must not be treated as a VPC deletion receipt.'); + +$deletingProjection = strpos($script, "if (raw === 'deleting') return 'deleting';"); +$failedProjection = strpos($script, "if (raw === 'degraded' || sync === 'failed')"); +$assert($deletingProjection !== false && $failedProjection !== false + && $deletingProjection < $failedProjection, + 'A deleting root must stay a locked tombstone when engine sync is failed or uncertain.'); + $assert(str_contains($script, 'X-CSRF-Token') && str_contains($script, 'credentials: \'same-origin\'') && str_contains($script, 'request_key') @@ -101,7 +126,7 @@ $assert(str_contains($script, 'X-CSRF-Token') && str_contains($script, 'rule_key: row.dataset.aclKey'), 'VPC mutations must use same-origin CSRF protection and separate idempotency/entity keys.'); -foreach (['applying', 'degraded', 'uncertain', 'waiting', 'active'] as $status) { +foreach (['applying', 'degraded', 'uncertain', 'waiting', 'active', 'deleting'] as $status) { $assert(str_contains($script, $status), 'VPC UI lost state projection for ' . $status . '.'); } diff --git a/web/tests/ManualAccountDeletionSqliteTest.php b/web/tests/ManualAccountDeletionSqliteTest.php index 08c89377..3c8ea76a 100644 --- a/web/tests/ManualAccountDeletionSqliteTest.php +++ b/web/tests/ManualAccountDeletionSqliteTest.php @@ -80,6 +80,26 @@ $schema->create('cloud_invoice_payments', static function (Blueprint $table) { $schema->create('cloud_mail_outbox', static function (Blueprint $table) { $table->increments('id'); $table->integer('invoice_id'); }); +$schema->create('cloud_private_cidr_claims', static function (Blueprint $table) { + $table->increments('id'); $table->string('allocation_key', 32); +}); +$schema->create('cloud_private_networks', static function (Blueprint $table) { + $table->increments('id'); $table->string('vpc_key', 32); + $table->string('allocation_key', 32); $table->integer('user_id'); + $table->string('state', 16); $table->string('active_cidr', 18)->nullable(); +}); +$schema->create('cloud_vpcs', static function (Blueprint $table) { + $table->increments('id'); $table->string('vpc_key', 32); + $table->integer('user_id'); $table->integer('private_network_id'); + $table->string('state', 16); +}); +foreach (['cloud_vpc_routes', 'cloud_vpc_acl_rules', 'cloud_vpc_private_ips', + 'cloud_vpc_members', 'cloud_vpc_subnets', 'cloud_vpc_host_deployments', + 'cloud_vpc_operations'] as $tableName) { + $schema->create($tableName, static function (Blueprint $table): void { + $table->increments('id'); $table->integer('vpc_id'); + }); +} $schema->create('email_verify', static function (Blueprint $table) { $table->increments('id'); $table->string('email'); @@ -157,6 +177,22 @@ Capsule::table('cloud_invoice_payments')->insert([ Capsule::table('cloud_mail_outbox')->insert([ ['invoice_id' => 103], ['invoice_id' => 303], ]); +$deletedVpcKey = str_repeat('a', 32); +$deletedAllocationKey = str_repeat('b', 32); +Capsule::table('cloud_private_networks')->insert([ + 'id' => 1001, 'vpc_key' => $deletedVpcKey, + 'allocation_key' => $deletedAllocationKey, 'user_id' => 1, + 'state' => 'released', 'active_cidr' => null, +]); +Capsule::table('cloud_vpcs')->insert([ + 'id' => 1002, 'vpc_key' => $deletedVpcKey, 'user_id' => 1, + 'private_network_id' => 1001, 'state' => 'deleted', +]); +foreach (['cloud_vpc_routes', 'cloud_vpc_acl_rules', 'cloud_vpc_private_ips', + 'cloud_vpc_members', 'cloud_vpc_subnets', 'cloud_vpc_host_deployments', + 'cloud_vpc_operations'] as $tableName) { + Capsule::table($tableName)->insert(['vpc_id' => 1002]); +} Capsule::table('email_verify')->insert([ ['email' => 'deleted@example.com'], ['email' => 'other@example.com'], ]); @@ -189,6 +225,15 @@ $assert(Capsule::table('cloud_jobs')->orderBy('id')->pluck('id')->all() 'Tenant deletion retained delegated jobs or removed unrelated administrator jobs.'); $assert(Capsule::table('cloud_mail_outbox')->pluck('invoice_id')->all() === [303], 'Cloud invoice mail history deletion was not isolated.'); +$assert(Capsule::table('cloud_vpcs')->count() === 0 + && Capsule::table('cloud_private_networks')->count() === 0, + 'Terminal VPC roots or released private-network history survived account deletion.'); +foreach (['cloud_vpc_routes', 'cloud_vpc_acl_rules', 'cloud_vpc_private_ips', + 'cloud_vpc_members', 'cloud_vpc_subnets', 'cloud_vpc_host_deployments', + 'cloud_vpc_operations'] as $tableName) { + $assert(Capsule::table($tableName)->count() === 0, + 'Terminal VPC child history survived account deletion: ' . $tableName); +} $assert(Capsule::table('email_verify')->pluck('email')->all() === ['other@example.com'], 'Email verification cleanup was not isolated.'); $assert(Capsule::table('configurations')->where('name', 'email_blacklist')->value('value') === "deleted@example.com\n@blocked.example", 'Manual blacklist was changed.'); $assert($result['cache_error'] === false && $result['log_rows_deleted'] === 0, 'Disabled optional services were accessed.'); @@ -226,4 +271,159 @@ $assert(User::find(5) !== null && Capsule::table('cloud_orders')->where('id', 502)->exists(), 'Unpaid-provisioning deletion was not rolled back.'); +Capsule::table('user')->insert([ + 'id' => 6, 'email' => 'active-vpc@example.com', 'is_admin' => 0, 'ref_by' => 0, +]); +$activeVpcKey = str_repeat('c', 32); +Capsule::table('cloud_private_networks')->insert([ + 'id' => 6001, 'vpc_key' => $activeVpcKey, + 'allocation_key' => str_repeat('d', 32), 'user_id' => 6, + 'state' => 'active', 'active_cidr' => '10.60.0.0/24', +]); +Capsule::table('cloud_vpcs')->insert([ + 'id' => 6002, 'vpc_key' => $activeVpcKey, 'user_id' => 6, + 'private_network_id' => 6001, 'state' => 'active', +]); +try { + (new UserDataDeletionService())->deleteUsers([6], false); + throw new RuntimeException('Account with a live VPC was deleted.'); +} catch (RuntimeException $error) { + $assert(str_contains($error->getMessage(), '仍有 VPC 私网'), + 'Live-VPC account deletion returned the wrong error.'); +} +$assert(User::find(6) !== null, 'Live-VPC account deletion was not rolled back.'); + +Capsule::table('user')->insert([ + 'id' => 7, 'email' => 'claimed-vpc@example.com', 'is_admin' => 0, 'ref_by' => 0, +]); +$claimedVpcKey = str_repeat('e', 32); +$claimedAllocationKey = str_repeat('f', 32); +Capsule::table('cloud_private_networks')->insert([ + 'id' => 7001, 'vpc_key' => $claimedVpcKey, + 'allocation_key' => $claimedAllocationKey, 'user_id' => 7, + 'state' => 'released', 'active_cidr' => null, +]); +Capsule::table('cloud_vpcs')->insert([ + 'id' => 7002, 'vpc_key' => $claimedVpcKey, 'user_id' => 7, + 'private_network_id' => 7001, 'state' => 'deleted', +]); +Capsule::table('cloud_private_cidr_claims')->insert([ + 'allocation_key' => $claimedAllocationKey, +]); +try { + (new UserDataDeletionService())->deleteUsers([7], false); + throw new RuntimeException('Account with a residual private CIDR claim was deleted.'); +} catch (RuntimeException $error) { + $assert(str_contains($error->getMessage(), '仍有 CIDR 声明'), + 'Residual-claim account deletion returned the wrong error.'); +} +$assert(User::find(7) !== null + && Capsule::table('cloud_vpcs')->where('id', 7002)->exists(), + 'Residual-claim account deletion discarded fail-closed VPC evidence.'); + +Capsule::table('user')->insert([ + 'id' => 8, 'email' => 'missing-vpc-table@example.com', + 'is_admin' => 0, 'ref_by' => 0, +]); +$missingTableVpcKey = str_repeat('1', 32); +Capsule::table('cloud_private_networks')->insert([ + 'id' => 8001, 'vpc_key' => $missingTableVpcKey, + 'allocation_key' => str_repeat('2', 32), 'user_id' => 8, + 'state' => 'released', 'active_cidr' => null, +]); +Capsule::table('cloud_vpcs')->insert([ + 'id' => 8002, 'vpc_key' => $missingTableVpcKey, 'user_id' => 8, + 'private_network_id' => 8001, 'state' => 'deleted', +]); +$schema->drop('cloud_vpc_operations'); +try { + (new UserDataDeletionService())->deleteUsers([8], false); + throw new RuntimeException('Account cleanup ignored a missing VPC child ledger.'); +} catch (RuntimeException $error) { + $assert(str_contains($error->getMessage(), '子账本不完整'), + 'Missing VPC child ledger returned the wrong error.'); +} +$assert(User::find(8) !== null + && Capsule::table('cloud_vpcs')->where('id', 8002)->exists() + && Capsule::table('cloud_private_networks')->where('id', 8001)->exists(), + 'Missing child schema did not roll back account/VPC cleanup.'); + +$schema->create('cloud_vpc_operations', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('sensitive_payload'); +}); +Capsule::table('cloud_vpc_operations')->insert([ + 'sensitive_payload' => 'must-not-be-orphaned', +]); +Capsule::table('user')->insert([ + 'id' => 9, 'email' => 'missing-vpc-column@example.com', + 'is_admin' => 0, 'ref_by' => 0, +]); +$missingColumnVpcKey = str_repeat('3', 32); +Capsule::table('cloud_private_networks')->insert([ + 'id' => 9001, 'vpc_key' => $missingColumnVpcKey, + 'allocation_key' => str_repeat('4', 32), 'user_id' => 9, + 'state' => 'released', 'active_cidr' => null, +]); +Capsule::table('cloud_vpcs')->insert([ + 'id' => 9002, 'vpc_key' => $missingColumnVpcKey, 'user_id' => 9, + 'private_network_id' => 9001, 'state' => 'deleted', +]); +try { + (new UserDataDeletionService())->deleteUsers([9], false); + throw new RuntimeException('Account cleanup ignored a malformed VPC child ledger.'); +} catch (RuntimeException $error) { + $assert(str_contains($error->getMessage(), '子账本不完整'), + 'Malformed VPC child ledger returned the wrong error.'); +} +$assert(User::find(9) !== null + && Capsule::table('cloud_vpcs')->where('id', 9002)->exists() + && Capsule::table('cloud_vpc_operations')->where( + 'sensitive_payload', 'must-not-be-orphaned' + )->exists(), + 'Malformed child schema orphaned VPC or sensitive operation history.'); + +// Both core tables existing with incomplete ownership columns must never be +// confused with a deployment where the VPC feature is wholly absent. This +// reproduces the dangerous false/false hasColumns branch that used to delete +// the account and leave an unverifiable CIDR claim/history relationship. +$schema->drop('cloud_vpcs'); +$schema->drop('cloud_private_networks'); +$schema->create('cloud_private_networks', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('vpc_key', 32); + $table->integer('user_id'); + $table->string('state', 16); +}); +$schema->create('cloud_vpcs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('vpc_key', 32); + $table->integer('user_id'); + $table->string('state', 16); +}); +Capsule::table('user')->insert([ + 'id' => 10, 'email' => 'malformed-vpc-core@example.com', + 'is_admin' => 0, 'ref_by' => 0, +]); +$malformedCoreVpcKey = str_repeat('5', 32); +Capsule::table('cloud_private_networks')->insert([ + 'id' => 10001, 'vpc_key' => $malformedCoreVpcKey, + 'user_id' => 10, 'state' => 'released', +]); +Capsule::table('cloud_vpcs')->insert([ + 'id' => 10002, 'vpc_key' => $malformedCoreVpcKey, + 'user_id' => 10, 'state' => 'deleted', +]); +try { + (new UserDataDeletionService())->deleteUsers([10], false); + throw new RuntimeException('Account cleanup ignored malformed VPC core ledgers.'); +} catch (RuntimeException $error) { + $assert(str_contains($error->getMessage(), '删除账本不完整'), + 'Malformed VPC core ledgers returned the wrong error.'); +} +$assert(User::find(10) !== null + && Capsule::table('cloud_vpcs')->where('id', 10002)->exists() + && Capsule::table('cloud_private_networks')->where('id', 10001)->exists(), + 'Malformed VPC core ledgers did not roll back account cleanup.'); + echo 'Manual account deletion SQLite checks passed.' . PHP_EOL;