From 732a336d8c14b1c858841f80e9d2eb7e4ff00522 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 10 Oct 2026 14:05:07 +0800 Subject: [PATCH] fix(vpc): allow restoring interrupted deletion --- engine/tests/test_engine_boundary.py | 61 +- engine/tests/test_vpc.py | 149 +++ engine/virsh_panel/internal.py | 25 + engine/virsh_panel/vpc.py | 244 ++++ web/app/Controllers/CloudVpcController.php | 7 + web/app/Services/Cloud/CloudEngineClient.php | 49 +- web/app/Services/Cloud/CloudVpcService.php | 1094 ++++++++++++++++- web/config/routes_cloud.php | 2 + web/public/cloud/vpc.js | 89 +- web/resources/views/venix/layout_assets.tpl | 2 +- web/tests/CloudFirewallUiContractTest.php | 2 +- .../CloudHostFabricEndpointContractTest.php | 2 +- .../CloudVpcControlPlaneContractTest.php | 20 +- web/tests/CloudVpcDeleteRestoreTest.php | 692 +++++++++++ web/tests/CloudVpcEngineErrorContractTest.php | 42 + .../CloudVpcTombstoneConvergenceTest.php | 52 + web/tests/CloudVpcUserPageBehaviorTest.mjs | 93 ++ web/tests/CloudVpcUserPageContractTest.php | 37 +- 18 files changed, 2630 insertions(+), 32 deletions(-) create mode 100644 web/tests/CloudVpcDeleteRestoreTest.php diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index 61ad05ad..166b074a 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -38,7 +38,7 @@ from virsh_panel.libvirt_service import ( ) from virsh_panel.metrics import host_scope from virsh_panel.user_isos import user_iso_scope -from virsh_panel.vpc import parse_vpc_delete_plan +from virsh_panel.vpc import parse_vpc_delete_plan, parse_vpc_plan SECRET = b"s" * 32 @@ -595,6 +595,65 @@ def test_vpc_root_delete_endpoint_returns_exact_durable_receipt(boundary): assert invalid.json()["error_code"] == "vpc_delete_manifest_invalid" +def test_vpc_delete_restore_endpoint_dispatches_complete_fenced_plan(boundary): + client, engine, _store, _logs, _services = boundary + value = { + "schema_version": 1, + "vpc_id": "vpc-boundary-restore", + "region": "hk", + "fabric_id": "fabric-boundary", + "fabric_revision": 1, + "cidr": "10.32.0.0/24", + "desired_revision": 4, + "operation_id": "vpc-boundary-restore-0001", + "subnets": [], + "hosts": [], + "members": [], + "routes": [], + "acls": [], + } + value["desired_digest"] = parse_vpc_plan(value).desired_digest + value["cancelled_delete"] = { + "operation_id": "vpc-boundary-delete-0001", + "desired_revision": 3, + "desired_digest": "a" * 64, + } + captured = {} + + async def restore_delete(vpc_id, submitted): + captured.update(vpc_id=vpc_id, submitted=submitted) + return { + "vpc_id": vpc_id, + "operation_id": submitted["operation_id"], + "desired_revision": submitted["desired_revision"], + "desired_digest": submitted["desired_digest"], + "state": "uncertain", + } + + engine.vpc.restore_delete = restore_delete + path = "/internal/v1/vpcs/vpc-boundary-restore/delete/restore" + restored = call( + client, "POST", path, {**USER, "can_manage": False}, value) + assert restored.status_code == 200, restored.text + assert restored.json()["state"] == "uncertain" + assert captured == { + "vpc_id": "vpc-boundary-restore", + "submitted": value, + } + + missing_cas = dict(value) + missing_cas.pop("cancelled_delete") + invalid = call(client, "POST", path, SUPER, missing_cas) + assert invalid.status_code == 400 + assert invalid.json()["error_code"] == "vpc_delete_restore_required" + assert call(client, "GET", path, SUPER).status_code == 404 + assert call( + client, "POST", + "/internal/v1/vpcs/vpc-boundary-restore/delete/restore/extra", + SUPER, value, + ).status_code == 404 + + def test_host_public_network_pools_are_revisioned_idempotent_and_aggregated( boundary): diff --git a/engine/tests/test_vpc.py b/engine/tests/test_vpc.py index 0638f85b..02e835a2 100644 --- a/engine/tests/test_vpc.py +++ b/engine/tests/test_vpc.py @@ -124,6 +124,26 @@ def delete_payload(*, vpc_id="vpc-hk-1", desired_revision=2, return value +def restore_delete_payload(delete_target, **overrides): + value = payload( + vpc_id=delete_target["vpc_id"], + region=delete_target["region"], + fabric_id=delete_target["fabric_id"], + fabric_revision=delete_target["fabric_revision"], + cidr=delete_target["cidr"], + desired_revision=delete_target["desired_revision"] + 1, + operation_id="vpc-restore-delete-00000001", + ) + value.update(overrides) + value["desired_digest"] = parse_vpc_plan(value).desired_digest + value["cancelled_delete"] = { + "operation_id": delete_target["operation_id"], + "desired_revision": delete_target["desired_revision"], + "desired_digest": delete_target["desired_digest"], + } + return value + + def confirm_delete_hosts(plan, record): """Fill the exact bounded absence proof required for a delete receipt.""" for host in record.get("hosts", []): @@ -2258,6 +2278,135 @@ def test_root_delete_accepts_revision_jump_and_superseded_unknown_host(): assert recreated.value.error_code == "vpc_deleted" +def test_restore_delete_fences_partial_cleanup_without_host_mutation(): + store = MemoryStore() + fabric = FakeFabric(hosts=("host-a", "host-b")) + coordinator = VpcCoordinator(store, fabric) + asyncio.run(coordinator.apply(payload())) + target = delete_payload( + desired_revision=2, + operation_id="vpc-delete-cancelled-0001", + hosts=("host-a", "host-b"), + retiring_members=(retiring_member_a(),), + ) + + async def partial_cleanup(plan, record, gate, previous_plan=None, **kwargs): + host_a = next( + host for host in record["hosts"] + if host["host_id"] == "host-a") + confirm_delete_hosts(plan, {"hosts": [host_a]}) + host_b = next( + host for host in record["hosts"] + if host["host_id"] == "host-b") + host_b.update({ + "state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": "vpc_host_unavailable", + }) + + coordinator._reconcile_dataplane = partial_cleanup + with pytest.raises(VpcError) as incomplete: + asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert incomplete.value.error_code == "vpc_delete_incomplete" + + restore = restore_delete_payload(target) + wrong_cas = json.loads(json.dumps(restore)) + wrong_cas["cancelled_delete"]["desired_digest"] = "0" * 64 + with pytest.raises(VpcError) as stale: + asyncio.run(coordinator.restore_delete("vpc-hk-1", wrong_cas)) + assert stale.value.error_code == "vpc_delete_restore_conflict" + + expanded = restore_delete_payload( + target, + hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-c", "members": []}, + ], + ) + with pytest.raises(VpcError) as host_expansion: + asyncio.run(coordinator.restore_delete("vpc-hk-1", expanded)) + assert (host_expansion.value.error_code + == "vpc_delete_restore_host_set_mismatch") + + different_member = restore_delete_payload( + target, + hosts=[{"host_id": "host-a", "members": [{ + "member_id": "member-a", "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}], + ) + with pytest.raises(VpcError) as manifest_mismatch: + asyncio.run(coordinator.restore_delete( + "vpc-hk-1", different_member)) + assert (manifest_mismatch.value.error_code + == "vpc_delete_restore_manifest_mismatch") + + async def must_not_mutate(*_args, **_kwargs): + raise AssertionError("delete restore reached host mutation") + + coordinator._reconcile_dataplane = must_not_mutate + fabric.get_status = lambda _fabric_id: (_ for _ in ()).throw( + FabricError("offline", 503, "fabric_status_unavailable")) + restored = asyncio.run(coordinator.restore_delete("vpc-hk-1", restore)) + assert restored["state"] == "uncertain" + assert restored["desired_revision"] == 3 + assert restored["operation_id"] == "vpc-restore-delete-00000001" + assert isinstance(restored["observed"], dict) + assert isinstance(restored["hosts"], list) + durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) + assert "delete_target" not in durable + hosts = {host["host_id"]: host for host in durable["hosts"]} + assert hosts["host-a"]["removed"] is False + assert hosts["host-a"]["retiring_members"] == [] + # host-b was still required by the admitted delete marker but is omitted + # from the restored live plan. Keep it as an ordinary host tombstone so + # later reconciliation finishes its host-resource cleanup. + assert hosts["host-b"]["retiring"] is True + assert hosts["host-b"]["retiring_members"] == [] + + # A lost success response replays the same ordinary target without SSH. + assert asyncio.run(coordinator.restore_delete( + "vpc-hk-1", restore)) == restored + + # The higher ordinary target is the fence against the admitted old delete. + with pytest.raises(VpcError) as old_delete: + asyncio.run(coordinator.delete("vpc-hk-1", target)) + assert old_delete.value.error_code == "vpc_revision_conflict" + + +def test_restore_delete_rejects_final_receipt(): + store = MemoryStore() + coordinator = VpcCoordinator(store, FakeFabric(hosts=())) + target = delete_payload( + vpc_id="vpc-final-delete", + cidr="10.24.0.0/24", + desired_revision=2, + operation_id="vpc-final-delete-0001", + hosts=(), + retiring_members=(), + ) + receipt = asyncio.run(coordinator.delete("vpc-final-delete", target)) + assert receipt["deleted"] is True + restore = restore_delete_payload( + target, + subnets=[], members=[], hosts=[], routes=[], acls=[], + ) + + with pytest.raises(VpcError) as completed: + asyncio.run(coordinator.restore_delete("vpc-final-delete", restore)) + + assert completed.value.status == 409 + assert completed.value.error_code == "vpc_delete_already_completed" + + def test_delete_manifest_accepts_unicast_with_odd_last_octet_only(): derived = derive_private_mac("never", "m") assert derived.endswith(":49") diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index f98dac62..77ea6c50 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -146,6 +146,7 @@ WRITE_PATHS = { r"fabric/(?:apply|probe)", r"fabric/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}/reconcile", r"vpcs/apply", r"vpcs/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}/reconcile", + r"vpcs/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}/delete/restore", r"vpcs/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}/delete", r"vm-templates/(?:import|upload)", r"rescue-templates", r"user-isos/import", @@ -1134,6 +1135,30 @@ class EngineApplication: r"vpcs/([A-Za-z0-9][A-Za-z0-9_.-]{0,63})/delete", resource, ) + restore_match = re.fullmatch( + r"vpcs/([A-Za-z0-9][A-Za-z0-9_.-]{0,63})/delete/restore", + resource, + ) + if restore_match is not None: + try: + value = json.loads(body) + except (TypeError, ValueError) as exc: + raise VpcError("VPC 删除恢复计划 JSON 无效", 422) from exc + if not isinstance(value, dict): + raise VpcError("VPC 删除恢复计划必须是对象", 422) + if not isinstance(value.get("operation_id"), str): + raise VpcError("VPC 删除恢复计划缺少 operation_id", 400, + "vpc_operation_required") + if not isinstance(value.get("desired_digest"), str): + raise VpcError("VPC 删除恢复计划缺少 desired_digest", 400, + "vpc_digest_required") + if not isinstance(value.get("cancelled_delete"), dict): + raise VpcError("VPC 删除恢复计划缺少 cancelled_delete", 400, + "vpc_delete_restore_required") + result = await self.vpc.restore_delete( + restore_match[1], value) + await json_response(send, 200, result) + return if delete_match is not None: try: value = json.loads(body) diff --git a/engine/virsh_panel/vpc.py b/engine/virsh_panel/vpc.py index 2567809c..f1cb7ffc 100644 --- a/engine/virsh_panel/vpc.py +++ b/engine/virsh_panel/vpc.py @@ -912,6 +912,46 @@ def parse_vpc_delete_plan( return replace(plan, desired_digest=computed_digest), members +def parse_vpc_delete_restore_plan( + value: Mapping[str, Any], +) -> tuple[VpcPlan, dict[str, Any]]: + """Parse a normal desired plan plus the exact delete marker it cancels.""" + if not isinstance(value, Mapping): + raise VpcError("VPC 删除恢复计划必须是对象", 422, + "vpc_delete_restore_invalid") + cancelled = value.get("cancelled_delete") + if not isinstance(cancelled, Mapping): + raise VpcError("VPC 删除恢复计划缺少取消目标", 422, + "vpc_delete_restore_invalid") + try: + _field_set(cancelled, { + "operation_id", "desired_revision", "desired_digest", + }, "VPC 删除取消目标") + operation_id = cancelled.get("operation_id") + if (not isinstance(operation_id, str) + or _OPERATION_ID.fullmatch(operation_id) is None): + raise VpcError("VPC 删除取消 operation_id 无效") + desired_revision = _revision( + cancelled.get("desired_revision"), + "VPC 删除取消 desired_revision", + ) + desired_digest = _digest( + cancelled.get("desired_digest"), + "VPC 删除取消 desired_digest", + ) + except VpcError as exc: + raise VpcError(str(exc), 422, + "vpc_delete_restore_invalid") from exc + normal = dict(value) + normal.pop("cancelled_delete", None) + plan = parse_vpc_plan(normal) + return plan, { + "operation_id": operation_id, + "desired_revision": desired_revision, + "desired_digest": desired_digest, + } + + def _plan_from_record(value: Mapping[str, Any]) -> VpcPlan: if not isinstance(value, Mapping): raise VpcError("VPC 持久化记录损坏", 500, "vpc_record_corrupt") @@ -3527,6 +3567,210 @@ class VpcCoordinator: async with self._lock(vpc_id), self._catalog_mutation_lock(): return await _await_mutation_completion(delete_locked()) + async def restore_delete( + self, vpc_id: str, value: Mapping[str, Any], + ) -> dict[str, Any]: + """Fence a cancelled root delete with a newer ordinary desired plan. + + This transition deliberately performs no host operation. The newer + desired revision first makes an admitted stale delete retry harmless; + the ordinary reconciler can then rebuild or remove each host resource + from the resulting normal desired record. + """ + vpc_id = _identifier(vpc_id, "VPC") + plan, cancelled = parse_vpc_delete_restore_plan(value) + if plan.vpc_id != vpc_id: + raise VpcError("VPC 删除恢复路径与目标不一致", 409, + "vpc_delete_restore_target_mismatch") + + async def restore_locked() -> dict[str, Any]: + # A final receipt is the irreversible boundary. Do not revive an + # identity after every host has been positively removed, even if + # an old marker triple happens to match the caller's CAS token. + if self._load_delete_receipt(vpc_id) is not None: + raise VpcError("VPC 删除已经完成,不能恢复", 409, + "vpc_delete_already_completed") + + previous = self._load("vpc_desired", vpc_id) + if previous is None: + raise VpcError("VPC 删除恢复目标不存在", 409, + "vpc_delete_restore_conflict") + marker = self._delete_marker(previous) + if marker is None: + # A lost restore response is safe to replay only when all + # envelope and canonical target fields match exactly. Never + # turn this path into a generic same-revision apply. + current = _plan_from_record(previous) + if (current.operation_id == plan.operation_id + and current.desired_revision == plan.desired_revision + and current.desired_digest == plan.desired_digest + and current.canonical() == plan.canonical()): + return self._public(previous) + raise VpcError("VPC 删除恢复目标已发生变化", 409, + "vpc_delete_restore_conflict") + + if any(cancelled[field] != marker[field] for field in ( + "operation_id", "desired_revision", "desired_digest")): + raise VpcError("VPC 删除恢复 CAS 与当前删除目标不一致", 409, + "vpc_delete_restore_conflict") + + # Parsing the durable delete record validates both its canonical + # plan and the digest-covered retiring-member manifest. A broken + # stored marker is an engine record failure, not bad tenant input. + try: + delete_plan = _plan_from_record(previous) + durable_payload = self._delete_payload_from_record(previous) + _, retiring_members = parse_vpc_delete_plan(durable_payload) + except VpcError as exc: + if exc.error_code == "vpc_record_corrupt": + raise + raise VpcError("VPC 删除持久化目标无效", 500, + "vpc_record_corrupt") from exc + if delete_plan.vpc_id != vpc_id: + raise VpcError("VPC 删除持久化身份不一致", 500, + "vpc_record_corrupt") + + if plan.desired_revision <= delete_plan.desired_revision: + raise VpcError("VPC 删除恢复版本必须高于删除目标", 409, + "vpc_revision_conflict") + if plan.operation_id == delete_plan.operation_id: + raise VpcError("VPC 删除恢复必须使用新的操作编号", 409, + "vpc_delete_restore_conflict") + if (plan.region, plan.fabric_id, plan.cidr) != ( + delete_plan.region, delete_plan.fabric_id, + delete_plan.cidr): + raise VpcError("VPC 删除恢复不能改变目录身份", 409, + "vpc_delete_restore_identity_mismatch") + + marker_host_ids = set(marker["all_host_ids"]) + previous_hosts = self._host_closure(previous) + if (not set(marker["required_host_ids"]).issubset( + delete_plan.host_ids) + or not delete_plan.host_ids.issubset(marker_host_ids) + or not set(previous_hosts).issubset(marker_host_ids)): + raise VpcError("VPC 删除持久化宿主机闭集不一致", 500, + "vpc_record_corrupt") + if not plan.host_ids.issubset(marker_host_ids): + raise VpcError("VPC 删除恢复不能引入删除闭集外宿主机", 409, + "vpc_delete_restore_host_set_mismatch") + + retiring_by_id = { + member.member_id: member for member in retiring_members + } + for member in plan.members: + retired = retiring_by_id.get(member.member_id) + if (retired is None + or member.instance_uuid != retired.instance_uuid + or member.host_id != retired.host_id + or member.private_ip != retired.private_ip + or _effective_private_mac( + plan.vpc_id, member.member_id, + member.private_mac) != retired.private_mac): + raise VpcError( + "VPC 删除恢复成员不属于原删除清单", + 409, "vpc_delete_restore_manifest_mismatch") + + # The global endpoint catalog is local durable state, so it can + # still be checked while Fabric status is unavailable. The live + # Fabric gate is recorded as uncertain/degraded instead of being + # an admission requirement: cancelling delete must be able to + # establish its newer fence before any later SSH reconciliation. + self._validate_global_fabric_endpoints(plan) + gate = self._fabric_gate(plan) + state = "applying" if gate.get("active") else ( + "degraded" if gate.get("available") else "uncertain") + previous_observed_revision = previous.get( + "observed_revision", 0) + previous_observed_digest = previous.get("observed_digest") + if (isinstance(previous_observed_revision, bool) + or not isinstance(previous_observed_revision, int) + or previous_observed_revision < 0 + or previous_observed_revision + > delete_plan.desired_revision): + raise VpcError("VPC 持久化观测版本无效", 500, + "vpc_record_corrupt") + if previous_observed_digest is not None: + try: + previous_observed_digest = _digest( + previous_observed_digest, "observed_digest") + except VpcError as exc: + raise VpcError("VPC 持久化观测摘要无效", 500, + "vpc_record_corrupt") from exc + for old_host in previous_hosts.values(): + old_host_revision = old_host.get("observed_revision", 0) + if (isinstance(old_host_revision, bool) + or not isinstance(old_host_revision, int) + or old_host_revision < 0 + or old_host_revision + > delete_plan.desired_revision): + raise VpcError("VPC 宿主观测版本无效", 500, + "vpc_record_corrupt") + old_host_digest = old_host.get("observed_digest") + if old_host_digest is not None: + try: + _digest(old_host_digest, "observed_digest") + except VpcError as exc: + raise VpcError("VPC 宿主观测摘要无效", 500, + "vpc_record_corrupt") from exc + + record = self._desired_record( + plan, gate, state=state, + observed_revision=previous_observed_revision, + observed_digest=previous_observed_digest, + previous_hosts=previous_hosts, + previous_plan=delete_plan, + ) + + restored_member_ids = { + member.member_id for member in plan.members + } + remaining_by_host: dict[str, list[dict[str, Any]]] = {} + for member in retiring_members: + if member.member_id in restored_member_ids: + continue + # Root-delete manifests do not need a subnet identifier, but + # ordinary host tombstones do. Cleanup only consumes the VM, + # host, IP and MAC identity, so this explicit synthetic value + # safely preserves the remaining removal obligation. + remaining_by_host.setdefault(member.host_id, []).append({ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "host_id": member.host_id, + "private_ip_cidr": f"{member.private_ip}/32", + "subnet_id": "retiring", + "private_mac": member.private_mac, + }) + active_host_ids = { + member.host_id for member in plan.members + } + for host in record.get("hosts", []): + if not isinstance(host, dict): + continue + host_id = host.get("host_id") + if not isinstance(host_id, str): + continue + remaining = remaining_by_host.get(host_id, []) + host["retiring_members"] = list(remaining) + if host.get("retiring") is True: + host["member_ids"] = [ + item["member_id"] for item in remaining + ] + if host_id in active_host_ids: + # A partial root-delete pass may already have published an + # exact absence proof. Active desired membership revokes + # only that old proof; it does not claim the host is ready. + host["removed"] = False + + self._finish(record, gate) + # Persist the ordinary target before projecting child records. + # This is the revision fence; no SSH/libvirt call occurs here. + self._save("vpc_desired", vpc_id, record) + self._save_children(record) + return self._public(record) + + async with self._lock(vpc_id), self._catalog_mutation_lock(): + return await _await_mutation_completion(restore_locked()) + async def apply(self, value: Mapping[str, Any]) -> dict[str, Any]: plan = parse_vpc_plan(value) async with self._lock(plan.vpc_id), self._catalog_mutation_lock(): diff --git a/web/app/Controllers/CloudVpcController.php b/web/app/Controllers/CloudVpcController.php index c884fb23..5dec0f93 100644 --- a/web/app/Controllers/CloudVpcController.php +++ b/web/app/Controllers/CloudVpcController.php @@ -45,6 +45,13 @@ final class CloudVpcController return $this->json($response, $value, 202); } + public function restoreDelete($request, $response, $args) + { + $value = CloudVpcService::restoreDelete((string) $args['vpc_key'], + $this->actor($request), CloudInput::object($request)); + return $this->json($response, $value, 202); + } + public function attachMember($request, $response, $args) { $value = CloudVpcService::attachMember((string) $args['vpc_key'], diff --git a/web/app/Services/Cloud/CloudEngineClient.php b/web/app/Services/Cloud/CloudEngineClient.php index ae18cfa8..068575db 100644 --- a/web/app/Services/Cloud/CloudEngineClient.php +++ b/web/app/Services/Cloud/CloudEngineClient.php @@ -103,6 +103,17 @@ final class CloudEngineClient $safeVpcServerErrorCode ); } + $safeVpcRestoreErrorCode = self::safeVpcRestoreErrorCode( + $method, $signedPath, $result, $response->getStatusCode() + ); + if ($safeVpcRestoreErrorCode !== null) { + throw new CloudError( + self::safeVpcErrorMessage($safeVpcRestoreErrorCode) + ?? '无法安全恢复 VPC 删除,请刷新后重试', + (int) $response->getStatusCode(), + $safeVpcRestoreErrorCode + ); + } // Only explicitly reviewed endpoint-specific policies return // short, credential-free guidance useful to the caller. Do not // turn this into a general engine-error passthrough: arbitrary @@ -133,6 +144,35 @@ final class CloudEngineClient ) === 1 ? 'vpc_delete_incomplete' : null; } + /** + * Preserve only reviewed, machine-readable delete-restore conflicts. + * Free-form engine detail is never exposed across this boundary. + */ + private static function safeVpcRestoreErrorCode( + string $method, ?string $path, array $result, ?int $status = null + ): ?string { + if (strtoupper($method) !== 'POST' || $status === null + || !in_array($status, [400, 409, 422], true) || $path === null + || preg_match( + '#^/internal/v1/vpcs/[0-9a-f]{32}/delete/restore$#D', + $path + ) !== 1) { + return null; + } + $code = $result['error_code'] ?? null; + if (!is_string($code)) { + return null; + } + return in_array($code, [ + 'vpc_delete_already_completed', + 'vpc_delete_restore_conflict', + 'vpc_revision_conflict', + 'vpc_delete_restore_identity_mismatch', + 'vpc_delete_restore_manifest_mismatch', + 'vpc_delete_restore_host_set_mismatch', + ], true) ? $code : null; + } + private static function safeAllowlistedError(?string $path, array $result, ?int $status = null): ?string { if ($path === null) { @@ -163,7 +203,7 @@ final class CloudEngineClient $route ) === 1; $vpc = preg_match( - '#^/internal/v1/vpcs(?:/(?:apply|[0-9a-f]{32}(?:/(?:reconcile|delete))?))?$#D', + '#^/internal/v1/vpcs(?:/(?:apply|[0-9a-f]{32}(?:/(?:reconcile|delete(?:/restore)?))?))?$#D', $route ) === 1; $fabric = preg_match( @@ -289,6 +329,11 @@ final class CloudEngineClient 'vpc_delete_target_not_empty' => 'VPC 删除目标仍包含活动资源,请刷新后重试', 'vpc_delete_host_set_mismatch' => 'VPC 删除宿主机集合不一致,正在核对', 'vpc_delete_receipt_conflict' => 'VPC 删除回执与当前操作冲突,请联系管理员', + 'vpc_delete_already_completed' => 'VPC 删除已完成,不能再恢复', + 'vpc_delete_restore_conflict' => 'VPC 删除恢复与当前状态冲突,请刷新后重试', + 'vpc_delete_restore_identity_mismatch' => 'VPC 删除恢复标识与当前操作不一致,请刷新后重试', + 'vpc_delete_restore_manifest_mismatch' => 'VPC 删除恢复成员清单与当前账本不一致', + 'vpc_delete_restore_host_set_mismatch' => 'VPC 删除恢复宿主机集合与当前账本不一致', 'vpc_private_remove_requires_stopped' => 'VPC 成员虚拟机仍在运行,请先关机后重试删除', 'vpc_private_remove_unsafe' => 'VPC 成员存在 managed-save 或原生快照,暂不能移除私网', 'vpc_private_interface_conflict' => 'VPC 私网网卡身份不一致,请联系管理员核对', @@ -510,7 +555,7 @@ final class CloudEngineClient ?array $payload = null, ?string $expectedOperationId = null): array { if (!preg_match( - '#^/internal/v1/vpcs(?:/(?:apply|[0-9a-f]{32}(?:/(?:reconcile|delete))?))?$#D', + '#^/internal/v1/vpcs(?:/(?:apply|[0-9a-f]{32}(?:/(?:reconcile|delete(?:/restore)?))?))?$#D', $resource )) { throw new CloudError('VPC 引擎资源路径无效', 422); diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php index 2340fbf9..8b8882e3 100644 --- a/web/app/Services/Cloud/CloudVpcService.php +++ b/web/app/Services/Cloud/CloudVpcService.php @@ -335,6 +335,127 @@ final class CloudVpcService }, 3); } + /** + * Fence and cancel the current root-delete target without guessing host + * state. The prior immutable non-delete plan is first revalidated + * against the retained identity ledger. PHP publishes it only after the + * engine has durably fenced the exact delete operation. + */ + public static function restoreDelete( + string $vpcKey, + array $actor, + array $input, + ?CloudEngineClient $engine = null + ): array { + CloudInput::only($input, [ + 'confirmation_name', 'request_key', 'expected_revision', + 'expected_operation_id', + ]); + $confirmationName = $input['confirmation_name'] ?? null; + if (!is_string($confirmationName)) { + throw new CloudError('请输入 VPC 名称确认恢复'); + } + $requestKey = self::requestKey($input['request_key'] ?? null); + $expectedRevision = self::positiveInt( + $input['expected_revision'] ?? null, + 1, + 8999999999999999, + 'VPC 删除版本无效' + ); + $expectedOperationId = self::assertKey( + $input['expected_operation_id'] ?? null, + 'VPC 删除操作' + ); + $requestHash = self::requestHash('vpc_delete_restore', [ + 'confirmation_name' => $confirmationName, + 'expected_revision' => $expectedRevision, + 'expected_operation_id' => $expectedOperationId, + ]); + + // Enforce global write permission before opening the restore + // transaction. lockOwnedForDelete() then repeats tenant ownership + // under the row lock without reopening ordinary mutations. + $vpc = self::findOwned($vpcKey, $actor, true); + $prepared = $vpc->getConnection()->transaction(function () use ( + $vpc, $actor, $confirmationName, $requestKey, $requestHash, + $expectedRevision, $expectedOperationId + ): array { + return self::prepareDeleteRestoreLocked( + (string) $vpc->vpc_key, + $actor, + $confirmationName, + $requestKey, + $requestHash, + $expectedRevision, + $expectedOperationId + ); + }, 3); + if (($prepared['published'] ?? false) === true) { + return self::show($vpcKey, $actor); + } + + /** @var CloudVpcOperation $restoreOperation */ + $restoreOperation = $prepared['operation']; + $restorePayload = $prepared['payload']; + $engine ??= new CloudEngineClient(null, self::ENGINE_TIMEOUT_SECONDS); + try { + $result = $engine->vpc( + 'POST', + '/internal/v1/vpcs/' . $vpcKey . '/delete/restore', + $actor, + $restorePayload, + (string) $restoreOperation->operation_id + ); + } catch (CloudError $error) { + self::releaseDeleteRestoreFence( + (int) $vpc->id, + (string) $restoreOperation->operation_id, + $error + ); + throw $error; + } + + $expectedDigest = (string) ($restorePayload['desired_digest'] ?? ''); + $newRevision = (int) ($restorePayload['desired_revision'] ?? 0); + if (!is_array($result) + || (string) ($result['vpc_id'] ?? '') !== $vpcKey + || (string) ($result['operation_id'] ?? '') + !== (string) $restoreOperation->operation_id + || (int) ($result['desired_revision'] ?? 0) !== $newRevision + || !self::digestMatches( + $expectedDigest, + (string) ($result['desired_digest'] ?? '') + ) + || !in_array((string) ($result['state'] ?? ''), [ + 'applying', 'degraded', 'uncertain', + ], true) + || !is_array($result['observed'] ?? null) + || !is_array($result['hosts'] ?? null)) { + // A 2xx response can only be emitted after the engine has written + // its durable restore fence. Keep both PHP operations fenced so a + // late delete worker cannot race a guessed retry. + throw new CloudError( + '引擎 VPC 删除恢复回执与操作标识不一致,正在核对', + 502 + ); + } + + $vpc->getConnection()->transaction(function () use ( + $vpc, $restoreOperation, $restorePayload, $result, + $expectedRevision, $expectedOperationId + ): void { + self::publishDeleteRestoreLocked( + (int) $vpc->id, + (string) $restoreOperation->operation_id, + $restorePayload, + $result, + $expectedRevision, + $expectedOperationId + ); + }, 3); + return self::show($vpcKey, $actor); + } + public static function attachMember(string $vpcKey, array $actor, array $input): array { // Legacy subnet/IP fields are admitted only far enough to replay an @@ -1971,16 +2092,22 @@ final class CloudVpcService && ($hosts === [] || $onlyKnownDeletedHostEvidence) && $allKnownDeletedProofValid && !$pendingHostTombstone; - // A newly reserved VPC deliberately has no participating host until - // its first member is attached. The current engine evidence can + // A newly reserved VPC, including one restored from an interrupted + // root deletion, deliberately has no participating host until its + // first member is attached. The current engine evidence can // therefore complete only the control-plane reservation, never the // private data plane. Capture the original child-tombstone gate // before the generic terminal-proof cleanup below; a malformed - // create ledger must not become successful merely because a route + // empty ledger must not become successful merely because a route // or ACL tombstone was closed while processing the observation. + $emptyReservationOperation = in_array( + (string) $operation->kind, + ['vpc_create', 'vpc_delete_restore'], + true + ); $emptyAutomaticSubnetValid = false; if ($noDataPlaneTerminalProof - && (string) $operation->kind === 'vpc_create' + && $emptyReservationOperation && $desiredContractValid) { try { $emptyAutomaticSubnet = self::lockedAutomaticSubnet($vpc); @@ -1994,7 +2121,7 @@ final class CloudVpcService $emptyAutomaticSubnetValid = false; } } - $emptyCreateLedgerValid = $desiredContractValid + $emptyReservationLedgerValid = $desiredContractValid && is_array($desired ?? null) && (int) ($desired['desired_revision'] ?? 0) === $revision && ($desired['hosts'] ?? null) === [] @@ -2017,8 +2144,8 @@ final class CloudVpcService && !CloudVpcAclRule::where('vpc_id', $vpc->id) ->whereNotIn('state', ['deleting', 'deleted'])->exists(); $emptyControlPlaneComplete = $noDataPlaneTerminalProof - && (string) $operation->kind === 'vpc_create' - && $emptyCreateLedgerValid + && $emptyReservationOperation + && $emptyReservationLedgerValid && !$pendingChildTombstone; $terminalSubnetReleased = 0; $storedMemberRemovalReleased = 0; @@ -2316,17 +2443,926 @@ final class CloudVpcService }, 3); } - /** Build the exact typed document signed into a VPC operation. */ - public static function desiredDocument(CloudVpc $vpc): array + /** + * Phase one of delete restore. The new operation is deliberately not a + * worker-claimable status and the old delete operation is paused in the + * same transaction. Root/child desired state remains the delete target. + */ + private static function prepareDeleteRestoreLocked( + string $vpcKey, + array $actor, + string $confirmationName, + string $requestKey, + string $requestHash, + int $expectedRevision, + string $expectedOperationId + ): array { + $vpc = self::lockOwnedForDelete($vpcKey, $actor); + if (!hash_equals((string) $vpc->name, $confirmationName)) { + throw new CloudError('确认名称与 VPC 名称不一致'); + } + + $existing = CloudVpcOperation::where('vpc_id', $vpc->id) + ->where('idempotency_key', $requestKey)->lockForUpdate()->first(); + if ($existing === null) { + // A browser reload can lose its request key after phase one or an + // ambiguous engine response. The unique next revision is already + // reserved, so an exact same-intent request takes over that row. + // Do not restrict this lookup to phase-one statuses: the engine + // response can also be lost after publish, and the ordinary apply + // lane may already have advanced the same operation. + $pendingRestore = CloudVpcOperation::where('vpc_id', $vpc->id) + ->where('kind', 'vpc_delete_restore') + ->where('desired_revision', $expectedRevision + 1) + ->lockForUpdate()->first(); + if ($pendingRestore !== null) { + if (!hash_equals( + (string) $pendingRestore->request_hash, + $requestHash + )) { + throw new CloudError( + 'VPC 删除恢复已有参数不同的待核对请求', + 409 + ); + } + $existing = $pendingRestore; + } + } + if ($existing !== null) { + if ((string) $existing->kind !== 'vpc_delete_restore' + || !hash_equals((string) $existing->request_hash, $requestHash)) { + throw new CloudError('同一请求幂等键的参数不一致', 409); + } + if (self::deleteRestoreWasPublished( + $vpc, + $existing, + $expectedRevision, + $expectedOperationId + )) { + return ['published' => true, 'operation' => $existing]; + } + } + + if ((string) $vpc->state !== 'deleting' + || (int) $vpc->desired_revision !== $expectedRevision + || (string) ($vpc->last_operation_id ?? '') !== $expectedOperationId) { + throw new CloudError('VPC 删除目标已变化,请刷新后重试', 409); + } + $delete = CloudVpcOperation::where('vpc_id', $vpc->id) + ->where('operation_id', $expectedOperationId) + ->where('desired_revision', $expectedRevision) + ->where('kind', 'vpc_delete')->lockForUpdate()->first(); + if ($delete === null) { + throw new CloudError('VPC 删除操作记录不存在', 409); + } + self::assertCurrentDeleteOperationLocked($vpc, $delete); + + $now = time(); + if ($existing !== null) { + if ((int) $existing->desired_revision !== $expectedRevision + 1 + || !in_array((string) $existing->status, [ + 'fencing', 'fence_uncertain', 'failed', + ], true)) { + throw new CloudError('VPC 删除恢复请求状态冲突', 409); + } + $payload = self::restorePayloadFromSourceLocked( + $vpc, + $delete, + (int) $existing->desired_revision, + (string) $existing->operation_id + ); + $stored = json_decode((string) $existing->payload_json, true); + if (!is_array($stored) + || self::canonicalJson($stored) !== self::canonicalJson($payload)) { + throw new CloudError('VPC 删除恢复操作记录损坏', 503); + } + if ((string) $delete->status !== 'fenced') { + if (!self::deleteRestoreStatusSafe($delete, $now)) { + throw new CloudError('VPC 删除任务正在执行,请稍后重试', 409); + } + $metadata = self::deleteRestoreFenceMetadata($delete); + $existing->observed_json = json_encode($metadata, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + } + $delete->status = 'fenced'; + $delete->error_code = 'vpc_delete_restore_pending'; + $delete->finished_at = null; + $delete->lease_token = null; + $delete->lease_until = 0; + $delete->updated_at = $now; + $delete->save(); + $existing->status = 'fencing'; + $existing->error_code = null; + $existing->finished_at = null; + $existing->lease_token = null; + $existing->lease_until = 0; + $existing->updated_at = $now; + $existing->save(); + return [ + 'published' => false, + 'operation' => $existing, + 'payload' => $payload, + ]; + } + + if (!self::deleteRestoreStatusSafe($delete, $now)) { + throw new CloudError('VPC 删除任务正在执行,请稍后重试', 409); + } + $operationId = self::randomKey(); + $revision = $expectedRevision + 1; + $payload = self::restorePayloadFromSourceLocked( + $vpc, + $delete, + $revision, + $operationId + ); + $metadata = self::deleteRestoreFenceMetadata($delete); + $operation = new CloudVpcOperation(); + $operation->operation_id = $operationId; + $operation->vpc_id = $vpc->id; + $operation->user_id = $vpc->user_id; + $operation->kind = 'vpc_delete_restore'; + $operation->desired_revision = $revision; + $operation->idempotency_key = $requestKey; + $operation->request_hash = $requestHash; + $operation->status = 'fencing'; + $operation->payload_json = json_encode($payload, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); + $operation->observed_json = json_encode($metadata, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + $operation->evidence_version = 0; + $operation->error_code = null; + $operation->created_at = $now; + $operation->updated_at = $now; + $operation->finished_at = null; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->attempts = 0; + $operation->save(); + + $delete->status = 'fenced'; + $delete->error_code = 'vpc_delete_restore_pending'; + $delete->finished_at = null; + $delete->lease_token = null; + $delete->lease_until = 0; + $delete->updated_at = $now; + $delete->save(); + return [ + 'published' => false, + 'operation' => $operation, + 'payload' => $payload, + ]; + } + + /** + * Prove that phase two committed independently of mutable root state. + * The normal payload and superseded exact delete row are the durable + * publish markers; a phase-one failure still retains cancelled_delete. + */ + private static function deleteRestoreWasPublished( + CloudVpc $vpc, + CloudVpcOperation $restore, + int $deleteRevision, + string $deleteOperationId + ): bool { + if ((int) $restore->desired_revision !== $deleteRevision + 1 + || !in_array((string) $restore->status, [ + 'queued', 'running', 'uncertain', 'waiting_observed', + 'succeeded', 'failed', 'superseded', + ], true)) { + return false; + } + $payload = json_decode((string) $restore->payload_json, true); + if (!is_array($payload)) { + throw new CloudError('VPC 删除恢复操作记录损坏', 503); + } + if (array_key_exists('cancelled_delete', $payload)) { + return false; + } + $document = self::normalDocumentFromOperationPayload( + $payload, + (string) $restore->operation_id, + (int) $restore->desired_revision + ); + if ((string) ($document['vpc_id'] ?? '') !== (string) $vpc->vpc_key) { + throw new CloudError('VPC 删除恢复操作记录损坏', 503); + } + return CloudVpcOperation::where('vpc_id', $vpc->id) + ->where('operation_id', $deleteOperationId) + ->where('desired_revision', $deleteRevision) + ->where('kind', 'vpc_delete') + ->where('status', 'superseded') + ->exists(); + } + + private static function deleteRestoreStatusSafe( + CloudVpcOperation $operation, + ?int $now = null + ): bool { + $now ??= time(); + return in_array((string) $operation->status, [ + 'queued', 'running', 'uncertain', 'failed', 'waiting_observed', + ], true) && (int) $operation->lease_until <= $now; + } + + /** A phase-one crash remains explicitly resumable after browser reload. */ + private static function hasResumableDeleteRestore( + CloudVpc $vpc, + CloudVpcOperation $delete + ): bool { + if ((string) $delete->status !== 'fenced' + || (int) $delete->lease_until > time()) { + return false; + } + $restore = CloudVpcOperation::where('vpc_id', $vpc->id) + ->where('kind', 'vpc_delete_restore') + ->where('desired_revision', (int) $delete->desired_revision + 1) + ->whereIn('status', ['fencing', 'fence_uncertain', 'failed']) + ->first(); + if ($restore === null) { + return false; + } + $payload = json_decode((string) $restore->payload_json, true); + $cancelled = is_array($payload) + && is_array($payload['cancelled_delete'] ?? null) + ? $payload['cancelled_delete'] : null; + $expectedHash = self::requestHash('vpc_delete_restore', [ + 'confirmation_name' => (string) $vpc->name, + 'expected_revision' => (int) $delete->desired_revision, + 'expected_operation_id' => (string) $delete->operation_id, + ]); + return is_array($cancelled) + && hash_equals($expectedHash, (string) $restore->request_hash) + && (string) ($cancelled['operation_id'] ?? '') + === (string) $delete->operation_id + && (int) ($cancelled['desired_revision'] ?? 0) + === (int) $delete->desired_revision + && self::digestMatches( + (string) $vpc->desired_digest, + (string) ($cancelled['desired_digest'] ?? '') + ); + } + + private static function deleteRestoreFenceMetadata( + CloudVpcOperation $delete + ): array { + $status = (string) $delete->status; + if ($status === 'running') { + // An expired worker may still return later. It cannot regain its + // old lease, and a failed restore resumes the delete as uncertain. + $status = 'uncertain'; + } + return self::canonicalize([ + 'phase' => 'fencing', + 'delete_operation_id' => (string) $delete->operation_id, + 'delete_revision' => (int) $delete->desired_revision, + 'prior_status' => $status, + 'prior_error_code' => self::safeErrorCode( + (string) ($delete->error_code ?? '') + ), + 'prior_finished_at' => $delete->finished_at === null + ? null : (int) $delete->finished_at, + ]); + } + + /** Restore the old delete retry lane after a non-accepted engine call. */ + private static function releaseDeleteRestoreFence( + int $vpcId, + string $restoreOperationId, + CloudError $error + ): void { + $connection = (new CloudVpc())->getConnection(); + $connection->transaction(function () use ( + $vpcId, $restoreOperationId, $error + ): void { + $vpc = CloudVpc::where('id', $vpcId)->lockForUpdate()->first(); + $restore = CloudVpcOperation::where('vpc_id', $vpcId) + ->where('operation_id', $restoreOperationId) + ->where('kind', 'vpc_delete_restore')->lockForUpdate()->first(); + if ($vpc === null || $restore === null + || (string) $restore->status !== 'fencing') { + return; + } + $payload = json_decode((string) $restore->payload_json, true); + $cancelled = is_array($payload) + && is_array($payload['cancelled_delete'] ?? null) + ? $payload['cancelled_delete'] : null; + $metadata = json_decode((string) ($restore->observed_json ?? ''), true); + if (!is_array($cancelled) || !is_array($metadata)) { + return; + } + $deleteOperationId = (string) ($cancelled['operation_id'] ?? ''); + $deleteRevision = (int) ($cancelled['desired_revision'] ?? 0); + $deleteDigest = (string) ($cancelled['desired_digest'] ?? ''); + if ((string) $vpc->state !== 'deleting' + || (string) ($vpc->last_operation_id ?? '') !== $deleteOperationId + || (int) $vpc->desired_revision !== $deleteRevision + || !self::digestMatches((string) $vpc->desired_digest, $deleteDigest)) { + return; + } + $delete = CloudVpcOperation::where('vpc_id', $vpcId) + ->where('operation_id', $deleteOperationId) + ->where('desired_revision', $deleteRevision) + ->where('kind', 'vpc_delete') + ->where('status', 'fenced')->lockForUpdate()->first(); + if ($delete === null) { + return; + } + $priorStatus = (string) ($metadata['prior_status'] ?? ''); + if (!in_array($priorStatus, [ + 'queued', 'uncertain', 'failed', 'waiting_observed', + ], true)) { + $priorStatus = 'uncertain'; + } + $now = time(); + $delete->status = $priorStatus; + $delete->error_code = self::safeErrorCode( + (string) ($metadata['prior_error_code'] ?? '') + ); + $delete->finished_at = $priorStatus === 'failed' + ? (int) ($metadata['prior_finished_at'] ?? $now) : null; + $delete->lease_token = null; + $delete->lease_until = 0; + $delete->updated_at = $now; + $delete->save(); + + $uncertain = $error->httpStatus >= 500; + $restore->status = $uncertain ? 'fence_uncertain' : 'failed'; + $restore->error_code = self::safeErrorCode( + (string) ($error->errorCode ?? '') + ) ?? ($uncertain + ? 'vpc_delete_restore_uncertain' + : 'vpc_delete_restore_failed'); + $restore->finished_at = $uncertain ? null : $now; + $restore->lease_token = null; + $restore->lease_until = 0; + $restore->updated_at = $now; + $restore->save(); + }, 3); + } + + private static function assertCurrentDeleteOperationLocked( + CloudVpc $vpc, + CloudVpcOperation $delete + ): void { + $payload = json_decode((string) $delete->payload_json, true); + if (!is_array($payload) + || (string) ($payload['operation_id'] ?? '') + !== (string) $delete->operation_id + || (int) ($payload['desired_revision'] ?? 0) + !== (int) $delete->desired_revision + || !self::digestMatches( + (string) $vpc->desired_digest, + (string) ($payload['desired_digest'] ?? '') + )) { + throw new CloudError('VPC 删除操作记录损坏', 503); + } + $document = $payload; + unset($document['operation_id'], $document['desired_digest']); + $current = self::deletionDocument($vpc); + if (self::canonicalJson($document) !== self::canonicalJson($current) + || !self::digestMatches( + (string) $vpc->desired_digest, + hash('sha256', self::canonicalJson($current)) + )) { + throw new CloudError('VPC 删除目标与当前账本不一致', 409); + } + } + + /** + * Revalidate the newest pre-delete immutable plan and mint a new normal + * desired target. The delete marker is transport-only and not included + * in the normal-plan digest. + */ + private static function restorePayloadFromSourceLocked( + CloudVpc $vpc, + CloudVpcOperation $delete, + int $newRevision, + string $newOperationId + ): array { + $source = CloudVpcOperation::where('vpc_id', $vpc->id) + ->where('desired_revision', '<', (int) $delete->desired_revision) + ->where('kind', '<>', 'vpc_delete') + ->orderByDesc('desired_revision')->orderByDesc('id') + ->lockForUpdate()->first(); + if ($source === null) { + throw new CloudError('VPC 删除前的有效目标不存在', 409); + } + if ((int) $source->desired_revision + !== (int) $delete->desired_revision - 1) { + // Desired revisions are allocated transactionally and never skip. + // Falling back to an older surviving row would silently roll back + // the tenant's latest member/route/ACL intent. + throw new CloudError('VPC 删除前的连续目标账本缺失', 503); + } + $raw = json_decode((string) $source->payload_json, true); + $sourceDocument = self::normalDocumentFromOperationPayload( + $raw, + (string) $source->operation_id, + (int) $source->desired_revision + ); + $selection = self::restoreSelectionFromDocument($sourceDocument); + $sourceVpc = clone $vpc; + $sourceVpc->desired_revision = (int) $source->desired_revision; + $rebuiltSource = self::desiredDocumentForSelection($sourceVpc, $selection); + if (self::canonicalJson($rebuiltSource) + !== self::canonicalJson($sourceDocument)) { + throw new CloudError('VPC 删除前的目标与当前资源账本不一致', 409); + } + + // A prior detach plan can retain an empty host as its typed removal + // target. Once that deployment has exact terminal absence proof, the + // later root-delete marker excludes it. Reintroducing it here would + // both revive a tombstone and expand the engine restore host set. + $restoreSelection = self::withoutTerminalEmptyHostTombstones( + $vpc, + $selection, + $sourceDocument + ); + self::assertRestoreSelectionNotTerminal($vpc, $restoreSelection); + + $restoreVpc = clone $vpc; + $restoreVpc->desired_revision = $newRevision; + $document = self::desiredDocumentForSelection( + $restoreVpc, + $restoreSelection + ); + $digest = hash('sha256', self::canonicalJson($document)); + $document['desired_digest'] = $digest; + $document['operation_id'] = $newOperationId; + $document['cancelled_delete'] = self::canonicalize([ + 'operation_id' => (string) $delete->operation_id, + 'desired_revision' => (int) $delete->desired_revision, + 'desired_digest' => (string) $vpc->desired_digest, + ]); + return self::canonicalize($document); + } + + private static function withoutTerminalEmptyHostTombstones( + CloudVpc $vpc, + array $selection, + array $sourceDocument + ): array { + $emptyHosts = []; + foreach ($sourceDocument['hosts'] as $host) { + if (($host['members'] ?? null) === []) { + $emptyHosts[(string) $host['host_id']] = true; + } + } + if ($emptyHosts === []) { + return $selection; + } + $terminalHosts = CloudVpcHostDeployment::where('vpc_id', $vpc->id) + ->whereIn('host_id', array_map( + static fn (string $hostId): int => (int) $hostId, + array_keys($emptyHosts) + )) + ->where('state', 'deleted')->orderBy('host_id') + ->lockForUpdate()->get(); + if ($terminalHosts->isEmpty()) { + return $selection; + } + $terminalSet = []; + foreach ($terminalHosts as $deployment) { + // `deleted` alone is not an absence proof. Older workers and + // partial failures may have written that state before persisting + // the matching observed revision/digest. Silently dropping such + // a host would expand the restore trust boundary beyond evidence. + if (!self::hostDeploymentRemovalProven($deployment)) { + throw new CloudError( + 'VPC 删除恢复宿主机缺少完整移除证明', + 409 + ); + } + $terminalSet[(string) $deployment->host_id] = true; + } + $selection['hosts'] = array_values(array_filter( + $selection['hosts'], + static fn (string $hostId): bool => !isset($terminalSet[$hostId]) + )); + return $selection; + } + + /** A terminal identity may prove absence, but may never be resurrected. */ + private static function assertRestoreSelectionNotTerminal( + CloudVpc $vpc, + array $selection + ): void { + foreach ([ + [CloudVpcSubnet::class, 'subnet_key', $selection['subnets']], + [CloudVpcMember::class, 'member_key', $selection['members']], + [CloudVpcRoute::class, 'route_key', $selection['routes']], + [CloudVpcAclRule::class, 'rule_key', $selection['acls']], + ] as [$model, $keyField, $keys]) { + if ($keys !== [] && $model::where('vpc_id', $vpc->id) + ->whereIn($keyField, $keys)->where('state', 'deleted')->exists()) { + throw new CloudError('VPC 删除恢复目标已被最终删除', 409); + } + } + if ($selection['hosts'] !== [] && CloudVpcHostDeployment::where( + 'vpc_id', $vpc->id + )->whereIn('host_id', array_map( + static fn (string $hostId): int => (int) $hostId, + $selection['hosts'] + ))->where('state', 'deleted')->exists()) { + throw new CloudError('VPC 删除恢复宿主机已被最终删除', 409); + } + } + + /** Extract and verify the immutable normal plan from an operation body. */ + private static function normalDocumentFromOperationPayload( + $payload, + string $operationId, + int $revision + ): array { + if (!is_array($payload) + || (string) ($payload['operation_id'] ?? '') !== $operationId + || (int) ($payload['desired_revision'] ?? 0) !== $revision + || preg_match('/^[a-f0-9]{64}$/D', + (string) ($payload['desired_digest'] ?? '')) !== 1) { + throw new CloudError('VPC 历史目标记录损坏', 503); + } + $documentKeys = [ + 'schema_version', 'vpc_id', 'region', 'fabric_id', + 'fabric_revision', 'cidr', 'desired_revision', 'hosts', + 'subnets', 'members', 'routes', 'acls', + ]; + $allowedKeys = array_fill_keys(array_merge( + $documentKeys, + ['desired_digest', 'operation_id', 'cancelled_delete'] + ), true); + foreach (array_keys($payload) as $key) { + if (!is_string($key) || !isset($allowedKeys[$key])) { + throw new CloudError('VPC 历史目标记录包含未知字段', 503); + } + } + $document = []; + foreach ($documentKeys as $key) { + if (!array_key_exists($key, $payload)) { + throw new CloudError('VPC 历史目标记录不完整', 503); + } + $document[$key] = $payload[$key]; + } + $expectedDigest = (string) $payload['desired_digest']; + if (!hash_equals( + $expectedDigest, + hash('sha256', self::canonicalJson($document)) + )) { + throw new CloudError('VPC 历史目标摘要损坏', 503); + } + return self::canonicalize($document); + } + + /** Validate list shape and return exact identity sets for reconstruction. */ + private static function restoreSelectionFromDocument(array $document): array { + $selection = [ + 'hosts' => [], 'subnets' => [], 'members' => [], + 'routes' => [], 'acls' => [], + ]; + $fields = [ + 'subnets' => 'subnet_id', + 'members' => 'member_id', + 'routes' => 'route_id', + 'acls' => 'acl_id', + ]; + foreach ($fields as $field => $idField) { + $rows = $document[$field] ?? null; + if (!is_array($rows) || !array_is_list($rows)) { + throw new CloudError('VPC 历史目标列表损坏', 503); + } + $seen = []; + foreach ($rows as $row) { + if (!is_array($row)) { + throw new CloudError('VPC 历史目标列表损坏', 503); + } + $id = $row[$idField] ?? null; + if (!is_string($id) || preg_match(self::KEY_PATTERN, $id) !== 1 + || isset($seen[$id])) { + throw new CloudError('VPC 历史目标资源标识损坏', 503); + } + $seen[$id] = true; + $selection[$field][] = $id; + } + sort($selection[$field], SORT_STRING); + } + + $hosts = $document['hosts'] ?? null; + if (!is_array($hosts) || !array_is_list($hosts)) { + throw new CloudError('VPC 历史目标宿主机列表损坏', 503); + } + $seenHosts = []; + foreach ($hosts as $host) { + $hostId = is_array($host) ? ($host['host_id'] ?? null) : null; + $memberIds = is_array($host) ? ($host['members'] ?? null) : null; + if (!is_string($hostId) + || preg_match('/^[1-9][0-9]*$/D', $hostId) !== 1 + || isset($seenHosts[$hostId]) + || !is_array($memberIds) || !array_is_list($memberIds)) { + throw new CloudError('VPC 历史目标宿主机列表损坏', 503); + } + $seenMembers = []; + foreach ($memberIds as $memberId) { + if (!is_string($memberId) + || preg_match(self::KEY_PATTERN, $memberId) !== 1 + || isset($seenMembers[$memberId])) { + throw new CloudError('VPC 历史目标宿主机成员损坏', 503); + } + $seenMembers[$memberId] = true; + } + $seenHosts[$hostId] = true; + $selection['hosts'][] = $hostId; + } + sort($selection['hosts'], SORT_STRING); + return $selection; + } + + /** + * Phase two CAS: publish the already fenced normal target and make it + * available to the ordinary apply/reconcile lane. + */ + private static function publishDeleteRestoreLocked( + int $vpcId, + string $restoreOperationId, + array $restorePayload, + array $engineResult, + int $deleteRevision, + string $deleteOperationId + ): void { + $vpc = CloudVpc::where('id', $vpcId)->lockForUpdate()->first(); + if ($vpc === null + || (string) $vpc->state !== 'deleting' + || (int) $vpc->desired_revision !== $deleteRevision + || (string) ($vpc->last_operation_id ?? '') !== $deleteOperationId) { + throw new CloudError('VPC 删除恢复 CAS 失败,请按原操作核对', 409); + } + $restore = CloudVpcOperation::where('vpc_id', $vpcId) + ->where('operation_id', $restoreOperationId) + ->where('kind', 'vpc_delete_restore') + ->where('status', 'fencing')->lockForUpdate()->first(); + $delete = CloudVpcOperation::where('vpc_id', $vpcId) + ->where('operation_id', $deleteOperationId) + ->where('desired_revision', $deleteRevision) + ->where('kind', 'vpc_delete') + ->where('status', 'fenced')->lockForUpdate()->first(); + if ($restore === null || $delete === null + || (int) $restore->desired_revision !== $deleteRevision + 1) { + throw new CloudError('VPC 删除恢复防护记录已变化', 409); + } + if (!self::digestMatches( + (string) $vpc->desired_digest, + (string) ($restorePayload['cancelled_delete']['desired_digest'] ?? '') + ) || (string) ($restorePayload['cancelled_delete']['operation_id'] ?? '') + !== $deleteOperationId + || (int) ($restorePayload['cancelled_delete']['desired_revision'] ?? 0) + !== $deleteRevision) { + throw new CloudError('VPC 删除恢复防护标识不一致', 409); + } + + // Rebuild once more under the phase-two locks. Any child identity + // drift between the engine fence and this CAS leaves both operations + // fenced and requires the same request to be reconciled. + $rebuiltPayload = self::restorePayloadFromSourceLocked( + $vpc, + $delete, + (int) $restore->desired_revision, + (string) $restore->operation_id + ); + if (self::canonicalJson($rebuiltPayload) + !== self::canonicalJson($restorePayload)) { + throw new CloudError('VPC 删除恢复目标在 CAS 前已变化', 409); + } + + $normalDocument = self::normalDocumentFromOperationPayload( + $restorePayload, + (string) $restore->operation_id, + (int) $restore->desired_revision + ); + $selection = self::restoreSelectionFromDocument($normalDocument); + $digest = (string) $restorePayload['desired_digest']; + $revision = (int) $restore->desired_revision; + self::applyDeleteRestoreSelectionLocked( + $vpc, + $selection, + $normalDocument, + $revision, + $digest, + (string) $restore->operation_id + ); + + $vpc->desired_revision = $revision; + $vpc->desired_digest = $digest; + $vpc->state = 'applying'; + $vpc->sync_state = 'pending'; + $vpc->last_operation_id = (string) $restore->operation_id; + $vpc->last_error_code = null; + $vpc->updated_at = $now = time(); + $vpc->save(); + $published = self::desiredDocument($vpc); + if (self::canonicalJson($published) !== self::canonicalJson($normalDocument) + || !hash_equals($digest, + hash('sha256', self::canonicalJson($published)))) { + throw new CloudError('VPC 删除恢复发布目标校验失败', 409); + } + + $normalPayload = $normalDocument; + $normalPayload['desired_digest'] = $digest; + $normalPayload['operation_id'] = (string) $restore->operation_id; + $restore->payload_json = json_encode(self::canonicalize($normalPayload), + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); + $restore->observed_json = json_encode($engineResult, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); + $restore->status = 'queued'; + $restore->error_code = null; + $restore->finished_at = null; + $restore->lease_token = null; + $restore->lease_until = 0; + $restore->updated_at = $now; + $restore->save(); + + $delete->status = 'superseded'; + $delete->error_code = 'vpc_delete_restored'; + $delete->finished_at = $now; + $delete->lease_token = null; + $delete->lease_until = 0; + $delete->updated_at = $now; + $delete->save(); + } + + /** Apply only identities present in the verified pre-delete target. */ + private static function applyDeleteRestoreSelectionLocked( + CloudVpc $vpc, + array $selection, + array $document, + int $revision, + string $digest, + string $operationId + ): void { + $now = time(); + $selectedSubnets = array_fill_keys($selection['subnets'], true); + $selectedMembers = array_fill_keys($selection['members'], true); + $selectedRoutes = array_fill_keys($selection['routes'], true); + $selectedAcls = array_fill_keys($selection['acls'], true); + $subnets = CloudVpcSubnet::where('vpc_id', $vpc->id) - ->whereNotIn('state', ['deleted', 'deleting'])->orderBy('subnet_key')->get(); + ->lockForUpdate()->get(); + foreach ($subnets as $row) { + $selected = isset($selectedSubnets[(string) $row->subnet_key]); + if ((string) $row->state === 'deleted') { + if ($selected) { + throw new CloudError('VPC 删除恢复子网已被最终删除', 409); + } + // Terminal absence proof belongs to its historical target. + // Do not retag it with the restore revision or digest. + continue; + } + $row->state = $selected ? 'applying' : 'deleting'; + $row->desired_revision = $revision; + $row->desired_digest = $digest; + if ($selected) { + $row->last_error_code = null; + } + $row->updated_at = $now; + $row->save(); + } + $members = CloudVpcMember::where('vpc_id', $vpc->id) - ->whereNotIn('state', ['deleted', 'deleting'])->orderBy('member_key')->get(); - $routes = CloudVpcRoute::where('vpc_id', $vpc->id) - ->whereNotIn('state', ['deleted', 'deleting'])->orderBy('route_key')->get(); - $acl = CloudVpcAclRule::where('vpc_id', $vpc->id) - ->whereNotIn('state', ['deleted', 'deleting'])->orderBy('priority')->orderBy('rule_key')->get(); + ->lockForUpdate()->get(); + $selectedPrivateIds = []; + foreach ($members as $row) { + $selected = isset($selectedMembers[(string) $row->member_key]); + if ((string) $row->state === 'deleted') { + if ($selected) { + throw new CloudError('VPC 删除恢复成员已被最终删除', 409); + } + continue; + } + if ($selected) { + $selectedPrivateIds[(int) $row->private_ip_id] = true; + } + $row->state = $selected ? 'applying' : 'deleting'; + $row->desired_revision = $revision; + $row->desired_digest = $digest; + if ($selected) { + $row->last_operation_id = $operationId; + $row->last_error_code = null; + } + $row->updated_at = $now; + $row->save(); + } + + $privateIps = CloudVpcPrivateIp::where('vpc_id', $vpc->id) + ->lockForUpdate()->get(); + foreach ($privateIps as $row) { + $selected = isset($selectedPrivateIds[(int) $row->id]); + if ((string) $row->state === 'deleted') { + if ($selected) { + throw new CloudError('VPC 删除恢复私网 IP 已被最终删除', 409); + } + continue; + } + $row->state = $selected ? 'applying' : 'deleting'; + $row->desired_revision = $revision; + $row->desired_digest = $digest; + if ($selected) { + $row->last_error_code = null; + } + $row->updated_at = $now; + $row->save(); + } + + foreach ([ + [CloudVpcRoute::class, 'route_key', $selectedRoutes], + [CloudVpcAclRule::class, 'rule_key', $selectedAcls], + ] as [$model, $keyField, $selectedKeys]) { + $rows = $model::where('vpc_id', $vpc->id)->lockForUpdate()->get(); + foreach ($rows as $row) { + $selected = isset($selectedKeys[(string) $row->{$keyField}]); + if ((string) $row->state === 'deleted') { + if ($selected) { + throw new CloudError('VPC 删除恢复资源已被最终删除', 409); + } + continue; + } + $row->state = $selected ? 'applying' : 'deleting'; + $row->desired_revision = $revision; + $row->desired_digest = $digest; + $row->updated_at = $now; + $row->save(); + } + } + + $hostMembers = []; + foreach ($document['hosts'] as $host) { + $hostMembers[(string) $host['host_id']] = count($host['members']); + } + $hosts = CloudVpcHostDeployment::where('vpc_id', $vpc->id) + ->lockForUpdate()->get(); + foreach ($hosts as $row) { + $hostId = (string) $row->host_id; + $selected = array_key_exists($hostId, $hostMembers); + $hasMembers = $selected && $hostMembers[$hostId] > 0; + if ((string) $row->state === 'deleted') { + if ($selected) { + throw new CloudError('VPC 删除恢复宿主机账本已被最终删除', 409); + } + continue; + } + if ($selected) { + $row->state = $hasMembers ? 'applying' : 'deleting'; + } else { + $row->state = 'deleting'; + } + $row->fabric_revision = (int) $vpc->fabric_revision; + $row->desired_revision = $revision; + $row->desired_digest = $digest; + $row->last_operation_id = $operationId; + $row->updated_at = $now; + $row->save(); + } + } + + /** Build the exact typed document signed into a VPC operation. */ + public static function desiredDocument(CloudVpc $vpc): array + { + return self::desiredDocumentForSelection($vpc, null); + } + + /** + * Rebuild a prior immutable target from the current identity ledger. + * A non-null selection ignores lifecycle state but never invents an ID; + * it is used only while fencing a root deletion restore. + */ + private static function desiredDocumentForSelection( + CloudVpc $vpc, + ?array $selection + ): array { + $subnetQuery = CloudVpcSubnet::where('vpc_id', $vpc->id); + $memberQuery = CloudVpcMember::where('vpc_id', $vpc->id); + $routeQuery = CloudVpcRoute::where('vpc_id', $vpc->id); + $aclQuery = CloudVpcAclRule::where('vpc_id', $vpc->id); + if ($selection === null) { + $subnetQuery->whereNotIn('state', ['deleted', 'deleting']); + $memberQuery->whereNotIn('state', ['deleted', 'deleting']); + $routeQuery->whereNotIn('state', ['deleted', 'deleting']); + $aclQuery->whereNotIn('state', ['deleted', 'deleting']); + } else { + $subnetQuery->whereIn('subnet_key', $selection['subnets']); + $memberQuery->whereIn('member_key', $selection['members']); + $routeQuery->whereIn('route_key', $selection['routes']); + $aclQuery->whereIn('rule_key', $selection['acls']); + } + $subnets = $subnetQuery->orderBy('subnet_key')->get(); + $members = $memberQuery->orderBy('member_key')->get(); + $routes = $routeQuery->orderBy('route_key')->get(); + $acl = $aclQuery->orderBy('priority')->orderBy('rule_key')->get(); + if ($selection !== null + && (count($selection['subnets']) !== $subnets->count() + || count($selection['members']) !== $members->count() + || count($selection['routes']) !== $routes->count() + || count($selection['acls']) !== $acl->count())) { + throw new CloudError('VPC 删除恢复目标引用的资源记录缺失', 409); + } $region = CloudRegion::where('id', (int) $vpc->region_id)->first(); $fabricRegion = (string) ($vpc->fabric_region ?? ''); if ($region === null || $fabricRegion === '' @@ -2356,7 +3392,9 @@ final class CloudVpcService || (int) $privateIp->subnet_id !== (int) $row->subnet_id || (int) $privateIp->instance_id !== (int) $row->instance_id || (string) $privateIp->private_ip !== (string) $row->private_ip - || in_array((string) $privateIp->state, ['deleted', 'deleting'], true)) { + || ($selection === null + ? in_array((string) $privateIp->state, ['deleted', 'deleting'], true) + : (string) $privateIp->state === 'deleted')) { throw new CloudError('VPC 成员引用记录缺失,无法生成引擎目标', 409); } if ((string) $instance->status !== 'active') { @@ -2390,8 +3428,19 @@ final class CloudVpcService // deployment row is deleting. This gives the engine an explicit, // typed removal target. The host row remains a tombstone until a // later data-plane readback explicitly proves removal. - $deployments = CloudVpcHostDeployment::where('vpc_id', $vpc->id) - ->whereNotIn('state', ['deleted'])->get(); + $deploymentQuery = CloudVpcHostDeployment::where('vpc_id', $vpc->id); + if ($selection === null) { + $deploymentQuery->whereNotIn('state', ['deleted']); + } else { + $deploymentQuery->whereIn('host_id', array_map( + static fn (string $hostId): int => (int) $hostId, + $selection['hosts'] + )); + } + $deployments = $deploymentQuery->get(); + if ($selection !== null && count($selection['hosts']) !== $deployments->count()) { + throw new CloudError('VPC 删除恢复目标引用的宿主机记录缺失', 409); + } foreach ($deployments as $deployment) { $hostMembers[(string) $deployment->host_id] ??= []; } @@ -4817,6 +5866,14 @@ final class CloudVpcService && (int) $currentOperation->evidence_version >= self::OBSERVATION_EVIDENCE_VERSION; } + $canRestoreDelete = (string) $vpc->state === 'deleting' + && $currentOperation instanceof CloudVpcOperation + && (string) $currentOperation->kind === 'vpc_delete' + && (string) $currentOperation->operation_id === $lastOperationId + && (int) $currentOperation->desired_revision + === (int) $vpc->desired_revision + && (self::deleteRestoreStatusSafe($currentOperation) + || self::hasResumableDeleteRestore($vpc, $currentOperation)); $usable = (string) $vpc->state === 'active' && (string) $vpc->sync_state === 'applied' && (int) $vpc->desired_revision === (int) $vpc->observed_revision @@ -4912,6 +5969,7 @@ final class CloudVpcService 'usable' => $usable, 'member_count' => $runtimeMembers->count(), 'setup_state' => $awaitingMembers ? 'awaiting_members' : null, + 'can_restore_delete' => $canRestoreDelete, 'last_operation_id' => $vpc->last_operation_id, 'last_error_code' => self::safeErrorCode($rawErrorCode), 'created_at' => (int) $vpc->created_at, 'updated_at' => (int) $vpc->updated_at, diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index f744ac51..78f43a67 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -28,6 +28,8 @@ $app->group('/api/v1', function () { $this->get('/vpcs/{vpc_key:[0-9a-f]{32}}', 'App\Controllers\CloudVpcController:show'); $this->delete('/vpcs/{vpc_key:[0-9a-f]{32}}', 'App\Controllers\CloudVpcController:delete'); + $this->post('/vpcs/{vpc_key:[0-9a-f]{32}}/delete/restore', + 'App\Controllers\CloudVpcController:restoreDelete'); $this->post('/vpcs/{vpc_key:[0-9a-f]{32}}/members', 'App\Controllers\CloudVpcController:attachMember'); $this->delete('/vpcs/{vpc_key:[0-9a-f]{32}}/members/{member_key:[0-9a-f]{32}}', 'App\Controllers\CloudVpcController:detachMember'); diff --git a/web/public/cloud/vpc.js b/web/public/cloud/vpc.js index 66065dc9..919bc8dd 100644 --- a/web/public/cloud/vpc.js +++ b/web/public/cloud/vpc.js @@ -6,7 +6,8 @@ const state = { selectedKey: '', selected: null, busy: false, stopped: false, detailDirty: false, listEpoch: 0, detailEpoch: 0, listController: null, detailController: null, bootstrapEpoch: 0, refreshEpoch: 0, bootstrapController: null, pollTimer: 0, - deleteRequestKeys: new Map(), createFocusReturn: null, + deleteRequestKeys: new Map(), restoreDeleteRequestKeys: new Map(), + createFocusReturn: null, }; const STATUS_LABELS = Object.freeze({ @@ -14,7 +15,9 @@ const STATUS_LABELS = Object.freeze({ applying: '应用中', waiting: '等待宿主机回读', uncertain: '结果待核对', degraded: '部分失败', active: '已生效', suspended: '网络已暂停', deleting: '清理中', succeeded: '已完成', failed: '失败', queued: '排队中', - running: '执行中', unknown: '状态未知', + running: '执行中', waiting_observed: '等待回读', + fencing: '正在建立撤销围栏', fence_uncertain: '撤销结果待核对', + fenced: '已暂停', superseded: '已被新版本取代', unknown: '状态未知', }); const EMPTY_MEMBER_GATE_CODES = Object.freeze(new Set([ @@ -24,7 +27,7 @@ const EMPTY_MEMBER_GATE_CODES = Object.freeze(new Set([ const OPERATION_LABELS = Object.freeze({ vpc_create: '创建 VPC', member_attach: '加入实例', member_detach: '移除实例', routes_replace: '更新路由', acl_replace: '更新 ACL', vpc_delete: '删除 VPC', - subnet_remove: '移除子网', reconcile: '重新核对', + vpc_delete_restore: '撤销删除', subnet_remove: '移除子网', reconcile: '重新核对', }); function element(tag, className = '', textContent = '') { @@ -225,6 +228,8 @@ function normalizeRoot(value) { setup_state: value.setup_state ? String(value.setup_state) : '', setup_state_known: setupStateKnown, member_count_known: memberCountKnown, + can_restore_delete: value.can_restore_delete === true, + last_operation_id: value.last_operation_id ? String(value.last_operation_id) : '', last_error_code: value.last_error_code ? String(value.last_error_code) : '', }; } @@ -309,6 +314,7 @@ function finishVpcDeletion(message = 'VPC 已完成宿主机清理,私网 CIDR const deletedKey = state.selectedKey; if (deletedKey) { state.deleteRequestKeys.delete(deletedKey); + state.restoreDeleteRequestKeys.delete(deletedKey); // Remove the exact completed root immediately. A best-effort list refresh // follows, but a transient refresh failure must not leave a clickable // stale row that the detail endpoint has already proved deleted. @@ -327,7 +333,7 @@ function finishVpcDeletion(message = 'VPC 已完成宿主机清理,私网 CIDR if (message) { setNotice(message, 'success'); toast(message, 'success'); } } -function lockDeletingDetail(container, reconcile) { +function lockDeletingDetail(container, reconcile, restoreDelete) { if (!selectedRootDeleting()) return; container.dataset.rootDeleting = 'true'; for (const control of container.querySelectorAll('button, input, select, textarea')) { @@ -336,6 +342,10 @@ function lockDeletingDetail(container, reconcile) { // Reconcile does not change desired state; it is the explicit retry path // for a deletion whose host readback is incomplete or uncertain. if (reconcile) reconcile.disabled = state.busy; + // Delete restoration is a distinct, server-authorized higher revision. It + // remains available beside reconcile while every ordinary mutation stays + // locked behind the durable deletion tombstone. + if (restoreDelete) restoreDelete.disabled = state.busy; } function refreshRouteEditorState() { @@ -520,6 +530,7 @@ function renderDetail() { : (rootHint || '配置仍在宿主机应用或核对,期间不要重复创建同一成员。'))))); const actions = element('div', 'vpc-detail-actions'); let reconcile = null; + let restoreDelete = null; if (status === 'awaiting_members') { const addMember = button('添加实例', 'button button-primary small'); addMember.dataset.vpcAction = 'add-member'; @@ -531,6 +542,12 @@ function renderDetail() { reconcile.disabled = state.busy; actions.append(reconcile); } + if (status === 'deleting' && root.can_restore_delete === true) { + restoreDelete = button('撤销删除', 'button button-outline small'); + restoreDelete.dataset.vpcAction = 'restore-delete'; + restoreDelete.disabled = state.busy; + actions.append(restoreDelete); + } const remove = button('删除 VPC', 'button button-outline small danger'); remove.dataset.vpcAction = 'delete'; remove.disabled = state.busy || status === 'deleting'; actions.append(remove); @@ -576,7 +593,7 @@ function renderDetail() { memberSelect.value = previousView.memberInstance; } clearDetailDirty(); - lockDeletingDetail(container, reconcile); + lockDeletingDetail(container, reconcile, restoreDelete); } function renderMembers(doc) { @@ -1096,6 +1113,67 @@ async function reconcile() { await mutate(`/api/v1/vpcs/${encodeURIComponent(state.selectedKey)}/reconcile`, 'POST', {}, '已请求重新核对 VPC 状态。'); } +function restoreDeleteIntent(root) { + const revision = Number(root?.desired_revision); + const operationId = String(root?.last_operation_id || ''); + const vpcKey = String(root?.vpc_key || ''); + if (!Number.isInteger(revision) || revision < 1 + || !/^[0-9a-f]{32}$/.test(operationId) + || !/^[0-9a-f]{32}$/.test(vpcKey)) { + throw new Error('VPC 删除目标身份不完整,请刷新后重试。'); + } + const fingerprint = `${revision}:${operationId}`; + const previous = state.restoreDeleteRequestKeys.get(vpcKey); + const requestId = previous?.fingerprint === fingerprint + ? previous.requestKey : requestKey('vpc-delete-restore'); + state.restoreDeleteRequestKeys.set(vpcKey, { + fingerprint, + requestKey: requestId, + }); + return { + path: `/api/v1/vpcs/${encodeURIComponent(vpcKey)}/delete/restore`, + body: { + confirmation_name: String(root.name || ''), + request_key: requestId, + expected_revision: revision, + expected_operation_id: operationId, + }, + }; +} + +function acknowledgeDeleteRestore(vpcKey) { + // A successful restore closes both sides of the delete cycle. Reusing the + // old delete idempotency key would make a later intentional delete replay + // the superseded operation instead of creating a new desired revision. + state.restoreDeleteRequestKeys.delete(vpcKey); + state.deleteRequestKeys.delete(vpcKey); +} + +async function restoreVpcDelete() { + if (state.busy || !selectedDocumentCurrent()) return; + const root = state.selected.vpc; + if (rootStatus(root) !== 'deleting' || root.can_restore_delete !== true) { + setError('当前 VPC 删除不可撤销,请先刷新详情。'); + return; + } + const confirmation = prompt(`撤销删除会以新版本恢复 VPC“${root.name}”删除前的目标,并重新挂载私网网卡、路由和 ACL;实例网络可能短暂中断。请输入完整 VPC 名称确认:`); + if (confirmation === null) return; + if (confirmation !== root.name) { + toast('VPC 名称不匹配,未撤销删除。', 'error'); + return; + } + let intent; + try { + intent = restoreDeleteIntent(root); + } catch (error) { + setError(error.message); + return; + } + const payload = await mutate(intent.path, 'POST', intent.body, + '撤销删除目标已提交,正在以新版本恢复删除前配置;宿主机精确回读前仍不可操作。'); + if (payload !== null) acknowledgeDeleteRestore(root.vpc_key); +} + async function removeMember(key) { if (!state.selectedKey || state.busy || !allowSelectedMutation() || !confirmDetailMutation() || !confirm('确认让这个实例退出 VPC?宿主机回读确认前,私网 IP 不会重新分配。')) return; @@ -1139,6 +1217,7 @@ function handleClick(event) { return; } if (action === 'reconcile') { reconcile(); return; } + if (action === 'restore-delete') { restoreVpcDelete(); return; } if (action === 'delete') { deleteVpc(); return; } const add = event.target.closest('[data-editor-add]')?.dataset.editorAdd; if (add) { diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 752605cc..7d2e5147 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026101003' scope='global'} +{assign var=venix_asset_version value='2026101004' scope='global'} diff --git a/web/tests/CloudFirewallUiContractTest.php b/web/tests/CloudFirewallUiContractTest.php index 8c91ae63..433b4be6 100644 --- a/web/tests/CloudFirewallUiContractTest.php +++ b/web/tests/CloudFirewallUiContractTest.php @@ -18,7 +18,7 @@ $assert(str_contains($template, 'data-instance-setting-tab="firewall"') && str_contains($template, 'id="vm-firewall-rules"'), 'Instance settings must expose the integrated firewall tab and whole-policy editor.'); -$assert(str_contains($assets, "venix_asset_version value='2026101003'"), +$assert(str_contains($assets, "venix_asset_version value='2026101004'"), 'Shared Venix asset cache key must publish the firewall JavaScript and CSS.'); $assert(str_contains($template, '反欺骗保护始终开启') diff --git a/web/tests/CloudHostFabricEndpointContractTest.php b/web/tests/CloudHostFabricEndpointContractTest.php index 0163e9de..dee975c8 100644 --- a/web/tests/CloudHostFabricEndpointContractTest.php +++ b/web/tests/CloudHostFabricEndpointContractTest.php @@ -128,7 +128,7 @@ $assert(str_contains($fabricClient, 'function endpointSource(value)') && str_contains($fabricClient, 'firewall_competing_service_unknown') && str_contains($fabricClient, 'firewall_manager_conflict'), 'Fabric readiness UI does not distinguish explicit and SSH-IPv4 fallback endpoints.'); -$assert(str_contains($assets, "venix_asset_version value='2026101003'"), +$assert(str_contains($assets, "venix_asset_version value='2026101004'"), 'Changed host/Fabric browser assets did not advance the shared cache key.'); echo "Cloud host Fabric endpoint contracts passed.\n"; diff --git a/web/tests/CloudVpcControlPlaneContractTest.php b/web/tests/CloudVpcControlPlaneContractTest.php index 421264c4..6a019a66 100644 --- a/web/tests/CloudVpcControlPlaneContractTest.php +++ b/web/tests/CloudVpcControlPlaneContractTest.php @@ -182,9 +182,10 @@ $assert(str_contains($service, 'waiting_observed') && str_contains($service, "(\$observed['private_dataplane_verified'] ?? null) === false") && str_contains($service, '$emptyControlPlaneComplete = $noDataPlaneTerminalProof') - && str_contains($service, "(string) \$operation->kind === 'vpc_create'") + && str_contains($service, '$emptyReservationOperation = in_array(') + && str_contains($service, "['vpc_create', 'vpc_delete_restore']") && str_contains($service, '$desiredContractValid') - && str_contains($service, '$emptyCreateLedgerValid') + && str_contains($service, '$emptyReservationLedgerValid') && str_contains($service, '!$pendingChildTombstone') && str_contains($service, '|| $emptyControlPlaneComplete') && str_contains($service, '&& !$emptyControlPlaneComplete') @@ -219,6 +220,21 @@ $assert(str_contains($service, 'public static function delete(') '#^/internal/v1/vpcs/[0-9a-f]{32}/delete$#D') && str_contains($cloudError, 'public readonly ?string $errorCode = null'), 'VPC root deletion lost confirmation, durable operation, exact receipt or CIDR CAS finalization.'); +$assert(str_contains($service, 'public static function restoreDelete(') + && str_contains($service, 'prepareDeleteRestoreLocked') + && str_contains($service, 'publishDeleteRestoreLocked') + && str_contains($service, "'vpc_delete_restore'") + && str_contains($service, "'fencing'") + && str_contains($service, 'cancelled_delete') + && str_contains($service, 'withoutTerminalEmptyHostTombstones') + && str_contains($service, "'can_restore_delete' => \$canRestoreDelete") + && str_contains($controller, 'public function restoreDelete(') + && str_contains($routes, + "'/vpcs/{vpc_key:[0-9a-f]{32}}/delete/restore'") + && str_contains($engineClient, + '[0-9a-f]{32}/delete/restore$#D') + && str_contains($engineClient, 'delete(?:/restore)?'), + 'VPC delete restore lost its fenced two-phase route, projection or engine boundary.'); $assert(!str_contains($routes, "/subnets'") && !str_contains($routes, '/subnets/{') && !str_contains($controller, 'public function addSubnet(') diff --git a/web/tests/CloudVpcDeleteRestoreTest.php b/web/tests/CloudVpcDeleteRestoreTest.php new file mode 100644 index 00000000..ab433a5d --- /dev/null +++ b/web/tests/CloudVpcDeleteRestoreTest.php @@ -0,0 +1,692 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); +$schema = $capsule->schema('default'); + +$schema->create('cloud_regions', static function (Blueprint $table): void { + $table->increments('id'); +}); +$schema->create('cloud_vpcs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('vpc_key', 32)->unique(); + $table->unsignedInteger('user_id'); + $table->unsignedInteger('region_id'); + $table->unsignedInteger('private_network_id'); + $table->string('name', 63); + $table->string('cidr', 18); + $table->string('fabric_id', 64); + $table->string('fabric_region', 64); + $table->unsignedBigInteger('fabric_revision'); + $table->unsignedInteger('max_members'); + $table->string('state', 16); + $table->unsignedBigInteger('desired_revision'); + $table->unsignedBigInteger('observed_revision'); + $table->string('desired_digest', 64); + $table->string('observed_digest', 64)->nullable(); + $table->string('sync_state', 16); + $table->string('last_operation_id', 32)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('last_success_at')->default(0); + $table->unsignedBigInteger('last_failure_at')->default(0); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +$schema->create('cloud_vpc_subnets', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('subnet_key', 32); + $table->unsignedInteger('vpc_id'); + $table->string('cidr', 18); + $table->unsignedBigInteger('network_address')->default(0); + $table->unsignedBigInteger('broadcast_address')->default(0); + $table->unsignedTinyInteger('prefix_length')->default(24); + $table->string('gateway_ip', 15)->nullable(); + $table->string('state', 16); + $table->unsignedBigInteger('desired_revision')->default(0); + $table->unsignedBigInteger('observed_revision')->default(0); + $table->string('desired_digest', 64)->nullable(); + $table->string('observed_digest', 64)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('created_at')->default(0); + $table->unsignedBigInteger('updated_at')->default(0); +}); +$schema->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('uuid', 36); + $table->string('status', 16)->default('active'); + $table->unsignedInteger('host_id')->default(1); +}); +$schema->create('cloud_vpc_members', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('member_key', 32); + $table->unsignedInteger('vpc_id'); + $table->unsignedInteger('subnet_id')->nullable(); + $table->unsignedInteger('private_ip_id')->nullable(); + $table->unsignedInteger('instance_id')->nullable(); + $table->unsignedInteger('host_id')->nullable(); + $table->string('private_ip', 15)->nullable(); + $table->string('state', 16); + $table->unsignedBigInteger('desired_revision')->default(0); + $table->unsignedBigInteger('observed_revision')->default(0); + $table->string('desired_digest', 64)->nullable(); + $table->string('observed_digest', 64)->nullable(); + $table->string('last_operation_id', 32)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('created_at')->default(0); + $table->unsignedBigInteger('updated_at')->default(0); +}); +$schema->create('cloud_vpc_private_ips', static function (Blueprint $table): void { + $table->increments('id'); + $table->unsignedInteger('vpc_id'); + $table->unsignedInteger('subnet_id')->nullable(); + $table->unsignedInteger('member_id')->nullable(); + $table->unsignedInteger('instance_id')->nullable(); + $table->string('private_ip', 15)->nullable(); + $table->string('state', 16); + $table->unsignedBigInteger('desired_revision')->default(0); + $table->unsignedBigInteger('observed_revision')->default(0); + $table->string('desired_digest', 64)->nullable(); + $table->string('observed_digest', 64)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('created_at')->default(0); + $table->unsignedBigInteger('updated_at')->default(0); +}); +$schema->create('cloud_vpc_routes', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('route_key', 32); + $table->unsignedInteger('vpc_id'); + $table->string('destination_cidr', 18)->nullable(); + $table->unsignedInteger('member_id')->nullable(); + $table->unsignedInteger('metric')->default(100); + $table->string('state', 16); + $table->unsignedBigInteger('desired_revision')->default(0); + $table->unsignedBigInteger('observed_revision')->default(0); + $table->string('desired_digest', 64)->nullable(); + $table->string('observed_digest', 64)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('created_at')->default(0); + $table->unsignedBigInteger('updated_at')->default(0); +}); +$schema->create('cloud_vpc_acl_rules', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('rule_key', 32); + $table->unsignedInteger('vpc_id'); + $table->unsignedInteger('priority')->default(1); + $table->string('direction', 8)->default('ingress'); + $table->string('action', 8)->default('allow'); + $table->string('protocol', 8)->default('all'); + $table->string('source_cidr', 18)->default('10.0.0.0/24'); + $table->string('destination_cidr', 18)->default('10.0.0.0/24'); + $table->unsignedInteger('port_from')->nullable(); + $table->unsignedInteger('port_to')->nullable(); + $table->string('state', 16); + $table->unsignedBigInteger('desired_revision')->default(0); + $table->unsignedBigInteger('observed_revision')->default(0); + $table->string('desired_digest', 64)->nullable(); + $table->string('observed_digest', 64)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('created_at')->default(0); + $table->unsignedBigInteger('updated_at')->default(0); +}); +$schema->create('cloud_vpc_host_deployments', static function (Blueprint $table): void { + $table->increments('id'); + $table->unsignedInteger('vpc_id'); + $table->unsignedInteger('host_id'); + $table->unsignedBigInteger('fabric_revision')->default(0); + $table->string('fabric_digest', 64)->nullable(); + $table->unsignedBigInteger('desired_revision')->default(0); + $table->unsignedBigInteger('observed_revision')->default(0); + $table->string('desired_digest', 64)->nullable(); + $table->string('observed_digest', 64)->nullable(); + $table->string('state', 16); + $table->string('last_operation_id', 32)->nullable(); + $table->string('last_error_code', 64)->nullable(); + $table->unsignedBigInteger('last_success_at')->default(0); + $table->unsignedBigInteger('last_failure_at')->default(0); + $table->unsignedBigInteger('created_at')->default(0); + $table->unsignedBigInteger('updated_at')->default(0); +}); +$schema->create('cloud_vpc_operations', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('operation_id', 32)->unique(); + $table->unsignedInteger('vpc_id'); + $table->unsignedInteger('user_id'); + $table->string('kind', 24); + $table->unsignedBigInteger('desired_revision'); + $table->string('idempotency_key', 128); + $table->string('request_hash', 64); + $table->string('status', 16); + $table->text('payload_json'); + $table->text('observed_json')->nullable(); + $table->unsignedSmallInteger('evidence_version')->default(0); + $table->string('error_code', 64)->nullable(); + $table->unsignedBigInteger('created_at')->default(0); + $table->unsignedBigInteger('updated_at')->default(0); + $table->unsignedBigInteger('finished_at')->nullable(); + $table->string('lease_token', 32)->nullable(); + $table->unsignedBigInteger('lease_until')->default(0); + $table->unsignedInteger('attempts')->default(0); + $table->unique(['vpc_id', 'idempotency_key']); + $table->unique(['vpc_id', 'desired_revision']); +}); +$schema->create('cloud_private_networks', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('vpc_key', 32)->unique(); + $table->string('allocation_key', 32)->unique(); + $table->unsignedInteger('user_id'); + $table->unsignedInteger('region_id'); + $table->string('cidr', 18); + $table->string('active_cidr', 18)->nullable(); + $table->unsignedBigInteger('network_address'); + $table->unsignedBigInteger('broadcast_address'); + $table->unsignedTinyInteger('prefix_length'); + $table->string('state', 16); + $table->unsignedBigInteger('revision'); + $table->unsignedBigInteger('released_at')->nullable(); + $table->unsignedBigInteger('updated_at'); +}); +$schema->create('cloud_private_cidr_claims', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('block_cidr', 18)->unique(); + $table->string('allocation_key', 32); +}); + +$canonicalize = new ReflectionMethod(CloudVpcService::class, 'canonicalize'); +$canonicalize->setAccessible(true); +$requestHashMethod = new ReflectionMethod(CloudVpcService::class, 'requestHash'); +$requestHashMethod->setAccessible(true); +$prepare = new ReflectionMethod(CloudVpcService::class, 'prepareDeleteRestoreLocked'); +$prepare->setAccessible(true); +$release = new ReflectionMethod(CloudVpcService::class, 'releaseDeleteRestoreFence'); +$release->setAccessible(true); +$publish = new ReflectionMethod(CloudVpcService::class, 'publishDeleteRestoreLocked'); +$publish->setAccessible(true); +$resumable = new ReflectionMethod(CloudVpcService::class, 'hasResumableDeleteRestore'); +$resumable->setAccessible(true); +$privateMacMethod = new ReflectionMethod(CloudVpcService::class, 'privateMacForMember'); +$privateMacMethod->setAccessible(true); + +$vpcKey = str_repeat('a', 32); +$sourceOperationId = str_repeat('1', 32); +$deleteOperationId = str_repeat('2', 32); +$subnetKey = str_repeat('3', 32); +$deletedSubnetKey = str_repeat('4', 32); +$allocationKey = str_repeat('5', 32); +$memberKey = str_repeat('8', 32); +$instanceUuid = '11111111-2222-4333-8444-555555555555'; +$privateMac = $privateMacMethod->invoke(null, $vpcKey, $memberKey); +$now = time(); +Capsule::table('cloud_regions')->insert(['id' => 1]); + +$sourceDocument = $canonicalize->invoke(null, [ + 'schema_version' => 1, + 'vpc_id' => $vpcKey, + 'region' => 'hk01', + 'fabric_id' => 'fabric-hk01', + 'fabric_revision' => 1, + 'cidr' => '10.0.0.0/24', + 'desired_revision' => 1, + // This was a detach removal target. It has since reached terminal proof + // and must validate as history without being revived by restore. + 'hosts' => [ + ['host_id' => '11', 'members' => []], + ['host_id' => '2', 'members' => [$memberKey]], + ], + 'subnets' => [['subnet_id' => $subnetKey, 'cidr' => '10.0.0.0/24']], + 'members' => [[ + 'member_id' => $memberKey, + 'instance_uuid' => $instanceUuid, + 'host_id' => '2', + 'private_ip_cidr' => '10.0.0.10/32', + 'subnet_id' => $subnetKey, + 'private_mac' => $privateMac, + ]], + 'routes' => [], + 'acls' => [], +]); +$sourceDigest = hash('sha256', json_encode( + $sourceDocument, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES +)); +$sourcePayload = $sourceDocument; +$sourcePayload['desired_digest'] = $sourceDigest; +$sourcePayload['operation_id'] = $sourceOperationId; + +$deleteDocument = $canonicalize->invoke(null, [ + 'schema_version' => 1, + 'vpc_id' => $vpcKey, + 'region' => 'hk01', + 'fabric_id' => 'fabric-hk01', + 'fabric_revision' => 1, + 'cidr' => '10.0.0.0/24', + 'desired_revision' => 2, + 'hosts' => [['host_id' => '2', 'members' => []]], + 'subnets' => [], + 'members' => [], + 'routes' => [], + 'acls' => [], + 'retiring_members' => [[ + 'member_id' => $memberKey, + 'instance_uuid' => $instanceUuid, + 'host_id' => '2', + 'private_ip_cidr' => '10.0.0.10/32', + 'private_mac' => $privateMac, + ]], +]); +$deleteDigest = hash('sha256', json_encode( + $deleteDocument, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES +)); +$deletePayload = $deleteDocument; +$deletePayload['desired_digest'] = $deleteDigest; +$deletePayload['operation_id'] = $deleteOperationId; + +Capsule::table('cloud_vpcs')->insert([ + 'id' => 1, 'vpc_key' => $vpcKey, 'user_id' => 9, 'region_id' => 1, + 'private_network_id' => 1, 'name' => 'restore-me', + 'cidr' => '10.0.0.0/24', 'fabric_id' => 'fabric-hk01', + 'fabric_region' => 'hk01', 'fabric_revision' => 1, 'max_members' => 200, + 'state' => 'deleting', 'desired_revision' => 2, 'observed_revision' => 1, + 'desired_digest' => $deleteDigest, 'observed_digest' => $sourceDigest, + 'sync_state' => 'uncertain', 'last_operation_id' => $deleteOperationId, + 'last_error_code' => 'vpc_result_uncertain', 'last_success_at' => 0, + 'last_failure_at' => $now, 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_vpc_subnets')->insert([ + [ + 'id' => 1, 'subnet_key' => $subnetKey, 'vpc_id' => 1, + 'cidr' => '10.0.0.0/24', 'state' => 'deleting', + 'desired_revision' => 2, 'desired_digest' => $deleteDigest, + ], + [ + 'id' => 2, 'subnet_key' => $deletedSubnetKey, 'vpc_id' => 1, + 'cidr' => '10.0.1.0/24', 'state' => 'deleted', + 'desired_revision' => 1, 'desired_digest' => $sourceDigest, + ], +]); +Capsule::table('cloud_instances')->insert([ + 'id' => 20, 'uuid' => $instanceUuid, 'status' => 'active', 'host_id' => 2, +]); +Capsule::table('cloud_vpc_members')->insert([ + 'id' => 30, 'member_key' => $memberKey, 'vpc_id' => 1, + 'subnet_id' => 1, 'private_ip_id' => 40, 'instance_id' => 20, + 'host_id' => 2, 'private_ip' => '10.0.0.10', 'state' => 'deleting', + 'desired_revision' => 2, 'desired_digest' => $deleteDigest, +]); +Capsule::table('cloud_vpc_private_ips')->insert([ + 'id' => 40, 'vpc_id' => 1, 'subnet_id' => 1, 'member_id' => 30, + 'instance_id' => 20, 'private_ip' => '10.0.0.10', + 'state' => 'deleting', 'desired_revision' => 2, + 'desired_digest' => $deleteDigest, +]); +Capsule::table('cloud_vpc_host_deployments')->insert([ + [ + 'vpc_id' => 1, 'host_id' => 11, 'fabric_revision' => 1, + 'desired_revision' => 1, 'desired_digest' => $sourceDigest, + 'state' => 'deleted', 'last_operation_id' => $sourceOperationId, + ], + [ + 'vpc_id' => 1, 'host_id' => 2, 'fabric_revision' => 1, + 'desired_revision' => 2, 'desired_digest' => $deleteDigest, + 'state' => 'deleting', 'last_operation_id' => $deleteOperationId, + ], +]); +Capsule::table('cloud_vpc_operations')->insert([ + [ + 'operation_id' => $sourceOperationId, 'vpc_id' => 1, 'user_id' => 9, + 'kind' => 'member_detach', 'desired_revision' => 1, + 'idempotency_key' => 'source-operation', + 'request_hash' => str_repeat('6', 64), 'status' => 'succeeded', + 'payload_json' => json_encode($sourcePayload, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), + 'evidence_version' => 2, 'created_at' => $now, 'updated_at' => $now, + ], + [ + 'operation_id' => $deleteOperationId, 'vpc_id' => 1, 'user_id' => 9, + 'kind' => 'vpc_delete', 'desired_revision' => 2, + 'idempotency_key' => 'delete-operation', + 'request_hash' => str_repeat('7', 64), 'status' => 'uncertain', + 'payload_json' => json_encode($deletePayload, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), + 'created_at' => $now, 'updated_at' => $now, + ], +]); +Capsule::table('cloud_private_networks')->insert([ + 'id' => 1, 'vpc_key' => $vpcKey, 'allocation_key' => $allocationKey, + 'user_id' => 9, 'region_id' => 1, 'cidr' => '10.0.0.0/24', + 'active_cidr' => '10.0.0.0/24', 'network_address' => 167772160, + 'broadcast_address' => 167772415, 'prefix_length' => 24, + 'state' => 'active', 'revision' => 1, 'released_at' => null, + 'updated_at' => $now, +]); +Capsule::table('cloud_private_cidr_claims')->insert([ + 'block_cidr' => '10.0.0.0/24', 'allocation_key' => $allocationKey, +]); + +$actor = ['user_id' => 9, 'is_admin' => false, 'can_manage' => false]; +$hash = $requestHashMethod->invoke(null, 'vpc_delete_restore', [ + 'confirmation_name' => 'restore-me', + 'expected_revision' => 2, + 'expected_operation_id' => $deleteOperationId, +]); +$operationCount = static fn (): int => (int) Capsule::table( + 'cloud_vpc_operations' +)->count(); +$expectNoWrite = static function (callable $callback, string $message) use ( + $assert, $operationCount +): void { + $before = $operationCount(); + try { + $callback(); + throw new RuntimeException($message); + } catch (CloudError) { + // Expected fail-closed rejection. + } + $assert($operationCount() === $before + && Capsule::table('cloud_vpc_operations') + ->where('operation_id', str_repeat('2', 32))->value('status') + === 'uncertain', + $message . ' caused a partial write.'); +}; + +$expectNoWrite(static function () use ( + $prepare, $actor, $vpcKey, $hash, $deleteOperationId +): void { + $prepare->invoke(null, $vpcKey, $actor, 'wrong-name', + 'restore-wrong-name', $hash, 2, $deleteOperationId); +}, 'Wrong confirmation name'); +$expectNoWrite(static function () use ( + $prepare, $actor, $vpcKey, $hash, $deleteOperationId +): void { + $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-wrong-cas', $hash, 3, $deleteOperationId); +}, 'Wrong revision CAS'); +$wrongOperationId = str_repeat('9', 32); +$wrongOperationHash = $requestHashMethod->invoke(null, 'vpc_delete_restore', [ + 'confirmation_name' => 'restore-me', + 'expected_revision' => 2, + 'expected_operation_id' => $wrongOperationId, +]); +$expectNoWrite(static function () use ( + $prepare, $actor, $vpcKey, $wrongOperationHash, $wrongOperationId +): void { + $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-wrong-operation', $wrongOperationHash, 2, $wrongOperationId); +}, 'Wrong operation CAS'); +try { + CloudVpcService::restoreDelete($vpcKey, [ + 'user_id' => 10, 'is_admin' => false, 'can_manage' => false, + ], [ + 'confirmation_name' => 'restore-me', + 'request_key' => 'restore-permission', + 'expected_revision' => 2, + 'expected_operation_id' => $deleteOperationId, + ]); + throw new RuntimeException('A non-owner restored the delete.'); +} catch (CloudError $error) { + $assert($error->httpStatus === 404 && $operationCount() === 2, + 'Permission rejection wrote state or returned the wrong status.'); +} + +Capsule::table('cloud_vpc_operations')->where('operation_id', $sourceOperationId) + ->update(['desired_revision' => 0]); +$expectNoWrite(static function () use ( + $prepare, $actor, $vpcKey, $hash, $deleteOperationId +): void { + $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-missing-adjacent-source', $hash, 2, $deleteOperationId); +}, 'Missing adjacent source revision'); +Capsule::table('cloud_vpc_operations')->where('operation_id', $sourceOperationId) + ->update(['desired_revision' => 1]); + +Capsule::table('cloud_vpc_operations')->where('operation_id', $sourceOperationId) + ->update(['payload_json' => '{}']); +$expectNoWrite(static function () use ( + $prepare, $actor, $vpcKey, $hash, $deleteOperationId +): void { + $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-corrupt', $hash, 2, $deleteOperationId); +}, 'Corrupt source payload'); +Capsule::table('cloud_vpc_operations')->where('operation_id', $sourceOperationId) + ->update(['payload_json' => json_encode($sourcePayload, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)]); + +Capsule::table('cloud_vpc_subnets')->where('id', 1) + ->update(['state' => 'deleted']); +$expectNoWrite(static function () use ( + $prepare, $actor, $vpcKey, $hash, $deleteOperationId +): void { + $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-terminal-source', $hash, 2, $deleteOperationId); +}, 'Terminal selected source identity'); +Capsule::table('cloud_vpc_subnets')->where('id', 1) + ->update(['state' => 'deleting']); + +// A deleted label without the matching observed revision/digest is not a +// terminal absence proof and must not silently shrink the engine host fence. +$expectNoWrite(static function () use ( + $prepare, $actor, $vpcKey, $hash, $deleteOperationId +): void { + $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-incomplete-host-proof', $hash, 2, $deleteOperationId); +}, 'Incomplete terminal empty-host proof'); +Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11)->update([ + 'observed_revision' => 1, + 'observed_digest' => $sourceDigest, + 'last_error_code' => null, + 'last_success_at' => $now, +]); + +$prepared = $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-request-one', $hash, 2, $deleteOperationId); +$restoreOperationId = (string) $prepared['operation']->operation_id; +$restorePayload = $prepared['payload']; +$assert(($restorePayload['hosts'] ?? null) === [[ + 'host_id' => '2', 'members' => [$memberKey], + ]] + && ($restorePayload['members'][0]['member_id'] ?? null) === $memberKey + && ($restorePayload['members'][0]['private_ip_cidr'] ?? null) + === '10.0.0.10/32' + && ($restorePayload['members'][0]['private_mac'] ?? null) === $privateMac + && Capsule::table('cloud_vpc_host_deployments') + ->where('host_id', 11)->value('state') === 'deleted', + 'Restore payload lost the active member or revived a terminal empty-host tombstone.'); +$assert(Capsule::table('cloud_vpcs')->where('id', 1)->value('state') === 'deleting' + && (int) Capsule::table('cloud_vpcs')->where('id', 1) + ->value('desired_revision') === 2 + && Capsule::table('cloud_vpc_subnets')->where('id', 1) + ->value('state') === 'deleting' + && Capsule::table('cloud_vpc_members')->where('id', 30) + ->value('state') === 'deleting' + && Capsule::table('cloud_vpc_private_ips')->where('id', 40) + ->value('state') === 'deleting' + && Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2) + ->value('state') === 'deleting' + && Capsule::table('cloud_vpc_operations') + ->where('operation_id', $deleteOperationId)->value('status') === 'fenced' + && Capsule::table('cloud_vpc_operations') + ->where('operation_id', $restoreOperationId)->value('status') === 'fencing', + 'Phase one changed root/children or failed to fence both operations atomically.'); +$assert($resumable->invoke( + null, + CloudVpc::findOrFail(1), + CloudVpcOperation::where('operation_id', $deleteOperationId)->firstOrFail() +) === true, + 'A phase-one crash was not exposed as safely resumable.'); + +// A deterministic engine rejection releases the old delete and leaves the +// restore row terminal/non-claimable. +$release->invoke(null, 1, $restoreOperationId, + new CloudError('conflict', 409, 'vpc_delete_restore_conflict')); +$assert(Capsule::table('cloud_vpc_operations') + ->where('operation_id', $deleteOperationId)->value('status') === 'uncertain' + && Capsule::table('cloud_vpc_operations') + ->where('operation_id', $restoreOperationId)->value('status') === 'failed', + 'Deterministic engine failure did not reopen the exact old delete lane.'); + +// A new browser request key must take over the same reserved operation. +$retry = $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-request-two', $hash, 2, $deleteOperationId); +$assert((string) $retry['operation']->operation_id === $restoreOperationId + && $operationCount() === 3, + 'New request key created a second restore operation instead of takeover.'); +$release->invoke(null, 1, $restoreOperationId, + new CloudError('timeout', 502)); +$assert(Capsule::table('cloud_vpc_operations') + ->where('operation_id', $deleteOperationId)->value('status') === 'uncertain' + && Capsule::table('cloud_vpc_operations') + ->where('operation_id', $restoreOperationId) + ->value('status') === 'fence_uncertain', + 'Ambiguous transport failure was claimable or left the old delete fenced.'); +$acceptedReplay = $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-request-three', $hash, 2, $deleteOperationId); +$assert((string) $acceptedReplay['operation']->operation_id + === $restoreOperationId + && $operationCount() === 3, + 'Engine-accepted replay did not retain one operation identity.'); + +$restorePayload = $acceptedReplay['payload']; +$engineResult = [ + 'vpc_id' => $vpcKey, + 'operation_id' => $restoreOperationId, + 'desired_revision' => 3, + 'desired_digest' => $restorePayload['desired_digest'], + 'state' => 'applying', + 'observed' => [], + 'hosts' => [], +]; +$publish->invoke(null, 1, $restoreOperationId, $restorePayload, + $engineResult, 2, $deleteOperationId); +$storedRestorePayload = json_decode((string) Capsule::table( + 'cloud_vpc_operations' +)->where('operation_id', $restoreOperationId)->value('payload_json'), true); +$assert(Capsule::table('cloud_vpcs')->where('id', 1)->value('state') === 'applying' + && (int) Capsule::table('cloud_vpcs')->where('id', 1) + ->value('desired_revision') === 3 + && Capsule::table('cloud_vpcs')->where('id', 1) + ->value('last_operation_id') === $restoreOperationId + && Capsule::table('cloud_vpc_subnets')->where('id', 1) + ->value('state') === 'applying' + && (int) Capsule::table('cloud_vpc_members')->where('id', 30) + ->value('desired_revision') === 3 + && Capsule::table('cloud_vpc_members')->where('id', 30) + ->value('state') === 'applying' + && Capsule::table('cloud_vpc_members')->where('id', 30) + ->value('desired_digest') === $restorePayload['desired_digest'] + && Capsule::table('cloud_vpc_members')->where('id', 30) + ->value('last_operation_id') === $restoreOperationId + && (int) Capsule::table('cloud_vpc_private_ips')->where('id', 40) + ->value('desired_revision') === 3 + && Capsule::table('cloud_vpc_private_ips')->where('id', 40) + ->value('state') === 'applying' + && Capsule::table('cloud_vpc_private_ips')->where('id', 40) + ->value('desired_digest') === $restorePayload['desired_digest'] + && Capsule::table('cloud_vpc_subnets')->where('id', 2) + ->value('state') === 'deleted' + && (int) Capsule::table('cloud_vpc_subnets')->where('id', 2) + ->value('desired_revision') === 1 + && (int) Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2) + ->value('desired_revision') === 3 + && Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2) + ->value('state') === 'applying' + && Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2) + ->value('desired_digest') === $restorePayload['desired_digest'] + && Capsule::table('cloud_vpc_host_deployments')->where('host_id', 2) + ->value('last_operation_id') === $restoreOperationId + && Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11) + ->value('state') === 'deleted' + && (int) Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11) + ->value('desired_revision') === 1 + && Capsule::table('cloud_vpc_host_deployments')->where('host_id', 11) + ->value('last_operation_id') === $sourceOperationId + && Capsule::table('cloud_vpc_operations') + ->where('operation_id', $deleteOperationId)->value('status') === 'superseded' + && Capsule::table('cloud_vpc_operations') + ->where('operation_id', $restoreOperationId)->value('status') === 'queued' + && !array_key_exists('cancelled_delete', $storedRestorePayload), + 'Engine success did not publish exactly the verified normal target.'); +$assert(Capsule::table('cloud_private_networks')->where('id', 1) + ->value('state') === 'active' + && Capsule::table('cloud_private_networks')->where('id', 1) + ->value('active_cidr') === '10.0.0.0/24' + && Capsule::table('cloud_private_cidr_claims')->count() === 1, + 'Delete restore released the CIDR reservation.'); + +try { + CloudVpcService::recordDeleted($vpcKey, 2, $deleteDigest, [ + 'vpc_id' => $vpcKey, 'operation_id' => $deleteOperationId, + 'desired_revision' => 2, 'desired_digest' => $deleteDigest, + 'deleted' => true, 'state' => 'deleted', 'readback' => 'confirmed', + 'removed_host_ids' => [], 'completed_at' => time(), + ], $deleteOperationId); + throw new RuntimeException('A late old delete receipt won after restore publish.'); +} catch (CloudError $error) { + $assert($error->httpStatus === 409 + && Capsule::table('cloud_vpcs')->where('id', 1)->value('state') + === 'applying', + 'Late delete receipt did not fail the root revision/operation CAS.'); +} + +// A completed phase two remains idempotent even after the ordinary apply lane +// fails or a later mutation supersedes it. The immutable normal payload plus +// the exact superseded delete row prove publish without trusting root state. +Capsule::table('cloud_vpc_operations') + ->where('operation_id', $restoreOperationId) + ->update(['status' => 'failed', 'error_code' => 'vpc_observation_incomplete']); +Capsule::table('cloud_vpcs')->where('id', 1)->update([ + 'state' => 'degraded', 'sync_state' => 'failed', + 'last_error_code' => 'vpc_observation_incomplete', +]); +$failedReplay = $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-request-one', $hash, 2, $deleteOperationId); +$assert(($failedReplay['published'] ?? false) === true + && (string) $failedReplay['operation']->operation_id === $restoreOperationId + && $operationCount() === 3, + 'Published restore lost idempotence after ordinary apply failed.'); + +Capsule::table('cloud_vpc_operations') + ->where('operation_id', $restoreOperationId) + ->update(['status' => 'superseded']); +Capsule::table('cloud_vpcs')->where('id', 1)->update([ + 'state' => 'applying', 'desired_revision' => 4, + 'last_operation_id' => str_repeat('f', 32), +]); +$supersededReplay = $prepare->invoke(null, $vpcKey, $actor, 'restore-me', + 'restore-request-after-reload', $hash, 2, $deleteOperationId); +$assert(($supersededReplay['published'] ?? false) === true + && (string) $supersededReplay['operation']->operation_id + === $restoreOperationId + && $operationCount() === 3, + 'Reload with a new key did not recognize a previously published restore.'); + +echo "VPC delete restore workflow contracts passed.\n"; diff --git a/web/tests/CloudVpcEngineErrorContractTest.php b/web/tests/CloudVpcEngineErrorContractTest.php index dc1f53c4..c703258e 100644 --- a/web/tests/CloudVpcEngineErrorContractTest.php +++ b/web/tests/CloudVpcEngineErrorContractTest.php @@ -71,6 +71,7 @@ $assert = static function ($condition, string $message): void { }; $actor = ['user_id' => 1, 'is_admin' => false, 'can_manage' => false]; $deletePath = '/internal/v1/vpcs/' . str_repeat('a', 32) . '/delete'; +$restorePath = $deletePath . '/restore'; $safeMessage = 'VPC 宿主机清理尚未全部确认,请稍后重新核对'; $request = static function (string $path, int $status, string $errorCode, @@ -127,5 +128,46 @@ foreach (['GET', 'PUT'] as $method) { 'A non-POST VPC delete error crossed the typed 503 trust boundary.'); } +foreach ([ + 'vpc_delete_already_completed', + 'vpc_delete_restore_conflict', + 'vpc_revision_conflict', + 'vpc_delete_restore_identity_mismatch', + 'vpc_delete_restore_manifest_mismatch', + 'vpc_delete_restore_host_set_mismatch', +] as $errorCode) { + $reviewedRestore = $request($restorePath, 409, $errorCode); + $assert($reviewedRestore->httpStatus === 409 + && $reviewedRestore->errorCode === $errorCode + && !str_contains($reviewedRestore->getMessage(), '/root') + && !str_contains($reviewedRestore->getMessage(), 'private'), + 'A reviewed delete-restore error lost its typed safe boundary.'); +} + +$restoreClient = new CloudEngineClient(new Client([ + new VpcErrorTestResponse(200, json_encode([ + 'vpc_id' => str_repeat('a', 32), + ])), +])); +$restoreResult = $restoreClient->vpc('POST', $restorePath, $actor, []); +$assert(($restoreResult['vpc_id'] ?? null) === str_repeat('a', 32), + 'CloudEngineClient::vpc rejected the exact delete-restore resource path.'); + +foreach ([ + $restorePath . '/extra', + '/internal/v1/vpcs/' . str_repeat('g', 32) . '/delete/restore', + '/internal/v1/vpcs/' . str_repeat('a', 32) . '/restore', +] as $invalidRestorePath) { + try { + (new CloudEngineClient(new Client()))->vpc( + 'POST', $invalidRestorePath, $actor, [] + ); + throw new RuntimeException('An invalid delete-restore path was accepted.'); + } catch (CloudError $error) { + $assert($error->httpStatus === 422, + 'Invalid delete-restore path returned the wrong local status.'); + } +} + echo "Cloud VPC engine error-code contract passed.\n"; } diff --git a/web/tests/CloudVpcTombstoneConvergenceTest.php b/web/tests/CloudVpcTombstoneConvergenceTest.php index 368ca470..0922a328 100644 --- a/web/tests/CloudVpcTombstoneConvergenceTest.php +++ b/web/tests/CloudVpcTombstoneConvergenceTest.php @@ -488,6 +488,58 @@ foreach ([ 'A VPC with a non-deleted host deployment was projected as awaiting members.'); } +// Restoring an interrupted root deletion can legitimately return to the same +// empty control-plane reservation as create. The newer restore revision must +// converge on the reviewed no-members proof instead of becoming a permanent +// failed operation that blocks all later VPC mutations. +$vpcKey29 = substr(hash('sha256', 'empty-vpc-29'), 0, 32); +$operation29 = substr(hash('sha256', 'empty-vpc-operation-29'), 0, 32); +$placeholderDigest29 = hash('sha256', 'empty-vpc-digest-29'); +$seedVpc(29, $vpcKey29, 58, $placeholderDigest29, $operation29, + 'vpc_delete_restore'); +$seedSubnet(2029, 29, substr(hash('sha256', 'empty-vpc-subnet-29'), 0, 32), + '10.49.0.0/24', 'applying', 58, $placeholderDigest29); +$desiredJson29 = $canonicalJsonMethod->invoke( + null, + CloudVpcService::desiredDocument(CloudVpc::find(29)) +); +$digest29 = hash('sha256', $desiredJson29); +Capsule::table('cloud_vpcs')->where('id', 29)->update([ + 'desired_digest' => $digest29, +]); +Capsule::table('cloud_vpc_subnets')->where('id', 2029)->update([ + 'desired_digest' => $digest29, +]); +Capsule::table('cloud_vpc_operations')->where('operation_id', $operation29)->update([ + 'payload_json' => $desiredJson29, +]); +CloudVpcService::recordObserved($vpcKey29, 58, $digest29, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, + 'revision' => 0, 'digest' => null, 'state' => 'degraded', + 'next_stage_allowed' => false, 'private_dataplane_verified' => false, + 'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'], + 'hosts' => [], +], $operation29); +$root29 = CloudVpc::where('id', 29)->first(); +$operationRow29 = Capsule::table('cloud_vpc_operations') + ->where('operation_id', $operation29)->first(); +$subnet29 = Capsule::table('cloud_vpc_subnets')->where('id', 2029)->first(); +$projection29 = $root29 === null ? [] : $publicRootMethod->invoke(null, $root29); +$assert($root29 !== null + && (string) ($operationRow29->status ?? '') === 'succeeded' + && ($operationRow29->error_code ?? null) === null + && (string) $root29->state === 'degraded' + && (string) $root29->sync_state === 'applied' + && (int) $root29->observed_revision === 58 + && (string) $root29->observed_digest === $digest29 + && $root29->last_error_code === null + && (string) ($subnet29->state ?? '') === 'active' + && (int) ($subnet29->observed_revision ?? 0) === 58 + && (string) ($subnet29->observed_digest ?? '') === $digest29 + && ($projection29['usable'] ?? true) === false + && ($projection29['setup_state'] ?? null) === 'awaiting_members', + 'An empty delete-restore did not converge to an awaiting-members reservation.'); + // Legacy workers persisted a safe empty create as failed before the control- // plane reservation became a supported terminal outcome. Its GET projection // is compatibility-only: revision/digest and child state remain untouched. diff --git a/web/tests/CloudVpcUserPageBehaviorTest.mjs b/web/tests/CloudVpcUserPageBehaviorTest.mjs index a6470f7e..9a6d4ae0 100644 --- a/web/tests/CloudVpcUserPageBehaviorTest.mjs +++ b/web/tests/CloudVpcUserPageBehaviorTest.mjs @@ -29,6 +29,10 @@ const source = moduleSource saveEditor, cancelReadbacks, applyBootstrapPayload, + lockDeletingDetail, + restoreDeleteIntent, + acknowledgeDeleteRestore, + restoreVpcDelete, };\n`; class Node { @@ -86,10 +90,12 @@ const document = { getElementById: id => nodes[id] || null, }; let clearedTimers = 0; +let uuidCounter = 0; const context = { document, HTMLElement: Node, Option: OptionNode, + crypto: { randomUUID: () => `00000000-0000-4000-8000-${String(++uuidCounter).padStart(12, '0')}` }, clearTimeout() { clearedTimers += 1; }, console, globalThis: null, @@ -112,6 +118,10 @@ const { saveEditor, cancelReadbacks, applyBootstrapPayload, + lockDeletingDetail, + restoreDeleteIntent, + acknowledgeDeleteRestore, + restoreVpcDelete, } = context.__vpcBehavior; const emptyReservation = { @@ -177,6 +187,89 @@ assert.equal(rootStatus({ member_count: 1, }), 'degraded', 'A non-empty VPC must not consume stale awaiting-members projection.'); +const restorableDelete = normalizeRoot({ + vpc_key: '1'.repeat(32), + name: 'restore-me', + state: 'deleting', + sync_state: 'uncertain', + desired_revision: 6, + last_operation_id: '2'.repeat(32), + can_restore_delete: true, +}); +assert.equal(restorableDelete.can_restore_delete, true, + 'Only the explicit server capability may enable deletion restoration.'); +assert.equal(normalizeRoot({ can_restore_delete: 'true' }).can_restore_delete, false, + 'A truthy non-boolean capability must fail closed.'); + +state.selectedKey = restorableDelete.vpc_key; +state.selected = { vpc: restorableDelete, members: [] }; +state.busy = false; +const ordinaryDeleteControl = new Node(); +const reconcileDeleteControl = new Node(); +const restoreDeleteControl = new Node(); +const deletingContainer = { + dataset: {}, + querySelectorAll() { + return [ordinaryDeleteControl, reconcileDeleteControl, restoreDeleteControl]; + }, +}; +lockDeletingDetail(deletingContainer, reconcileDeleteControl, restoreDeleteControl); +assert.equal(deletingContainer.dataset.rootDeleting, 'true'); +assert.equal(ordinaryDeleteControl.disabled, true, + 'Ordinary controls must remain locked throughout root deletion.'); +assert.equal(reconcileDeleteControl.disabled, false); +assert.equal(restoreDeleteControl.disabled, false, + 'The server-authorized restore action must remain available beside reconcile.'); + +state.busy = true; +ordinaryDeleteControl.disabled = false; +reconcileDeleteControl.disabled = false; +restoreDeleteControl.disabled = false; +lockDeletingDetail(deletingContainer, reconcileDeleteControl, restoreDeleteControl); +assert.equal(ordinaryDeleteControl.disabled, true); +assert.equal(reconcileDeleteControl.disabled, true); +assert.equal(restoreDeleteControl.disabled, true, + 'Busy state must lock both exceptional deletion actions.'); +state.busy = false; + +const firstRestoreIntent = restoreDeleteIntent(restorableDelete); +const secondRestoreIntent = restoreDeleteIntent(restorableDelete); +assert.equal(firstRestoreIntent.path, + `/api/v1/vpcs/${restorableDelete.vpc_key}/delete/restore`); +assert.equal(firstRestoreIntent.body.confirmation_name, 'restore-me'); +assert.equal(firstRestoreIntent.body.expected_revision, 6); +assert.equal(firstRestoreIntent.body.expected_operation_id, '2'.repeat(32)); +assert.match(firstRestoreIntent.body.request_key, /^vpc-delete-restore-/); +assert.equal(secondRestoreIntent.body.request_key, firstRestoreIntent.body.request_key, + 'An unacknowledged restore retry must reuse its idempotency key.'); +const nextDeleteRestoreIntent = restoreDeleteIntent({ + ...restorableDelete, + desired_revision: 8, + last_operation_id: '3'.repeat(32), +}); +assert.notEqual(nextDeleteRestoreIntent.body.request_key, + firstRestoreIntent.body.request_key, + 'A later delete identity must receive a new restore idempotency key.'); + +state.deleteRequestKeys.set(restorableDelete.vpc_key, 'stale-delete-request'); +acknowledgeDeleteRestore(restorableDelete.vpc_key); +assert.equal(state.deleteRequestKeys.has(restorableDelete.vpc_key), false, + 'A successful restore must retire the old delete idempotency key.'); +assert.equal(state.restoreDeleteRequestKeys.has(restorableDelete.vpc_key), false, + 'A successful restore must retire its restore idempotency intent.'); + +let restorePrompts = 0; +context.prompt = () => { restorePrompts += 1; return 'restore-me'; }; +state.busy = true; +await restoreVpcDelete(); +assert.equal(restorePrompts, 0, 'Busy state must reject restore before prompting.'); +state.busy = false; +state.selected.vpc.can_restore_delete = false; +await restoreVpcDelete(); +assert.equal(restorePrompts, 0, + 'A deleting root without the explicit server capability must reject restore.'); +state.selected.vpc.can_restore_delete = true; + state.selected = null; state.vpcs = [{ vpc_key: 'empty', ...emptyReservation }]; assert.equal(Boolean(needsPolling()), false, diff --git a/web/tests/CloudVpcUserPageContractTest.php b/web/tests/CloudVpcUserPageContractTest.php index cef37a9c..7efa70c1 100644 --- a/web/tests/CloudVpcUserPageContractTest.php +++ b/web/tests/CloudVpcUserPageContractTest.php @@ -179,7 +179,7 @@ $assert(str_contains($script, "remove.dataset.vpcAction = 'delete'") && str_contains($script, 'confirmation_name: root.name') && str_contains($script, "'DELETE'") && str_contains($script, 'selectedRootDeleting()') - && str_contains($script, 'lockDeletingDetail(container, reconcile)') + && str_contains($script, 'lockDeletingDetail(container, reconcile, restoreDelete)') && str_contains($script, 'finishVpcDeletion()') && str_contains($script, "rootStatus(item) !== 'deleted'") && str_contains($script, @@ -187,6 +187,41 @@ $assert(str_contains($script, "remove.dataset.vpcAction = 'delete'") && str_contains($script, "state.selectedKey = ''"), 'VPC root deletion must require an exact-name confirmation, lock mutations and clear completed selections.'); +$restoreStart = strpos($script, 'function restoreDeleteIntent('); +$restoreEnd = strpos($script, 'async function removeMember(', $restoreStart ?: 0); +$restoreBlock = $restoreStart !== false && $restoreEnd !== false + ? substr($script, $restoreStart, $restoreEnd - $restoreStart) : ''; +$lockDeleteStart = strpos($script, 'function lockDeletingDetail('); +$lockDeleteEnd = strpos($script, 'function refreshRouteEditorState(', $lockDeleteStart ?: 0); +$lockDeleteBlock = $lockDeleteStart !== false && $lockDeleteEnd !== false + ? substr($script, $lockDeleteStart, $lockDeleteEnd - $lockDeleteStart) : ''; +$assert($restoreBlock !== '' && $lockDeleteBlock !== '' + && str_contains($script, "vpc_delete_restore: '撤销删除'") + && str_contains($script, 'restoreDeleteRequestKeys: new Map()') + && str_contains($script, 'value.can_restore_delete === true') + && str_contains($script, "status === 'deleting' && root.can_restore_delete === true") + && str_contains($script, "restoreDelete.dataset.vpcAction = 'restore-delete'") + && str_contains($script, "if (action === 'restore-delete') { restoreVpcDelete(); return; }") + && str_contains($lockDeleteBlock, "container.querySelectorAll('button, input, select, textarea')") + && str_contains($lockDeleteBlock, 'if (reconcile) reconcile.disabled = state.busy') + && str_contains($lockDeleteBlock, 'if (restoreDelete) restoreDelete.disabled = state.busy') + && str_contains($restoreBlock, "requestKey('vpc-delete-restore')") + && str_contains($restoreBlock, 'const fingerprint = `${revision}:${operationId}`') + && str_contains($restoreBlock, 'previous?.fingerprint === fingerprint') + && str_contains($restoreBlock, '/delete/restore`') + && str_contains($restoreBlock, 'confirmation_name: String(root.name') + && str_contains($restoreBlock, 'expected_revision: revision') + && str_contains($restoreBlock, 'expected_operation_id: operationId') + && str_contains($restoreBlock, "await mutate(intent.path, 'POST', intent.body") + && str_contains($restoreBlock, '以新版本恢复') + && str_contains($restoreBlock, '重新挂载私网网卡、路由和 ACL') + && str_contains($restoreBlock, '实例网络可能短暂中断') + && str_contains($restoreBlock, '宿主机精确回读前仍不可操作') + && str_contains($restoreBlock, 'if (payload !== null) acknowledgeDeleteRestore') + && str_contains($restoreBlock, 'state.restoreDeleteRequestKeys.delete(vpcKey)') + && str_contains($restoreBlock, 'state.deleteRequestKeys.delete(vpcKey)'), + 'Deletion restore must be server-authorized, CAS-bound, idempotent and keep ordinary deleting controls locked.'); + $assert(str_contains($script, 'Absence from it is not a') && !str_contains($script, 'const deletionCompleted =') && !str_contains($script, "error?.status === 404 && key === state.selectedKey"),