diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 1ea5dfb6..b16be694 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,35 @@ # 本版验证记录 +## 2026-10-15:VPC 加入/退出持久恢复重构(验证未全绿) + +- 本轮将成员加入、退出和含成员根删除收敛到“实例先关机、desired 先落盘、同一 + `operation_id/revision/digest` 只读恢复”的路径;不再由 VPC 自动关机或开机。 + 已确认写入只有在严格 receipt 解析成功后才完成,响应未知时保留 claim 并禁止 + 重放宿主写入。 +- 旧 operation 的 root、catalog、VM 和 storage claim family 通过单次 CAS 交给 + 显式 reconcile;局部完成时只探测仍未 exact 的宿主。退出后的旧 guest execution + 账本使用独立清理 marker,并在实例停机、persistent XML、seed 和私网数据面均 + 精确不存在后才删除。 +- NetworkManager 的删除/active 回读改为枚举全部 UUID,并核对 id、interface-name、 + stable-id 和 type;ifupdown 同时扫描主配置和 fragments。查询失败、重复 profile、 + 重命名残留、symlink 或畸形 stanza 均失败关闭,不能把未知状态当成已删除。 +- 专项结果:claim family、revision durability、confirmed-write retry **42 passed**; + strict removal receipt+nft **299 passed**,persistence gate+dataplane **102 passed**, + inventory 定向场景 **68 passed**,生成的 apply/remove/observe Shell 通过 + `/bin/sh -n` 与 `/bin/bash -n`。Python `py_compile` 和 `git diff --check` 通过。 +- Python 3.12.14、pytest 9.1.1 的全量引擎结果为 **4145 passed、24 skipped、 + 34 failed、124 subtests passed**。其中 6 项是修改前已有基线:`test_app.py` + 2 项、`test_engine_boundary.py` 1 项、`test_registration_http.py` 3 项。本轮仍有 + 28 项未通过:`test_vpc_observer_optional_results.py` 24 个 Shell 场景缺少 + `MODE`;`test_vpc_route_readback.py` 3 项仍传旧值 `networkmanager` 而生产入口只 + 接受 `networkmanager_nmcli`;`test_vpc_operation_identity.py` 1 项证明单 root + handoff 仍会接受没有已验证 handoff receipt 的任意 retained resources。 +- 按本轮收口决定,这 28 项不再继续修改代码,原样登记并随提交推送。因此当前提交 + **没有通过全量验收,不应在线部署或用于证明真实加入/退出已彻底解决**。尚未执行 + PHP/Node、本地 MariaDB、真实 AlmaLinux/Debian、SSH/libvirt、QGA、跨宿主流量或 + 重启验收;生产环境仍应保持现有部署,不清理旧 operation/claim,也不重复提交 + 结果未知的写操作。 + ## 2026-10-14:VPC 配置完成与成员运行健康解耦 - 现场三条“待回读”并非同一种故障。VPC `5518137d9479fc40bf153c26a3f4c350` diff --git a/docs/VPC_V2_SIMPLIFICATION.md b/docs/VPC_V2_SIMPLIFICATION.md index fe839f9b..bc7fd690 100644 --- a/docs/VPC_V2_SIMPLIFICATION.md +++ b/docs/VPC_V2_SIMPLIFICATION.md @@ -1,11 +1,48 @@ # VPC v2 简化架构与迁移边界 -状态:设计提案,尚未实现。 +状态:v2 目标架构仍是设计提案;2026-10-15 已完成一轮 v1 过渡硬化,尚未实现 +Host Agent、mTLS、v2 schema 或执行后端迁移。 + +验证状态:本轮专项回归通过,但全量引擎仍有 28 项本轮未解决失败;具体测试名和 +部署边界见 `docs/VALIDATION.md` 的 2026-10-15 记录。该快照已按收口决定停止继续 +修改,不能作为生产已修复或可部署的验收结论。 审查基线:`venix` `86fa8f60865b37fa0f45cf620eedd453dcaa0494`。 适用范围:同地区、跨宿主机的 WireGuard + L3 私网。 +## 2026-10-15:v1 过渡硬化已经实现 + +本轮没有把现有 Python/SSH 执行器伪装成 v2,也没有增加第二个宿主写入者。它先把 +最常导致成员加入、退出和根删除卡住的 v1 边界收紧,使当前生产链路可以安全运行到 +真正的 Agent 迁移: + +- 新的成员加入、退出和含成员的根删除只接受所有受影响实例的最新持久化关机证据; + VPC 不再自动关机、自动开机。旧 power workflow 仅用于核对历史操作,不创建新的 + 电源写入。 +- PHP 在写入成员账本前完成停机预检;Engine 在任何宿主写入前再次锁定并核对本轮 + revision 涉及的完整 VM 集合。一个实例重新运行会阻止整轮写入,而不是留下半数 + 宿主已更新、半数未更新的目标。 +- 新 desired root、host 和成员目标在第一次远端写入前持久化。重启后的核对读取同一 + operation/revision/digest,不依赖请求进程内状态。 +- 宿主 apply/remove 只有在唯一、完整、严格匹配的 receipt 解析成功后才算已确认。 + 写入响应丢失、回执畸形或解析异常会保留 durable claim;后续只读核对不能重放写入。 +- 一次 revision 的 VM/storage/root claim family 作为整体恢复。恢复先固定 family + 大小和摘要,对每个成员及宿主执行只读证明,再用数据库 CAS 原子关闭;恢复过程中 + 发现新 desired bytes 时拒绝覆盖。 +- 退出和根删除要求 persistent XML、cloud-init seed、bridge、route/rule、三类 nft + 表、NetworkManager profile 或 ifupdown fragment、共享 Fabric state 和 WireGuard + peer closure 的不存在性证明。NetworkManager 二进制缺失或 daemon 查询失败属于 + 未确认,不能被计数管道误判为 profile 数量为零。 +- 旧记录缺少 `pending_member_mutations` 时仅在可证明的范围内迁移;空列表原样保留, + 不完整记录重建完整成员 fence,畸形或歧义记录进入 `vpc_record_corrupt`,不会猜测。 +- 宿主配置完成与来宾 QGA 健康继续分开。停机实例可以完成持久配置回读;开机后的 + 地址、路由和 link 状态作为运行健康继续核验,不把已经确认的宿主写入重新派发。 + +这些改动是 v1 到 v2 的安全桥梁。下文的单一 Host Agent、完整 host manifest、 +executor epoch 和精简 schema 仍需按迁移门槛单独实现;当前部署仍保持单 Engine +worker 和 Python/SSH 单写入者。 + ## 结论 VPC v2 不再扩展当前 v1 状态机。目标只有一条路线:保留 PHP 业务控制面和 diff --git a/engine/tests/test_libvirt_service.py b/engine/tests/test_libvirt_service.py index 64a7f661..20abb420 100644 --- a/engine/tests/test_libvirt_service.py +++ b/engine/tests/test_libvirt_service.py @@ -17,7 +17,7 @@ import uuid import copy import xml.etree.ElementTree as ET from contextlib import contextmanager -from unittest.mock import patch +from unittest.mock import Mock, patch SERVICE_PATH = pathlib.Path(__file__).resolve().parents[1] / "virsh_panel" / "libvirt_service.py" @@ -2947,6 +2947,102 @@ class VMServiceTests(unittest.TestCase): } return service, created, domain, seed, kwargs + def test_unclaimed_private_configuration_rejects_saved_restore_state(self): + service, created, domain, _seed, kwargs = ( + self.private_seed_readback_fixture()) + + domain.managed_save_image = True + with self.assertRaises(self.error_type) as managed_save: + service.read_private_configuration_unclaimed( + created["uuid"], **kwargs) + self.assertEqual( + managed_save.exception.error_code, + "vpc_vm_managed_save_present", + ) + + domain.managed_save_image = False + for reader in ( + None, + Mock(side_effect=FakeLibvirtError("managed save unavailable")), + ): + with self.subTest(managed_save_reader=reader), patch.object( + domain, "hasManagedSaveImage", reader), \ + self.assertRaises(self.error_type) as unavailable: + service.read_private_configuration_unclaimed( + created["uuid"], **kwargs) + self.assertEqual( + unavailable.exception.error_code, + "vpc_member_power_unavailable", + ) + + domain.snapshots = 1 + with self.assertRaises(self.error_type) as native_snapshot: + service.read_private_configuration_unclaimed( + created["uuid"], **kwargs) + self.assertEqual( + native_snapshot.exception.error_code, + "vpc_vm_native_snapshot_present", + ) + + domain.snapshots = 0 + with patch.object( + domain, "snapshotNum", + Mock(side_effect=FakeLibvirtError("snapshot state unavailable"))), \ + self.assertRaises(self.error_type) as snapshot_unavailable: + service.read_private_configuration_unclaimed( + created["uuid"], **kwargs) + self.assertEqual( + snapshot_unavailable.exception.error_code, + "vpc_vm_native_snapshot_state_unknown", + ) + + def test_unclaimed_private_absence_rejects_saved_restore_state(self): + created = self.service.create_vm(self.spec(name="absence-guard")) + domain = self.backend.domains[0] + domain.running = False + domain.live_root = None + + domain.managed_save_image = True + with self.assertRaises(self.error_type) as managed_save: + self.service.read_private_absence_unclaimed(created["uuid"]) + self.assertEqual( + managed_save.exception.error_code, + "vpc_private_remove_unsafe", + ) + + domain.managed_save_image = False + for reader in ( + None, + Mock(side_effect=FakeLibvirtError("managed save unavailable")), + ): + with self.subTest(managed_save_reader=reader), patch.object( + domain, "hasManagedSaveImage", reader), \ + self.assertRaises(self.error_type) as unavailable: + self.service.read_private_absence_unclaimed(created["uuid"]) + self.assertEqual( + unavailable.exception.error_code, + "vpc_member_power_unavailable", + ) + + domain.snapshots = 1 + with self.assertRaises(self.error_type) as native_snapshot: + self.service.read_private_absence_unclaimed(created["uuid"]) + self.assertEqual( + native_snapshot.exception.error_code, + "vpc_private_remove_unsafe", + ) + + domain.snapshots = 0 + with patch.object( + domain, "snapshotNum", + Mock(side_effect=FakeLibvirtError("snapshot state unavailable"))), \ + self.assertRaises(self.error_type) as snapshot_unavailable: + self.service.read_private_absence_unclaimed(created["uuid"]) + self.assertEqual( + snapshot_unavailable.exception.error_code, + "vpc_vm_native_snapshot_state_unknown", + ) + def test_private_seed_readback_is_exact_and_read_only(self): service, created, _domain, _seed, kwargs = ( self.private_seed_readback_fixture()) @@ -5178,6 +5274,44 @@ class VMServiceTests(unittest.TestCase): self.assertEqual(domain.XMLDesc(1), old_xml) self.assertNotEqual(bytes(seed.contents), old_seed) + def test_vpc_member_stop_proof_rejects_running_and_managed_save(self): + vm = self.service.create_vm(self.spec()) + domain = self.backend.domains[0] + + with self.assertRaises(self.error_type) as running: + self.service.require_vpc_members_stopped([vm["uuid"]]) + self.assertEqual( + running.exception.error_code, "vpc_member_requires_stopped") + + domain.running = False + domain.live_root = None + self.service.require_vpc_members_stopped([vm["uuid"]]) + + domain.managed_save_image = True + with self.assertRaises(self.error_type) as managed_save: + self.service.require_vpc_members_stopped([vm["uuid"]]) + self.assertEqual( + managed_save.exception.error_code, "vpc_member_requires_stopped") + + def test_vpc_member_stop_proof_fails_closed_on_managed_save_read_error(self): + vm = self.service.create_vm(self.spec()) + domain = self.backend.domains[0] + domain.running = False + domain.live_root = None + + for managed_save_reader in ( + None, + Mock(side_effect=FakeLibvirtError("managed save unavailable")), + ): + with self.subTest(reader=managed_save_reader), patch.object( + domain, "hasManagedSaveImage", managed_save_reader), \ + self.assertRaises(self.error_type) as unavailable: + self.service.require_vpc_members_stopped([vm["uuid"]]) + self.assertEqual( + unavailable.exception.error_code, + "vpc_member_power_unavailable", + ) + def test_create_list_resources_and_console(self): resources = self.service.resources() self.assertEqual(resources["networks"], ["default"]) diff --git a/engine/tests/test_vpc.py b/engine/tests/test_vpc.py index 07a3044f..bc0b5112 100644 --- a/engine/tests/test_vpc.py +++ b/engine/tests/test_vpc.py @@ -1002,7 +1002,11 @@ def test_cancelled_vpc_apply_drains_thread_and_saves_before_fabric_enters(): await asyncio.sleep(0) assert vpc_task.done() is False assert fabric_entered.is_set() is False - assert store.get("vpc_desired", "vpc-hk-1") is None + prewrite = json.loads(store.get("vpc_desired", "vpc-hk-1")) + expected = parse_vpc_plan(payload()) + assert prewrite["desired_revision"] == expected.desired_revision + assert prewrite["desired_digest"] == expected.desired_digest + assert "thread_mutation_settled" not in prewrite vpc_thread_release.set() with pytest.raises(asyncio.CancelledError): await vpc_task @@ -2154,13 +2158,15 @@ def test_lifecycle_conflicts_follow_pending_host_execution_closure(): plan, plan, execution_record) is False assert execution_record["hosts"][0].get("last_error_code") is None - # A full mutation still spans both hosts and keeps the original barrier. + # A full revision mutation spans both hosts. One lifecycle conflict blocks + # and marks the whole execution closure before any host write begins. assert coordinator._mark_lifecycle_conflicts( plan, plan, full_record) is True hosts = {host["host_id"]: host for host in full_record["hosts"]} assert hosts["host-a"]["last_error_code"] == ( "vpc_vm_lifecycle_conflict") - assert hosts["host-b"].get("last_error_code") is None + assert hosts["host-b"]["last_error_code"] == ( + "vpc_vm_lifecycle_conflict") def test_lifecycle_conflicts_keep_same_host_sibling_barrier(): @@ -2654,6 +2660,7 @@ def test_delete_pending_projection_does_not_probe_confirmed_offline_host(): "hosts": [{ "host_id": "host-b", "member_ids": [], "retiring": True, "retiring_members": [], + "pending_member_mutations": [], "desired_revision": plan.desired_revision, "desired_digest": plan.desired_digest, }], @@ -2715,6 +2722,8 @@ def test_member_tombstone_closes_from_config_proof_when_sibling_not_ready(): "public_keys": {"host-a": "a" * 44}, } coordinator._fabric_peers_for_plan = lambda _plan, _record: [] + coordinator._require_host_member_mutations_stopped = ( + lambda *_args, **_kwargs: None) coordinator._run_host_dataplane = lambda *args, **kwargs: { "state": "applying", "transport_verified": True, "private_dataplane_verified": False, @@ -2737,6 +2746,7 @@ def test_member_tombstone_closes_from_config_proof_when_sibling_not_ready(): "hosts": [{ "host_id": "host-a", "member_ids": ["member-b"], "retiring_members": [retiring_member_a()], + "pending_member_mutations": [INSTANCE_A], "desired_revision": plan.desired_revision, "desired_digest": plan.desired_digest, }], @@ -2791,6 +2801,7 @@ def test_reconcile_skips_exact_host_so_later_host_gets_next_budget(): "config_observed_revision": plan.desired_revision, "config_observed_digest": plan.desired_digest, "retiring_members": [], + "pending_member_mutations": [], "member_nics": [{ "member_id": member.member_id, "instance_uuid": member.instance_uuid, @@ -2867,6 +2878,7 @@ def test_reconcile_exact_active_hosts_return_without_periodic_probe(): "config_observed_revision": plan.desired_revision, "config_observed_digest": plan.desired_digest, "retiring_members": [], + "pending_member_mutations": [], "probe_attempted_at_ns": attempted_at[host["host_id"]], "member_nics": [{ "member_id": member.member_id, @@ -2900,7 +2912,7 @@ def test_reconcile_exact_active_hosts_return_without_periodic_probe(): assert record["private_dataplane_verified"] is True -def test_pending_host_probe_locks_only_its_execution_host_members(): +def test_exact_stopped_configuration_does_not_reacquire_vm_locks(): target = payload(hosts=[ {"host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": INSTANCE_A, @@ -2942,6 +2954,7 @@ def test_pending_host_probe_locks_only_its_execution_host_members(): "config_observed_revision": plan.desired_revision, "config_observed_digest": plan.desired_digest, "retiring_members": [], + "pending_member_mutations": [], "probe_attempted_at_ns": 200 if member.host_id == "host-a" else 100, "member_nics": [{ "member_id": member.member_id, @@ -2978,7 +2991,7 @@ def test_pending_host_probe_locks_only_its_execution_host_members(): coordinator._reconcile_dataplane_unlocked = no_host_work asyncio.run(coordinator._reconcile_pending_hosts( plan, record, gate, plan)) - assert acquired == [("host-b", INSTANCE_B)] + assert acquired == [] acquired.clear() asyncio.run(coordinator._reconcile_dataplane( @@ -3113,6 +3126,7 @@ def _exact_guest_pending_host(plan): "config_observed_revision": plan.desired_revision, "config_observed_digest": plan.desired_digest, "retiring_members": [], + "pending_member_mutations": [], "member_nics": [{ "member_id": member.member_id, "instance_uuid": member.instance_uuid, @@ -3262,6 +3276,10 @@ def test_runtime_readback_gate_rejects_mutation_and_stale_config(): retiring_host = dict(host, retiring=True) assert coordinator._runtime_readback_eligible( retiring_host, plan, "host-a") is False + pending_mutation = dict( + host, pending_member_mutations=[INSTANCE_A]) + assert coordinator._runtime_readback_eligible( + pending_mutation, plan, "host-a") is False wrong_desired_digest = dict(host, desired_digest="0" * 64) assert coordinator._runtime_readback_eligible( wrong_desired_digest, plan, "host-a") is False @@ -3838,6 +3856,217 @@ def test_guest_probe_limit_is_process_wide_across_vpc_requests(monkeypatch): assert all(result["config_verified"] is True for result in results) +@pytest.mark.parametrize("error_code", ( + "vpc_member_requires_stopped", + "vpc_member_power_unavailable", +)) +def test_member_stop_preflight_blocks_every_host_before_first_write( + error_code): + previous = parse_vpc_plan(payload()) + target = payload( + desired_revision=2, + operation_id="vpc-op-00000002", + hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-b", "members": [{ + "member_id": "member-b", "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}, + ], + ) + plan = parse_vpc_plan(target) + fabric_plan = FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, + ( + FabricNode("host-a", "100.64.0.1", "192.0.2.1:51820", 51820), + FabricNode("host-b", "100.64.0.2", "192.0.2.2:51820", 51820), + ), + "f" * 64, + ) + + class PreflightFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, _plan): + return { + host_id: (SimpleNamespace(manager=object()), {"host_id": host_id}) + for host_id in ("host-a", "host-b") + } + + coordinator = VpcCoordinator( + MemoryStore(), PreflightFabric(hosts=("host-a", "host-b")), + registry=object(), + ) + coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False + coordinator._validate_dataplane_resources = lambda _plan: None + coordinator._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, + } + coordinator._fabric_peers_for_plan = lambda _plan, _record: [] + checked = [] + writes = [] + + def preflight(_profile, host_id, instance_uuids): + checked.append((host_id, tuple(instance_uuids))) + raise VpcError("member not safely stopped", 409, error_code) + + coordinator._require_host_member_mutations_stopped = preflight + coordinator._run_host_dataplane = ( + lambda *args, **kwargs: writes.append(args[1]) or {}) + gate = {"active": True, "available": True} + record = coordinator._desired_record( + plan, gate, state="applying", previous_plan=previous) + + asyncio.run(coordinator._reconcile_dataplane_unlocked( + plan, record, gate, previous_plan=previous)) + + assert checked == [("host-b", (INSTANCE_B,))] + assert writes == [] + assert all(host["state"] != "uncertain" for host in record["hosts"]) + blocked = next(host for host in record["hosts"] + if host["host_id"] == "host-b") + assert blocked["state"] == "applying" + assert blocked["last_error_code"] == error_code + assert blocked["pending_member_mutations"] == [INSTANCE_B] + + +def test_global_preflight_host_resolution_failure_stays_retryable(): + plan = parse_vpc_plan(payload()) + fabric_plan = FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, + (FabricNode( + "host-a", "100.64.0.1", "192.0.2.1:51820", 51820),), + "f" * 64, + ) + + class UnavailableFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, _plan): + raise VpcError( + "host unavailable", 502, "vpc_host_unavailable") + + coordinator = VpcCoordinator( + MemoryStore(), UnavailableFabric(hosts=("host-a",)), + registry=object(), + ) + coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False + coordinator._validate_dataplane_resources = lambda _plan: None + coordinator._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44}, + } + writes = [] + coordinator._run_host_dataplane = ( + lambda *args, **kwargs: writes.append(args[1]) or {}) + gate = {"active": True, "available": True} + record = coordinator._desired_record(plan, gate, state="applying") + + asyncio.run(coordinator._reconcile_dataplane_unlocked( + plan, record, gate, previous_plan=None)) + + assert writes == [] + assert record["state"] == "applying" + assert record["hosts"][0]["state"] == "applying" + assert record["hosts"][0]["last_error_code"] == "vpc_host_unavailable" + assert record["hosts"][0]["pending_member_mutations"] == [INSTANCE_A] + + +def test_stopped_member_config_proof_clears_mutation_and_periodic_work(): + plan = parse_vpc_plan(payload()) + fabric_plan = FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, + (FabricNode( + "host-a", "100.64.0.1", "192.0.2.1:51820", 51820),), + "f" * 64, + ) + + class ConfigFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, _plan): + return { + "host-a": ( + SimpleNamespace(manager=object()), + {"host_id": "host-a"}, + ), + } + + coordinator = VpcCoordinator( + MemoryStore(), ConfigFabric(hosts=("host-a",)), registry=object()) + coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False + coordinator._validate_dataplane_resources = lambda _plan: None + coordinator._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44}, + } + coordinator._fabric_peers_for_plan = lambda _plan, _record: [] + coordinator._require_host_member_mutations_stopped = ( + lambda *_args, **_kwargs: None) + member = plan.members[0] + coordinator._run_host_dataplane = lambda *args, **kwargs: { + "state": "applying", + "transport_verified": True, + "private_dataplane_verified": False, + "config_verified": True, + "host_config_reusable": True, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "member_nics": [{ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "private_ip_cidr": f"{member.private_ip}/32", + "private_mac": derive_private_mac( + plan.vpc_id, member.member_id), + "config_verified": True, + "xml_verified": True, + "seed_verified": True, + "domain_running": False, + "guest_network_verified": False, + "guest_network_readback": "stopped", + "guest_reason_code": "vpc_private_guest_stopped", + "restart_required": False, + }], + "reason_code": "vpc_private_guest_stopped", + } + gate = {"active": True, "available": True} + record = coordinator._desired_record(plan, gate, state="applying") + assert record["hosts"][0]["pending_member_mutations"] == [INSTANCE_A] + + asyncio.run(coordinator._reconcile_dataplane_unlocked( + plan, record, gate, previous_plan=None)) + + host = record["hosts"][0] + assert host["pending_member_mutations"] == [] + assert host["state"] == "applying" + assert host["last_error_code"] == "vpc_private_guest_stopped" + assert coordinator._reconcile_host_selection(plan, record) == (set(), set()) + coordinator._save("vpc_desired", plan.vpc_id, record) + + @asynccontextmanager + async def unexpected_operation_scope(_plan): + raise AssertionError("periodic config-complete read must not claim") + yield + + def unexpected_catalog_lock(): + raise AssertionError("periodic config-complete read must not lock catalog") + + coordinator._operation_scope = unexpected_operation_scope + coordinator._catalog_mutation_lock = unexpected_catalog_lock + periodic = asyncio.run(coordinator.reconcile(plan.vpc_id)) + assert periodic["hosts"][0]["config_verified"] is True + assert periodic["hosts"][0]["private_dataplane_verified"] is False + + def test_request_probe_deadline_stops_next_host_and_keeps_config_proof( monkeypatch): target = payload(hosts=[ @@ -3888,6 +4117,8 @@ def test_request_probe_deadline_stops_next_host_and_keeps_config_proof( "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, } coordinator._fabric_peers_for_plan = lambda _plan, _record: [] + coordinator._require_host_member_mutations_stopped = ( + lambda *_args, **_kwargs: None) monkeypatch.setattr("virsh_panel.vpc.VPC_REQUEST_DEADLINE_SECONDS", 0.05) calls = [] diff --git a/engine/tests/test_vpc_apply_receipts.py b/engine/tests/test_vpc_apply_receipts.py index d87a342f..cbe35555 100644 --- a/engine/tests/test_vpc_apply_receipts.py +++ b/engine/tests/test_vpc_apply_receipts.py @@ -10,11 +10,19 @@ import re import shlex import subprocess import sys +from threading import RLock import pytest from test_vpc_dataplane import _active_observation_markers, _plan from virsh_panel import vpc_dataplane as dp +from virsh_panel.coordination import ( + CoordinationError, + DurableCoordinator, + current_execution, + execution_context, + resource_guard, +) SHELLS = [p for p in ('/bin/sh', '/bin/dash', '/bin/bash') if Path(p).exists()] RECEIPT = ','.join(c * 64 for c in 'abc') @@ -50,6 +58,7 @@ class Adapter(dp.VpcPrivateHostAdapter): self.calls = [] self.observe_error = None self.apply_error = None + self.remove_error = None self.observation = '\n'.join(_active_observation_markers(plan) + [ '__VC_VPC_FIREWALL_MODE__=native_nftables', '__VC_VPC_FIREWALL_STATE__=owned', @@ -69,6 +78,13 @@ class Adapter(dp.VpcPrivateHostAdapter): f'__VC_VPC_DIGEST__={plan.desired_digest}', f'__VC_VPC_NFT_RECEIPT__={RECEIPT}', ]) + self.removed = '\n'.join([ + '__VC_VPC_REMOVED__=yes', + f'__VC_VPC_OWNER__={dp.DATAPLANE_OWNER_MARKER}', + f'__VC_VPC_REVISION__={plan.desired_revision}', + f'__VC_VPC_DIGEST__={plan.desired_digest}', + '__VC_VPC_ROLLBACK__=none', + ]) def _deployment_contract(self): return ('networkmanager_nmcli', '/usr/sbin/ifup', '/usr/sbin/ifdown', @@ -82,13 +98,111 @@ class Adapter(dp.VpcPrivateHostAdapter): return self.applied if '__VC_VPC_REMOVED__=yes' in script: self.calls.append('remove') - return '__VC_VPC_REMOVED__=yes' + if self.remove_error: + raise self.remove_error + return self.removed self.calls.append('observe') if self.observe_error: raise self.observe_error return self.observation +class CoordinationStore: + def __init__(self): + self.records = {} + self.lock = RLock() + + def get(self, kind, key): + with self.lock: + return self.records.get((kind, key)) + + def insert(self, kind, key, value): + with self.lock: + if (kind, key) in self.records: + return False + self.records[kind, key] = value + return True + + def compare_and_set(self, kind, key, expected, value): + with self.lock: + if self.records.get((kind, key)) != expected: + return False + self.records[kind, key] = value + return True + + +def _empty_local(plan): + return replace( + plan, + members=tuple(member for member in plan.members + if member.host_id != plan.host_id), + ) + + +def _strict_removal_observation(*, closure='empty'): + values = { + 'BRIDGE': 'missing', + 'BRIDGE_OWNER': 'missing', + 'NFT_TABLE': 'missing', + 'NFT_TABLE_COUNT': '0', + 'ROUTE_SCOPE_COUNT': '0', + 'RULE_SCOPE_COUNT': '0', + 'NM_PROFILE_COUNT': '0', + 'NM_PROFILE_QUERY': 'ok', + 'IFUP_PROFILE_COUNT': '0', + 'IFUP_PROFILE_QUERY': 'ok', + 'NFT_MAIN_COUNT': '1', + 'NFT_SNAPSHOT': 'yes', + 'FIREWALL_MODE': 'native_nftables', + 'FIREWALL_STATE': 'owned', + 'NFT_FILE_PRESENT': 'no', + 'SYSCTL_FILE_PRESENT': 'no', + 'PROXY_FILE_PRESENT': 'no', + 'IF_FRAGMENT_PRESENT': 'no', + 'OWNER_FILE_PRESENT': 'no', + 'STATE_FILE_PRESENT': 'no', + 'TRANSACTION_PRESENT': 'no', + 'NFT_INCLUDE_PRESENT': 'no', + 'REMOVAL_BRIDGE_ABSENT': 'yes', + 'REMOVAL_ROUTE_ABSENT': 'yes', + 'REMOVAL_RULE_ABSENT': 'yes', + 'REMOVAL_NFT_ABSENT': 'yes', + 'REMOVAL_STATE_CLOSURE': closure, + 'REMOVAL_FABRIC_NFT': 'yes', + 'REMOVAL_WG_PEER_SET': 'exact', + } + return '\n'.join( + f'__VC_VPC_{key}__={value}' for key, value in values.items()) + + +def _assert_unknown_mutation_is_fenced(adapter, mutation, call_name, + exception_type): + coordinator = DurableCoordinator(CoordinationStore()) + first = coordinator.claim(operation_id='vpc-write-first') + with execution_context(coordinator, first): + with pytest.raises(exception_type): + with resource_guard(('catalog:network',)): + mutation() + assert current_execution().execution_unknown.is_set() + assert coordinator.release(first) + claims = list(coordinator.snapshot().values()) + assert len(claims) == 1 + assert claims[0]['kind'] == 'resource' + assert claims[0]['state'] == 'uncertain' + assert claims[0]['resources'] == ['catalog:network'] + assert adapter.calls.count(call_name) == 1 + + before = list(adapter.calls) + second = coordinator.claim(operation_id='vpc-write-second') + with execution_context(coordinator, second): + with pytest.raises(CoordinationError) as blocked: + with resource_guard(('catalog:network',), timeout=0): + mutation() + assert blocked.value.error_code == 'coordination_uncertain' + assert adapter.calls == before + assert coordinator.release(second) + + def test_success_is_reused_across_adapter_instances_without_activating_member(plan): first = Adapter(plan) assert first.apply(plan)['state'] == 'applying' @@ -159,23 +273,44 @@ def test_read_exception_invalidates_receipt_without_mutation(plan, error): @pytest.mark.parametrize('change', [ lambda s: s.replace('APPLIED__=yes', 'APPLIED__=no'), lambda s: s.replace('OWNER__=virt-cloud-vpc-dataplane-v1', 'OWNER__=foreign'), + lambda s: re.sub(r'REVISION__=[^\n]+', 'REVISION__=999', s), + lambda s: re.sub(r'DIGEST__=[^\n]+', 'DIGEST__=' + 'f' * 64, s), lambda s: s.replace('NFT_RECEIPT__=', 'MISSING_RECEIPT__='), lambda s: s.replace(RECEIPT, 'malformed'), lambda s: s + f'\n__VC_VPC_NFT_RECEIPT__={RECEIPT}', lambda s: s + '\n__VC_VPC_APPLIED__=yes', + lambda s: s + f'\n__VC_VPC_OWNER__={dp.DATAPLANE_OWNER_MARKER}', + lambda s: s + '\n' + next( + line for line in s.splitlines() if 'REVISION__=' in line), + lambda s: s + '\n' + next( + line for line in s.splitlines() if 'DIGEST__=' in line), lambda s: s.replace('REVISION__=', 'MISSING_REVISION__='), lambda s: s.replace('DIGEST__=', 'MISSING_DIGEST__='), ]) def test_ambiguous_or_incomplete_apply_never_seeds_skip(plan, change): adapter = Adapter(plan) adapter.applied = change(adapter.applied) - try: + with pytest.raises(dp.VpcDataplaneError) as rejected: adapter.apply(plan) - except dp.VpcDataplaneError: - pass + assert rejected.value.error_code == 'vpc_apply_unconfirmed' assert not dp._APPLY_RECEIPTS +@pytest.mark.parametrize('failure', ['lost', 'incomplete']) +def test_lost_or_incomplete_apply_response_retains_claim_and_blocks_replay( + plan, failure): + adapter = Adapter(plan) + if failure == 'lost': + adapter.apply_error = TimeoutError('apply response lost') + expected = TimeoutError + else: + adapter.applied = adapter.applied.replace( + 'DIGEST__=', 'MISSING_DIGEST__=') + expected = dp.VpcDataplaneError + _assert_unknown_mutation_is_fenced( + adapter, lambda: adapter.apply(plan), 'apply', expected) + + @pytest.mark.parametrize(('key', 'value'), [ ('ssh_config_file', '/generation-b/config'), ('host', '192.0.2.2'), ('port', 2222), ('fingerprint', 'other-key'), ('ssh_target', 'other-alias'), @@ -209,11 +344,425 @@ def test_repair_failure_discards_old_success(plan): def test_remove_discards_success_receipt(plan): adapter = Adapter(plan) adapter.apply(plan) - empty = replace(plan, members=tuple(m for m in plan.members if m.host_id != plan.host_id)) + empty = _empty_local(plan) adapter.remove(empty) assert not dp._APPLY_RECEIPTS +@pytest.mark.parametrize('change', [ + lambda s: s.replace('REMOVED__=yes', 'REMOVED__=no'), + lambda s: s.replace('OWNER__=virt-cloud-vpc-dataplane-v1', 'OWNER__=foreign'), + lambda s: re.sub(r'REVISION__=[^\n]+', 'REVISION__=999', s), + lambda s: re.sub(r'DIGEST__=[^\n]+', 'DIGEST__=' + 'f' * 64, s), + lambda s: s.replace('ROLLBACK__=none', 'ROLLBACK__=partial'), + lambda s: s.replace('REMOVED__=', 'MISSING_REMOVED__='), + lambda s: s.replace('OWNER__=', 'MISSING_OWNER__='), + lambda s: s.replace('REVISION__=', 'MISSING_REVISION__='), + lambda s: s.replace('DIGEST__=', 'MISSING_DIGEST__='), + lambda s: s.replace('ROLLBACK__=', 'MISSING_ROLLBACK__='), + lambda s: s + '\n' + next( + line for line in s.splitlines() if 'REMOVED__=' in line), + lambda s: s + '\n' + next( + line for line in s.splitlines() if 'OWNER__=' in line), + lambda s: s + '\n' + next( + line for line in s.splitlines() if 'REVISION__=' in line), + lambda s: s + '\n' + next( + line for line in s.splitlines() if 'DIGEST__=' in line), + lambda s: s + '\n' + next( + line for line in s.splitlines() if 'ROLLBACK__=' in line), +]) +def test_ambiguous_or_incomplete_remove_never_confirms(plan, change): + adapter = Adapter(plan) + adapter.removed = change(adapter.removed) + with pytest.raises(dp.VpcDataplaneError) as rejected: + adapter.remove(_empty_local(plan)) + assert rejected.value.error_code == 'vpc_remove_unconfirmed' + + +@pytest.mark.parametrize('failure', ['lost', 'incomplete']) +def test_lost_or_incomplete_remove_response_retains_claim_and_blocks_replay( + plan, failure): + adapter = Adapter(plan) + empty = _empty_local(plan) + if failure == 'lost': + adapter.remove_error = TimeoutError('remove response lost') + expected = TimeoutError + else: + adapter.removed = adapter.removed.replace( + 'ROLLBACK__=', 'MISSING_ROLLBACK__=') + expected = dp.VpcDataplaneError + _assert_unknown_mutation_is_fenced( + adapter, lambda: adapter.remove(empty), 'remove', expected) + + +@pytest.mark.parametrize('mutation_name', ['apply', 'remove']) +def test_receipt_marker_parser_exception_retains_claim_and_blocks_replay( + plan, monkeypatch, mutation_name): + adapter = Adapter(plan) + mutation = ( + (lambda: adapter.apply(plan)) + if mutation_name == 'apply' + else (lambda: adapter.remove(_empty_local(plan))) + ) + + def broken_markers(_output): + raise ValueError('marker parser failed') + + monkeypatch.setattr(dp, '_markers', broken_markers) + _assert_unknown_mutation_is_fenced( + adapter, mutation, mutation_name, ValueError) + + +@pytest.mark.parametrize('closure', ['owned', 'empty']) +def test_removal_observer_accepts_exact_remaining_fabric_closure(plan, closure): + adapter = Adapter(plan) + adapter.observation = _strict_removal_observation(closure=closure) + observed = adapter.observe_removal(_empty_local(plan)) + assert observed['removed'] is True + assert observed['readback'] == 'confirmed' + assert adapter.calls == ['observe'] + + +@pytest.mark.parametrize(('query', 'count'), [ + ('unavailable', '0'), + ('failed', '0'), + ('invalid', '0'), + ('ok', '1'), +]) +def test_networkmanager_removal_requires_successful_zero_profile_inventory( + plan, query, count): + adapter = Adapter(plan) + adapter.observation = (_strict_removal_observation() + .replace('NM_PROFILE_QUERY__=ok', + f'NM_PROFILE_QUERY__={query}') + .replace('NM_PROFILE_COUNT__=0', + f'NM_PROFILE_COUNT__={count}')) + with pytest.raises(dp.VpcDataplaneError) as rejected: + adapter.observe_removal(_empty_local(plan)) + assert rejected.value.error_code == 'vpc_remove_readback_mismatch' + + +def test_ifupdown_removal_uses_full_interfaces_inventory_without_nmcli(plan): + class IfupdownAdapter(Adapter): + def _deployment_contract(self): + return ('ifupdown', '/usr/sbin/ifup', '/usr/sbin/ifdown', + '/usr/sbin/ifquery', 'native_nftables') + + adapter = IfupdownAdapter(plan) + adapter.observation = _strict_removal_observation().replace( + 'NM_PROFILE_QUERY__=ok', 'NM_PROFILE_QUERY__=not_applicable') + observed = adapter.observe_removal(_empty_local(plan)) + assert observed['removed'] is True + assert adapter.calls == ['observe'] + + +def test_observer_nmcli_inventory_preserves_query_failure(plan): + rendered = dp.render_observe_script( + _empty_local(plan), deployment_mode='networkmanager_nmcli') + assert 'MODE=networkmanager_nmcli' in rendered + assert '"$NMCLI" -t --escape no -f UUID connection show' in rendered + assert 'vc_nm_inventory_capture scalar connection.id' in rendered + assert 'vc_nm_inventory_capture scalar connection.interface-name' in rendered + assert 'vc_nm_inventory_capture scalar connection.stable-id' in rendered + assert ('connection show --active' + not in dp.NETWORKMANAGER_VPC_TARGET_INVENTORY_SH) + assert '__VC_VPC_NM_PROFILE_QUERY__=$nm_profile_query' in rendered + assert '__VC_VPC_IFUP_PROFILE_QUERY__=$ifup_profile_query' in rendered + + +NM_INVENTORY_MOCK = r''' +import os +import sys + +args = sys.argv[1:] +if '-f' in args and args[args.index('-f') + 1] == 'UUID': + if os.environ.get('NM_LIST_FAIL') == 'yes': + raise SystemExit(9) + rows = os.environ.get('NM_UUID_ROWS', '') + state_path = os.environ.get('NM_LIST_STATE') + if state_path: + try: + seen = int(open(state_path).read()) + except FileNotFoundError: + seen = 0 + open(state_path, 'w').write(str(seen + 1)) + if seen: + rows = os.environ.get('NM_UUID_ROWS_AFTER', rows) + sys.stdout.write(rows) + raise SystemExit(0) +if '-g' not in args: + raise SystemExit(10) +field = args[args.index('-g') + 1] +if field == os.environ.get('NM_FAIL_FIELD'): + raise SystemExit(11) +values = { + 'connection.id': os.environ.get('NM_ID', 'unrelated'), + 'connection.interface-name': os.environ.get('NM_IFACE', '--'), + 'connection.stable-id': os.environ.get('NM_STABLE', '--'), + 'connection.type': os.environ.get('NM_TYPE', 'ethernet'), +} +if field not in values: + raise SystemExit(12) +sys.stdout.write(values[field] + '\n') +''' + + +@pytest.mark.parametrize('shell', SHELLS) +@pytest.mark.parametrize(('values', 'expected'), [ + ({ + 'NM_ID': 'renamed-panel-profile', + 'NM_IFACE': 'vcv-test', + 'NM_STABLE': 'virt-cloud-vpc-dataplane-v1 vpc=vpc-test', + 'NM_TYPE': 'bridge', + }, '1|ok|no'), + ({ + 'NM_ID': 'foreign-name', + 'NM_IFACE': 'vcv-test', + 'NM_STABLE': '--', + 'NM_TYPE': 'bridge', + }, '1|ok|no'), + ({ + 'NM_ID': 'foreign-name', + 'NM_IFACE': 'other0', + 'NM_STABLE': 'virt-cloud-vpc-dataplane-v1 vpc=vpc-test', + 'NM_TYPE': 'bridge', + }, '1|ok|no'), + ({ + 'NM_ID': 'unrelated', + 'NM_IFACE': 'other0', + 'NM_STABLE': '--', + 'NM_TYPE': 'ethernet', + }, '0|ok|no'), +]) +def test_nm_full_inventory_detects_renamed_interface_and_stable_id_profiles( + tmp_path, shell, values, expected): + nmcli = executable(tmp_path / 'nmcli-inventory', NM_INVENTORY_MOCK) + harness = '\n'.join(( + 'set -eu', + 'NMCLI=$1', + 'BRIDGE=vcv-test', + "NM_PROFILE_MARKER='virt-cloud-vpc-dataplane-v1 vpc=vpc-test'", + dp.NETWORKMANAGER_VPC_TARGET_INVENTORY_SH, + 'vc_nm_target_inventory', + "printf '%s|%s|%s\\n' \"$NM_TARGET_COUNT\" " + '"$NM_TARGET_QUERY" "$NM_TARGET_EXACT"', + )) + result = subprocess.run( + [shell, '-s', nmcli], input=harness, text=True, capture_output=True, + env=os.environ | { + 'NM_UUID_ROWS': '11111111-1111-4111-8111-111111111111\n', + **values, + }, check=False) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == expected + + +@pytest.mark.parametrize('shell', SHELLS) +@pytest.mark.parametrize(('values', 'query'), [ + ({'NM_LIST_FAIL': 'yes'}, 'failed'), + ({'NM_FAIL_FIELD': 'connection.interface-name'}, 'failed'), + ({ + 'NM_UUID_ROWS': ( + '11111111-1111-4111-8111-111111111111\n' + '11111111-1111-4111-8111-111111111111\n'), + }, 'invalid'), + ({'NM_ID': 'bad\nsecond-row'}, 'invalid'), + ({ + 'NM_LIST_STATE': 'SET_BY_TEST', + 'NM_UUID_ROWS_AFTER': '', + }, 'invalid'), +]) +def test_nm_full_inventory_fails_closed_on_query_or_ambiguous_output( + tmp_path, shell, values, query): + nmcli = executable(tmp_path / 'nmcli-failure', NM_INVENTORY_MOCK) + harness = '\n'.join(( + 'set -eu', + 'NMCLI=$1', + 'BRIDGE=vcv-test', + "NM_PROFILE_MARKER='virt-cloud-vpc-dataplane-v1 vpc=vpc-test'", + dp.NETWORKMANAGER_VPC_TARGET_INVENTORY_SH, + 'if vc_nm_target_inventory; then exit 91; fi', + "printf '%s\\n' \"$NM_TARGET_QUERY\"", + )) + env = os.environ | { + 'NM_UUID_ROWS': '11111111-1111-4111-8111-111111111111\n', + **values, + } + if env.get('NM_LIST_STATE') == 'SET_BY_TEST': + env['NM_LIST_STATE'] = str(tmp_path / 'list-state') + result = subprocess.run( + [shell, '-s', nmcli], input=harness, text=True, capture_output=True, + env=env, check=False) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == query + + +@pytest.mark.parametrize('shell', SHELLS) +@pytest.mark.parametrize(('source', 'content'), [ + ('main', 'auto lo vcv-test\niface lo inet loopback\n'), + ('fragment', 'iface vcv-test inet manual\n bridge_ports none\n'), + ('fragment', 'allow-hotplug eth0 vcv-test\n'), +]) +def test_ifupdown_full_inventory_finds_main_and_other_fragment_stanzas( + tmp_path, shell, source, content): + root = tmp_path / 'network' + fragments = root / 'interfaces.d' + fragments.mkdir(parents=True) + (root / 'interfaces').write_text( + content if source == 'main' else 'iface lo inet loopback\n') + if source == 'fragment': + (fragments / 'other-config').write_text(content) + harness = '\n'.join(( + 'set -eu', + 'IFUP_ROOT=$1', + 'BRIDGE=vcv-test', + dp.IFUPDOWN_VPC_TARGET_INVENTORY_SH, + 'vc_ifupdown_target_inventory', + "printf '%s|%s\\n' \"$IFUP_TARGET_COUNT\" \"$IFUP_TARGET_QUERY\"", + )) + result = subprocess.run( + [shell, '-s', str(root)], input=harness, text=True, + capture_output=True, check=False) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == '1|ok' + + +@pytest.mark.parametrize('shell', SHELLS) +def test_ifupdown_full_inventory_rejects_symlink_and_malformed_file( + tmp_path, shell): + root = tmp_path / 'network' + fragments = root / 'interfaces.d' + fragments.mkdir(parents=True) + (root / 'interfaces').write_text('iface lo inet loopback\n') + target = tmp_path / 'outside' + target.write_text('iface vcv-test inet manual\n') + (fragments / 'linked').symlink_to(target) + harness = '\n'.join(( + 'set -eu', + 'IFUP_ROOT=$1', + 'BRIDGE=vcv-test', + dp.IFUPDOWN_VPC_TARGET_INVENTORY_SH, + 'if vc_ifupdown_target_inventory; then exit 91; fi', + )) + linked = subprocess.run( + [shell, '-s', str(root)], input=harness, text=True, + capture_output=True, check=False) + assert linked.returncode == 0, linked.stderr + + (fragments / 'linked').unlink() + (fragments / 'malformed').write_text('iface\n') + malformed = subprocess.run( + [shell, '-s', str(root)], input=harness, text=True, + capture_output=True, check=False) + assert malformed.returncode == 0, malformed.stderr + + +@pytest.mark.parametrize(('query', 'count'), [ + ('invalid', '0'), + ('ok', '1'), +]) +def test_removal_receipt_rejects_stale_or_unreadable_ifupdown_inventory( + plan, query, count): + adapter = Adapter(plan) + adapter.observation = (_strict_removal_observation() + .replace('IFUP_PROFILE_QUERY__=ok', + f'IFUP_PROFILE_QUERY__={query}') + .replace('IFUP_PROFILE_COUNT__=0', + f'IFUP_PROFILE_COUNT__={count}')) + with pytest.raises(dp.VpcDataplaneError) as rejected: + adapter.observe_removal(_empty_local(plan)) + assert rejected.value.error_code == 'vpc_remove_readback_mismatch' + + +@pytest.mark.parametrize('shell', SHELLS) +@pytest.mark.parametrize(('mode', 'changes', 'expected'), [ + ('networkmanager_nmcli', {}, 'yes'), + ('networkmanager_nmcli', {'ifup_profile_count': '1'}, 'no'), + ('networkmanager_nmcli', {'ifup_profile_query': 'invalid'}, 'no'), + ('networkmanager_nmcli', {'nm_profile_count': '2'}, 'no'), + ('ifupdown', {}, 'yes'), + ('ifupdown', {'ifup_profile_count': '2'}, 'no'), + ('ifupdown', {'ifup_profile_query': 'invalid'}, 'no'), +]) +def test_active_persistence_requires_exact_full_manager_inventory( + plan, shell, mode, changes, expected): + rendered = dp.render_observe_script( + plan, deployment_mode=mode) + start = rendered.index('if [ "$firewall_mode_state" = owned ]') + end = rendered.index('for artifact_spec in', start) + gate = rendered[start:end] + values = { + 'MODE': mode, + 'firewall_mode_state': 'owned', + 'NFT_MAIN_COUNT': '1', + 'persistence_target': 'yes', + 'opposite_clean': 'yes', + 'nft_file_ok': 'yes', + 'sysctl_file_ok': 'yes', + 'nft_snapshot_state': 'yes', + 'nm_profile_query': ('ok' if mode == 'networkmanager_nmcli' + else 'not_applicable'), + 'nm_profile_count': ('1' if mode == 'networkmanager_nmcli' else '0'), + 'nm_owned': ('yes' if mode == 'networkmanager_nmcli' else 'no'), + 'nm_present': ('yes' if mode == 'networkmanager_nmcli' else 'no'), + 'ifup_profile_query': 'ok', + 'ifup_profile_count': ('0' if mode == 'networkmanager_nmcli' else '1'), + 'ifup_owned': ('no' if mode == 'networkmanager_nmcli' else 'yes'), + **changes, + } + harness = '\n'.join(( + 'set -eu', + *(key + '=' + shlex.quote(value) for key, value in values.items()), + 'persistence=no', + gate, + "printf '%s\\n' \"$persistence\"", + )) + result = subprocess.run( + [shell, '-s'], input=harness, text=True, + capture_output=True, check=False) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines()[-1] == expected + + +STRICT_REMOVAL_KEYS = tuple( + line.removeprefix('__VC_VPC_').split('__=', 1)[0] + for line in _strict_removal_observation().splitlines() +) + + +@pytest.mark.parametrize('key', STRICT_REMOVAL_KEYS) +def test_removal_observer_requires_every_unique_scalar_marker(plan, key): + adapter = Adapter(plan) + lines = _strict_removal_observation().splitlines() + prefix = f'__VC_VPC_{key}__=' + adapter.observation = '\n'.join( + line for line in lines if not line.startswith(prefix)) + with pytest.raises(dp.VpcDataplaneError) as missing: + adapter.observe_removal(_empty_local(plan)) + assert missing.value.error_code == 'vpc_remove_readback_mismatch' + + adapter.calls.clear() + matching = next(line for line in lines if line.startswith(prefix)) + adapter.observation = '\n'.join(lines + [matching]) + with pytest.raises(dp.VpcDataplaneError) as duplicate: + adapter.observe_removal(_empty_local(plan)) + assert duplicate.value.error_code == 'vpc_remove_readback_mismatch' + + +def test_removal_observer_rejects_active_only_fabric_markers(plan): + adapter = Adapter(plan) + adapter.observation = '\n'.join([ + _strict_removal_observation() + .replace('__VC_VPC_REMOVAL_FABRIC_NFT__=yes\n', '') + .replace('__VC_VPC_REMOVAL_WG_PEER_SET__=exact', ''), + '__VC_VPC_FABRIC_NFT__=yes', + '__VC_VPC_WG_PEER_SET__=exact', + ]) + with pytest.raises(dp.VpcDataplaneError) as rejected: + adapter.observe_removal(_empty_local(plan)) + assert rejected.value.error_code == 'vpc_remove_readback_mismatch' + + def test_cache_is_bounded_under_concurrent_adapters(monkeypatch): monkeypatch.setattr(dp, '_APPLY_RECEIPT_LIMIT', 8) with ThreadPoolExecutor(max_workers=8) as executor: @@ -250,6 +799,34 @@ def executable(path, body): return str(path) +@pytest.mark.parametrize('shell', SHELLS) +@pytest.mark.parametrize(('scenario', 'absent'), [ + ('missing', True), + ('healthy', False), + ('inventory_error', False), + ('malformed_inventory', False), + ('duplicate', False), +]) +def test_removal_nft_absence_requires_complete_strict_family_inventories( + tmp_path, plan, shell, scenario, absent): + nft = executable(tmp_path / 'nft-absence', NFT_MOCK) + rendered = dp.render_observe_script(_empty_local(plan)) + start = rendered.index('vc_vpc_target_nft_absent() {') + end = rendered.index('firewall_mode_state=owned', start) + function = rendered[start:end] + script = '\n'.join(( + 'set -eu', + 'NFT=$1', + 'TABLE=owned', + function, + 'vc_vpc_target_nft_absent', + )) + result = subprocess.run( + [shell, '-s', nft], input=script, text=True, capture_output=True, + env=os.environ | {'SCENARIO': scenario}, check=False) + assert (result.returncode == 0) is absent, result.stderr + + @pytest.mark.parametrize('shell', SHELLS) @pytest.mark.parametrize('scenario', ['healthy', 'verdict_drift', 'inventory_error', 'malformed_inventory', 'read_error', 'missing', 'duplicate', 'empty']) diff --git a/engine/tests/test_vpc_claim_family_recovery.py b/engine/tests/test_vpc_claim_family_recovery.py new file mode 100644 index 00000000..e951471d --- /dev/null +++ b/engine/tests/test_vpc_claim_family_recovery.py @@ -0,0 +1,1661 @@ +"""Exact VPC claim-family recovery without uncertain write replay.""" + +import asyncio +import hashlib +import json +import time +from types import SimpleNamespace + +import pytest + +from virsh_panel.coordination import ( + CoordinationClaimRejected, CoordinationError, DurableCoordinator, + current_execution, +) +from virsh_panel.fabric import FabricNode, FabricPlan +from test_durable_fabric_jobs import FabricStore +from test_engine_coordination import request, running_engine +from test_vpc import ( + FakeFabric, INSTANCE_A, INSTANCE_B, confirm_delete_hosts, delete_payload, + payload, +) +from test_vpc_operation_identity import ( + coordination_document, explicit_request, identity, retiring_members, seed, +) +from virsh_panel.vpc import ( + VPC_DELETE_RECEIPT_KIND, VPC_GUEST_EXEC_KIND, VpcCoordinator, + parse_vpc_delete_plan, parse_vpc_plan, +) +from virsh_panel.vpc_dataplane import derive_private_mac + + +def retained_family( + coordinator, plan, *, include_catalog=True, include_member=True, + include_storage_member=True, extra_resources=(), +): + member = plan.members[0] + vm_resource = f"vm:{member.host_id}:{member.instance_uuid}" + root = coordinator.claim( + operation_id=plan.operation_id, kind="background", + metadata=identity(plan)) + children = [] + if include_catalog: + children.append(coordinator.claim( + operation_id=plan.operation_id, + resources=("catalog:network",), + execution_id=root.execution_id, kind="resource", + metadata=identity(plan))) + if include_member: + children.append(coordinator.claim( + operation_id=plan.operation_id, + resources=(vm_resource,), + execution_id=root.execution_id, kind="resource", + metadata=identity(plan))) + if include_storage_member: + children.append(coordinator.claim( + operation_id=plan.operation_id, + resources=( + f"storage:{member.host_id}", + vm_resource, + ), + execution_id=root.execution_id, kind="resource", + metadata=identity(plan))) + extra = tuple( + coordinator.claim( + operation_id=plan.operation_id, resources=resources, + execution_id=root.execution_id, kind="resource", + metadata=identity(plan), + ) + for resources in extra_resources + ) + for claim in (*children, *extra, root): + assert coordinator.retain(claim) + rows = coordinator.snapshot() + return tuple(rows[token] for token in sorted( + (root.token, *(claim.token for claim in children), + *(claim.token for claim in extra)))) + + +def remaining_member_claims(plan): + return tuple( + (f"vm:{member.host_id}:{member.instance_uuid}",) + for member in plan.members[1:] + ) + + +def exact_configuration_record(vpc, plan, record): + candidate = json.loads(json.dumps(record)) + for host in candidate["hosts"]: + members = tuple( + member for member in plan.members + if member.host_id == host["host_id"]) + host.update({ + "state": "applying", + "transport_verified": True, + "private_dataplane_verified": False, + "config_verified": True, + "host_config_reusable": True, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "member_ids": [member.member_id for member in members], + "retiring_members": [], + "pending_member_mutations": [], + "member_nics": [{ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "private_ip_cidr": f"{member.private_ip}/32", + "private_mac": ( + member.private_mac or derive_private_mac( + plan.vpc_id, member.member_id)), + "config_verified": True, + "xml_verified": True, + "seed_verified": True, + "domain_running": False, + "guest_network_verified": False, + } for member in members], + }) + vpc._finish(candidate, vpc._fabric_gate(plan)) + return candidate + + +def retained_multi_host_family(coordinator, plan, identities): + root = coordinator.claim( + operation_id=plan.operation_id, kind="background", + metadata=identity(plan)) + children = [coordinator.claim( + operation_id=plan.operation_id, + resources=("catalog:network",), + execution_id=root.execution_id, kind="resource", + metadata=identity(plan))] + children.extend( + coordinator.claim( + operation_id=plan.operation_id, + resources=(f"vm:{host_id}:{instance_uuid}",), + execution_id=root.execution_id, kind="resource", + metadata=identity(plan), + ) + for host_id, instance_uuid in sorted(identities) + ) + for claim in (*children, root): + assert coordinator.retain(claim) + rows = coordinator.snapshot() + return tuple(rows[token] for token in sorted( + (root.token, *(claim.token for claim in children)))) + + +def exact_active_host(plan, host): + host_id = host["host_id"] + members = tuple( + member for member in plan.members if member.host_id == host_id) + exact = dict(host) + exact.update({ + "state": "active", + "removed": False, + "transport_verified": True, + "private_dataplane_verified": True, + "config_verified": True, + "host_config_reusable": True, + "desired_revision": plan.desired_revision, + "observed_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "observed_digest": plan.desired_digest, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "member_ids": [member.member_id for member in members], + "retiring_members": [], + "pending_member_mutations": [], + "member_nics": [{ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "private_ip_cidr": f"{member.private_ip}/32", + "private_mac": ( + member.private_mac or derive_private_mac( + plan.vpc_id, member.member_id)), + "config_verified": True, + "xml_verified": True, + "seed_verified": True, + "domain_running": True, + "guest_network_verified": True, + "restart_required": False, + } for member in members], + "readback": "confirmed", + "rollback": "none", + }) + return exact + + +def retained_partial_coordinator(engine, store, plan): + nodes = tuple( + FabricNode( + host_id, f"100.64.0.{index}", + f"192.0.2.{index}:51820", 51820, + ) + for index, host_id in enumerate(("host-a", "host-b"), 1) + ) + fabric_plan = FabricPlan( + plan.fabric_id, plan.region, "100.64.0.0/24", plan.fabric_revision, + 51820, nodes, "f" * 64, + ) + + class PendingHostFabric(FakeFabric): + def __init__(self): + super().__init__(hosts=("host-a", "host-b")) + self.component_requests = [] + + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, probe_plan): + requested = tuple(node.host_id for node in probe_plan.nodes) + self.component_requests.append(requested) + if "host-a" in requested: + raise AssertionError("exact host A is offline and must not be probed") + assert requested == ("host-b",) + return { + "host-b": ( + SimpleNamespace(manager=object()), + {"host_id": "host-b"}, + ), + } + + fabric = PendingHostFabric() + lifecycle_store = FabricStore() + registry = SimpleNamespace(components={ + host_id: SimpleNamespace(store=lifecycle_store) + for host_id in ("host-a", "host-b") + }) + vpc = VpcCoordinator( + store, fabric, registry=registry, + catalog_lock=engine._network_catalog_mutation_lock, + vm_lock=engine._target_lock, coordinator=engine.coordinator, + ) + vpc._validate_dataplane_resources = lambda _plan: None + vpc._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, + } + vpc._fabric_peers_for_plan = lambda _plan, _record: [] + return vpc, fabric + + +def test_exact_uncertain_family_is_atomically_replaced_with_one_receipt(): + store = FabricStore() + coordinator = DurableCoordinator(store, owner_id="d" * 64) + plan = parse_vpc_plan(payload(operation_id="a" * 32)) + family = retained_family(coordinator, plan) + parent = coordinator.claim( + kind="request", metadata={"explicit_vpc_reconcile": True}) + evidence = "exact durable target and readback-only host pass" + + rebound = coordinator.resolve_family_and_bind( + family, parent, evidence, + operation_id=plan.operation_id, metadata=identity(plan)) + + rows = coordinator.snapshot() + assert set(rows) == {parent.token} + assert rebound.operation_id == plan.operation_id + assert rebound.metadata == identity(plan) + expected_resources = tuple(sorted({ + resource + for row in family + for resource in row["resources"] + })) + assert rebound.resources == expected_resources + assert rows[parent.token]["resources"] == list(expected_resources) + receipt = coordination_document(store)["resolutions"][-1] + assert receipt["kind"] == "resolve_family_and_bind_v1" + assert receipt["source_family_size"] == len(family) + assert receipt["source_execution_id"] == family[0]["execution_id"] + assert receipt["replacement_token"] == parent.token + outcome, current = coordinator.classify_resolve_family_and_bind( + family, parent, evidence, + operation_id=plan.operation_id, metadata=identity(plan)) + assert outcome == "committed" + assert current["token"] == parent.token + + +def test_transferred_family_fences_reenter_and_settle_with_replacement_root(): + store = FabricStore() + coordinator = DurableCoordinator(store, owner_id="d" * 64) + foreign = DurableCoordinator(store, owner_id="e" * 64) + plan = parse_vpc_plan(payload(operation_id="a" * 32)) + family = retained_family(coordinator, plan) + resources = tuple(sorted({ + resource + for row in family + for resource in row["resources"] + })) + parent = coordinator.claim( + kind="request", metadata={"explicit_vpc_reconcile": True}) + rebound = coordinator.resolve_family_and_bind( + family, parent, "transfer exact family fences", + operation_id=plan.operation_id, metadata=identity(plan)) + + for resource in ("catalog:network", next( + item for item in resources if item.startswith("vm:"))): + with pytest.raises(CoordinationClaimRejected) as raised: + foreign.claim( + operation_id="foreign-operation", resources=(resource,), + kind="resource") + assert raised.value.error_code == "coordination_busy" + child = coordinator.claim( + operation_id=plan.operation_id, resources=(resource,), + execution_id=rebound.execution_id, kind="resource", + metadata=identity(plan)) + assert coordinator.release(child) + + assert coordinator.retain(rebound) + for resource in resources: + with pytest.raises(CoordinationClaimRejected) as raised: + foreign.claim( + operation_id="foreign-operation", resources=(resource,), + kind="resource") + assert raised.value.error_code == "coordination_uncertain" + retained = coordinator.snapshot()[rebound.token] + retry_parent = coordinator.claim( + kind="request", metadata={"explicit_vpc_reconcile": True}) + retried = coordinator.resolve_and_bind( + retained, retry_parent, "transfer retained aggregate fences", + operation_id=plan.operation_id, metadata=identity(plan)) + assert retried.resources == resources + assert coordinator.release(retried) + assert coordinator.snapshot() == {} + + +def test_family_handoff_rejects_an_omitted_same_execution_child(): + store = FabricStore() + coordinator = DurableCoordinator(store, owner_id="d" * 64) + plan = parse_vpc_plan(payload(operation_id="a" * 32)) + family = retained_family(coordinator, plan) + parent = coordinator.claim( + kind="request", metadata={"explicit_vpc_reconcile": True}) + before = coordinator.snapshot() + + root = next(row for row in family if row["kind"] == "background") + one_child = next(row for row in family if row["kind"] == "resource") + with pytest.raises(CoordinationClaimRejected) as raised: + coordinator.resolve_family_and_bind( + (root, one_child), parent, "incomplete family must stay fenced", + operation_id=plan.operation_id, metadata=identity(plan)) + + assert raised.value.error_code == "coordination_uncertain" + assert coordinator.snapshot() == before + assert coordination_document(store)["resolutions"] == [] + + +def test_explicit_vpc_reconcile_recovers_only_readback_safe_family( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, target = await seed(engine, store) + + async def interrupted(*_args, **_kwargs): + raise CoordinationError("coordination_unavailable", 503) + + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", interrupted) + status, body = await request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 503, body + family = engine.coordinator.snapshot() + assert len(family) == 4 + assert len({row["execution_id"] for row in family.values()}) == 1 + assert all(row["state"] == "uncertain" + for row in family.values()) + + # The seed harness has no real host adapter and therefore no + # exact configuration readback. Explicit authorization alone must + # not erase the family or replay its possibly-started write. + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 503, body + assert body["error_code"] == "coordination_uncertain" + assert engine.coordinator.snapshot() == family + + observed = 0 + + async def readback_only(*_args, **_kwargs): + nonlocal observed + observed += 1 + + monkeypatch.setattr( + vpc, "_retained_family_readback_safe", lambda _plan: True) + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", readback_only) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 200, body + assert observed == 1 + assert engine.coordinator.snapshot() == {} + receipts = coordination_document(store)["resolutions"] + assert len(receipts) == 1 + assert receipts[0]["kind"] == "resolve_family_and_bind_v1" + assert receipts[0]["source_family_size"] == 4 + + asyncio.run(run()) + + +def test_explicit_reconcile_persists_probe_marker_before_family_handoff( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) + family = retained_family( + engine.coordinator, plan, + extra_resources=remaining_member_claims(plan), + ) + assert not vpc._retained_family_readback_safe(plan) + observed = 0 + + async def exact_probe(plan_value, record): + nonlocal observed + observed += 1 + assert plan_value == plan + assert tuple(engine.coordinator.snapshot().values()) + return exact_configuration_record(vpc, plan, record) + + async def forbidden_write(*_args, **_kwargs): + pytest.fail("retained-family recovery entered the write/QGA path") + + monkeypatch.setattr( + vpc, "_observe_retained_family_hosts", exact_probe) + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", forbidden_write) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + + assert status == 200, ( + body, fabric.component_requests, readbacks) + assert observed == 1 + assert engine.coordinator.snapshot() == {} + durable = json.loads(store.get("vpc_desired", plan.vpc_id)) + assert vpc._retained_record_readback_safe(plan, durable) + assert vpc._retained_family_recovery_marker_matches( + plan, durable, family) + receipts = coordination_document(store)["resolutions"] + assert len(receipts) == 1 + assert receipts[0]["source_family_size"] == len(family) + + asyncio.run(run()) + + +def test_retained_family_probe_requires_every_vm_child( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) + retained_family( + engine.coordinator, plan, + include_member=False, include_storage_member=False, + ) + before_claims = engine.coordinator.snapshot() + before_record = store.get("vpc_desired", plan.vpc_id) + + async def forbidden_probe(*_args, **_kwargs): + pytest.fail("unfenced VM reached retained-family probe") + + monkeypatch.setattr( + vpc, "_observe_retained_family_hosts", forbidden_probe) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + + assert status == 503, body + assert body["error_code"] == "coordination_uncertain" + assert engine.coordinator.snapshot() == before_claims + assert store.get("vpc_desired", plan.vpc_id) == before_record + assert coordination_document(store)["resolutions"] == [] + + asyncio.run(run()) + + +def test_retained_family_marker_save_lost_response_is_classified( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) + family = retained_family( + engine.coordinator, plan, + extra_resources=remaining_member_claims(plan), + ) + original_compare_and_set = store.compare_and_set + lost = False + + async def exact_probe(_plan, record): + return exact_configuration_record(vpc, plan, record) + + def compare_and_set_then_lose(kind, key, expected, value): + nonlocal lost + committed = original_compare_and_set( + kind, key, expected, value) + try: + candidate = json.loads(value) + except (TypeError, ValueError): + candidate = {} + if (committed and kind == "vpc_desired" + and "claim_family_recovery" in candidate + and not lost): + lost = True + raise RuntimeError("marker CAS response lost") + return committed + + async def forbidden_write(*_args, **_kwargs): + pytest.fail("classified marker save replayed host work") + + monkeypatch.setattr( + vpc, "_observe_retained_family_hosts", exact_probe) + monkeypatch.setattr( + store, "compare_and_set", compare_and_set_then_lose) + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", forbidden_write) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + + assert status == 200, ( + body, fabric.component_requests, readbacks) + assert lost + durable = json.loads(store.get("vpc_desired", plan.vpc_id)) + assert vpc._retained_family_recovery_marker_matches( + plan, durable, family) + assert engine.coordinator.snapshot() == {} + + asyncio.run(run()) + + +def test_retained_family_probe_cannot_overwrite_newer_desired_revision( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) + retained_family( + engine.coordinator, plan, + extra_resources=remaining_member_claims(plan), + ) + before_claims = engine.coordinator.snapshot() + next_target = json.loads(json.dumps(target)) + next_target.pop("desired_digest", None) + next_target.update({ + "desired_revision": plan.desired_revision + 1, + "operation_id": "next-vpc-operation-0002", + }) + next_plan = parse_vpc_plan(next_target) + next_gate = vpc._fabric_gate(next_plan) + next_record = vpc._desired_record( + next_plan, next_gate, state="applying") + vpc._finish(next_record, next_gate) + next_raw = json.dumps( + next_record, ensure_ascii=True, separators=(",", ":")) + probes = 0 + host_writes = 0 + + async def replace_during_probe(_plan, record): + nonlocal probes + probes += 1 + store.set("vpc_desired", plan.vpc_id, next_raw) + return exact_configuration_record(vpc, plan, record) + + async def forbidden_write(*_args, **_kwargs): + nonlocal host_writes + host_writes += 1 + pytest.fail("stale retained recovery dispatched host mutation") + + monkeypatch.setattr( + vpc, "_observe_retained_family_hosts", + replace_during_probe) + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", forbidden_write) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + + assert status == 503, body + assert body["error_code"] == "coordination_uncertain" + assert probes == 1 + assert host_writes == 0 + assert store.get("vpc_desired", plan.vpc_id) == next_raw + assert engine.coordinator.snapshot() == before_claims + assert coordination_document(store)["resolutions"] == [] + + asyncio.run(run()) + + +def test_retained_family_hash_drift_blocks_cas_and_marker_resumes( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) + family = retained_family( + engine.coordinator, plan, + extra_resources=remaining_member_claims(plan), + ) + real_snapshot = engine.coordinator.snapshot + tampered = False + probes = 0 + + async def exact_probe(_plan, record): + nonlocal probes + probes += 1 + return exact_configuration_record(vpc, plan, record) + + def one_drifted_snapshot(): + nonlocal tampered + rows = real_snapshot() + durable = store.get("vpc_desired", plan.vpc_id) + if (not tampered and durable is not None + and "claim_family_recovery" in json.loads(durable)): + tampered = True + rows = json.loads(json.dumps(rows)) + old_token = next( + token for token, row in rows.items() + if row.get("execution_id") + == family[0]["execution_id"]) + rows[old_token]["metadata"] = {"drifted": True} + return rows + + async def forbidden_write(*_args, **_kwargs): + pytest.fail("marker recovery entered host write path") + + monkeypatch.setattr( + vpc, "_observe_retained_family_hosts", exact_probe) + monkeypatch.setattr( + engine.coordinator, "snapshot", one_drifted_snapshot) + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", forbidden_write) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 503, body + assert body["error_code"] == "coordination_uncertain" + assert tampered and probes == 1 + assert real_snapshot() + assert coordination_document(store)["resolutions"] == [] + + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 200, body + assert probes == 1 + assert real_snapshot() == {} + receipts = coordination_document(store)["resolutions"] + assert len(receipts) == 1 + assert receipts[0]["source_family_size"] == len(family) + + asyncio.run(run()) + + +def test_retained_host_probe_uses_only_unclaimed_readers( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) + record = vpc._load("vpc_desired", plan.vpc_id) + host = record["hosts"][0] + configuration_reads = [] + observations = [] + + class ReadOnlyService: + def __init__(self, *_args, **_kwargs): + pass + + def read_private_configuration_unclaimed( + self, instance_uuid, **kwargs): + configuration_reads.append((instance_uuid, kwargs)) + return { + "inactive": f"", + "live": None, + "seed_verified": True, + "guest_network": { + "state": "stopped", "verified": False, + "reason_code": "vpc_private_guest_stopped", + "observed_at": 1, + }, + } + + def ensure_private_interface(self, *_args, **_kwargs): + pytest.fail("retained probe called ensure_private_interface") + + def read_private_domain_pair(self, *_args, **_kwargs): + pytest.fail("retained probe entered QGA read path") + + def remove_private_interface(self, *_args, **_kwargs): + pytest.fail("retained probe called remove_private_interface") + + class ReadOnlyAdapter: + def __init__(self, *_args, **_kwargs): + pass + + def apply(self, *_args, **_kwargs): + pytest.fail("retained probe called host apply") + + def remove(self, *_args, **_kwargs): + pytest.fail("retained probe called host remove") + + def observe(self, dataplane, *, xml_by_instance): + observations.append((dataplane, xml_by_instance)) + return { + "state": "active", "config_verified": True, + "host_config_reusable": True, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "member_nics": [], "reason_code": None, + } + + monkeypatch.setattr( + "virsh_panel.libvirt_service.VMService", ReadOnlyService) + monkeypatch.setattr( + "virsh_panel.vpc_dataplane.VpcPrivateHostAdapter", + ReadOnlyAdapter) + public_key = "a" * 44 + result = vpc._run_retained_host_readback( + plan, host, SimpleNamespace(manager=object()), { + "libvirt_uri": "fake", "storage_pool": "images", + "iso_pool": "iso", "rescue_iso_pool": "rescue", + "default_network": "default", + }, [], {"host-a": public_key}, "f" * 64, + {public_key: "100.64.0.1/32"}, time.monotonic() + 5, + ) + + assert [item[0] for item in configuration_reads] == [ + member.instance_uuid for member in plan.members] + assert len(observations) == 1 + assert set(observations[0][1]) == { + member.instance_uuid for member in plan.members} + assert result["transport_verified"] is True + assert result["desired_digest"] == plan.desired_digest + + asyncio.run(run()) + + +def test_explicit_vpc_reconcile_keeps_unrelated_same_execution_child( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) + family = retained_family( + engine.coordinator, plan, + extra_resources=(("allocation:foreign",),), + ) + base_family = tuple( + row for row in family + if row.get("resources") != ["allocation:foreign"] + ) + assert vpc._retained_family_resources_match(plan, base_family) + assert not vpc._retained_family_resources_match(plan, family) + before = engine.coordinator.snapshot() + host_calls = 0 + + monkeypatch.setattr( + vpc, "_retained_family_readback_safe", lambda _plan: True) + + async def forbidden(*_args, **_kwargs): + nonlocal host_calls + host_calls += 1 + + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", forbidden) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + + assert status == 503, body + assert body["error_code"] == "coordination_uncertain" + assert host_calls == 0 + assert engine.coordinator.snapshot() == before + assert coordination_document(store)["resolutions"] == [] + + asyncio.run(run()) + + +@pytest.mark.parametrize("child_kind", ["vm", "storage_vm"]) +def test_explicit_vpc_reconcile_recovers_member_child_without_catalog( + tmp_path, monkeypatch, child_kind): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) + member = plan.members[0] + vm_resource = f"vm:{member.host_id}:{member.instance_uuid}" + child_resources = ((vm_resource,) if child_kind == "vm" else ( + f"storage:{member.host_id}", vm_resource, + )) + retained_family( + engine.coordinator, plan, + include_catalog=False, include_member=False, + include_storage_member=False, + extra_resources=(child_resources,), + ) + before = engine.coordinator.snapshot() + assert len(before) == 2 + host_calls = 0 + + monkeypatch.setattr( + vpc, "_retained_family_readback_safe", lambda _plan: True) + + async def readback_only(*_args, **_kwargs): + nonlocal host_calls + host_calls += 1 + + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", readback_only) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + + assert status == 200, body + assert host_calls == 1 + assert engine.coordinator.snapshot() == {} + receipts = coordination_document(store)["resolutions"] + assert len(receipts) == 1 + assert receipts[0]["kind"] == "resolve_family_and_bind_v1" + assert receipts[0]["source_family_size"] == 2 + + asyncio.run(run()) + + +@pytest.mark.parametrize(("with_vm_child", "keep_stale_record"), [ + (False, False), + (False, True), + (True, False), + (True, True), +]) +def test_compact_delete_receipt_recovers_only_root_and_catalog_family( + tmp_path, monkeypatch, with_vm_child, keep_stale_record): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, applied = await seed(engine, store) + target = delete_payload( + operation_id="b" * 32, + retiring_members=retiring_members(applied), + ) + plan = parse_vpc_delete_plan(target)[0] + release = engine.coordinator.release + lost_catalog_release = False + + async def confirmed(plan_value, record, *_args, **_kwargs): + confirm_delete_hosts(plan_value, record) + if not with_vm_child: + return + context = current_execution() + assert context is not None + member = parse_vpc_plan(applied).members[0] + extra = engine.coordinator.claim( + operation_id=plan.operation_id, + resources=( + f"vm:{member.host_id}:{member.instance_uuid}", + ), + execution_id=context.claim.execution_id, + kind="resource", metadata=identity(plan), + ) + assert engine.coordinator.retain(extra) + + def lose_catalog_release(claim): + nonlocal lost_catalog_release + resources = tuple( + claim.get("resources", ()) + if isinstance(claim, dict) else claim.resources + ) + if resources == ("catalog:network",) and not lost_catalog_release: + lost_catalog_release = True + raise RuntimeError("lost catalog release response") + return release(claim) + + monkeypatch.setattr(vpc, "_reconcile_dataplane", confirmed) + if keep_stale_record: + # Model a crash after the terminal receipt commit but before + # the independently stored desired row was purged. + monkeypatch.setattr(vpc, "_purge_deleted_vpc", lambda _key: None) + monkeypatch.setattr( + engine.coordinator, "release", lose_catalog_release) + with pytest.raises(CoordinationError) as interrupted: + await vpc.delete("vpc-hk-1", target) + assert interrupted.value.error_code == "coordination_uncertain" + assert lost_catalog_release + before = engine.coordinator.snapshot() + assert len(before) == (3 if with_vm_child else 2) + assert all(row["state"] == "uncertain" + for row in before.values()) + assert ( + store.get("vpc_desired", "vpc-hk-1") is not None + ) == keep_stale_record + + monkeypatch.setattr(engine.coordinator, "release", release) + monkeypatch.setattr( + vpc, "_reconcile_dataplane", + lambda *_args, **_kwargs: pytest.fail( + "confirmed delete receipt retried host dispatch"), + ) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/delete", target) + + if with_vm_child: + assert status == 503, body + assert body["error_code"] == "coordination_uncertain" + assert engine.coordinator.snapshot() == before + assert coordination_document(store)["resolutions"] == [] + else: + assert status == 200, body + assert body["deleted"] is True + assert engine.coordinator.snapshot() == {} + receipts = coordination_document(store)["resolutions"] + assert len(receipts) == 1 + assert receipts[0]["kind"] == "resolve_family_and_bind_v1" + assert receipts[0]["source_family_size"] == 2 + + asyncio.run(run()) + + +@pytest.mark.parametrize("fault", ["before", "after", "false_after"]) +def test_family_handoff_cas_fault_is_classified_without_host_replay( + tmp_path, monkeypatch, fault): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, _target = await seed(engine, store) + + async def interrupted(*_args, **_kwargs): + raise CoordinationError("coordination_unavailable", 503) + + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", interrupted) + status, _body = await request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 503 + old_family = engine.coordinator.snapshot() + assert len(old_family) == 4 + + monkeypatch.setattr( + vpc, "_retained_family_readback_safe", lambda _plan: True) + compare = engine.coordinator.store.compare_and_set + triggered = False + host_calls = 0 + + def fail_family_handoff(kind, key, expected, value): + nonlocal triggered + desired = json.loads(value) + handoff = any( + item.get("kind") == "resolve_family_and_bind_v1" + for item in desired.get("resolutions", [])) + if handoff and not triggered: + triggered = True + if fault == "before": + raise RuntimeError("family handoff did not commit") + assert compare(kind, key, expected, value) + if fault == "after": + raise RuntimeError("family handoff response lost") + return False + return compare(kind, key, expected, value) + + async def readback_only(*_args, **_kwargs): + nonlocal host_calls + host_calls += 1 + + monkeypatch.setattr( + engine.coordinator.store, "compare_and_set", + fail_family_handoff) + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", readback_only) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 503, body + assert triggered and host_calls == 0 + rows = engine.coordinator.snapshot() + assert not set(old_family).intersection(rows) + assert len(rows) == 1 + retained = next(iter(rows.values())) + assert retained["state"] == "uncertain" + inherited = sorted({ + resource + for row in old_family.values() + for resource in row["resources"] + }) + assert retained["resources"] == inherited + foreign = DurableCoordinator(store, owner_id="e" * 64) + for resource in inherited: + with pytest.raises(CoordinationClaimRejected) as raised: + foreign.claim( + operation_id="foreign-operation", + resources=(resource,), kind="resource") + assert raised.value.error_code == "coordination_uncertain" + + monkeypatch.setattr( + engine.coordinator.store, "compare_and_set", compare) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 200, body + assert host_calls == 1 + assert engine.coordinator.snapshot() == {} + + asyncio.run(run()) + + +def test_double_precommit_family_handoff_fault_releases_anonymous_root( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, _target = await seed(engine, store) + + async def interrupted(*_args, **_kwargs): + raise CoordinationError("coordination_unavailable", 503) + + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", interrupted) + status, _body = await request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 503 + old_family = engine.coordinator.snapshot() + assert len(old_family) == 4 + + monkeypatch.setattr( + vpc, "_retained_family_readback_safe", lambda _plan: True) + compare = engine.coordinator.store.compare_and_set + handoff_faults = 0 + release_response_lost = False + + def fail_two_handoffs_then_lose_release(kind, key, expected, value): + nonlocal handoff_faults, release_response_lost + if kind != "coordination": + return compare(kind, key, expected, value) + before = json.loads(expected) + after = json.loads(value) + handoff = any( + item.get("kind") == "resolve_family_and_bind_v1" + for item in after.get("resolutions", [])) + if handoff and handoff_faults < 2: + handoff_faults += 1 + raise RuntimeError("family handoff did not commit") + removed = set(before.get("claims", {})) - set( + after.get("claims", {})) + anonymous_release = any( + before["claims"][token].get("operation_id") is None + and before["claims"][token].get("metadata") + == {"explicit_vpc_reconcile": True} + for token in removed + ) + if anonymous_release and not release_response_lost: + assert compare(kind, key, expected, value) + release_response_lost = True + raise RuntimeError("anonymous release response lost") + return compare(kind, key, expected, value) + + host_calls = 0 + + async def readback_only(*_args, **_kwargs): + nonlocal host_calls + host_calls += 1 + + monkeypatch.setattr( + engine.coordinator.store, "compare_and_set", + fail_two_handoffs_then_lose_release) + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", readback_only) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 503, body + assert handoff_faults == 2 + assert release_response_lost + assert host_calls == 0 + rows = engine.coordinator.snapshot() + assert rows == old_family + assert not any( + row.get("operation_id") is None + and row.get("metadata") + == {"explicit_vpc_reconcile": True} + for row in rows.values() + ) + + monkeypatch.setattr( + engine.coordinator.store, "compare_and_set", compare) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 200, body + assert host_calls == 1 + assert engine.coordinator.snapshot() == {} + + asyncio.run(run()) + + +def test_explicit_retained_apply_probes_only_pending_host_after_exact_host_offline( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + target = payload( + operation_id="a" * 32, + hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", + "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-b", "members": [{ + "member_id": "member-b", + "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}, + ], + ) + plan = parse_vpc_plan(target) + vpc, fabric = retained_partial_coordinator( + engine, store, plan) + gate = vpc._fabric_gate(plan) + record = vpc._desired_record( + plan, gate, state="applying") + host_a = next( + host for host in record["hosts"] + if host["host_id"] == "host-a") + host_a.update(exact_active_host(plan, host_a)) + vpc._finish(record, gate) + vpc._save("vpc_desired", plan.vpc_id, record) + vpc._save_children(record) + family = retained_multi_host_family( + engine.coordinator, plan, + (("host-b", INSTANCE_B),), + ) + family_resources = { + resource for row in family for resource in row["resources"] + } + assert len(family) == 3 + assert f"vm:host-a:{INSTANCE_A}" not in family_resources + assert f"vm:host-b:{INSTANCE_B}" in family_resources + assert not vpc._retained_family_readback_safe(plan) + readbacks = [] + + def exact_pending_host(plan_value, host, *_args, **_kwargs): + assert plan_value == plan + readbacks.append(host["host_id"]) + assert host["host_id"] == "host-b" + return exact_active_host(plan, host) + + monkeypatch.setattr( + vpc, "_run_retained_host_readback", exact_pending_host) + engine.vpc = vpc + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + + assert status == 200, body + assert body["state"] == "active" + assert body["next_stage_allowed"] is True + assert fabric.component_requests == [("host-b",)] + assert readbacks == ["host-b"] + assert engine.coordinator.snapshot() == {} + durable = json.loads(store.get("vpc_desired", plan.vpc_id)) + assert vpc._retained_family_recovery_marker_matches( + plan, durable, family) + + asyncio.run(run()) + + +def test_explicit_retained_delete_probes_only_pending_host_after_exact_host_offline( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + applied = payload(hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-b", "members": [{ + "member_id": "member-b", "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}, + ]) + signed_members = retiring_members(applied) + target = delete_payload( + desired_revision=2, operation_id="b" * 32, + hosts=("host-a", "host-b"), + retiring_members=signed_members, + ) + plan = parse_vpc_delete_plan(target)[0] + vpc, fabric = retained_partial_coordinator( + engine, store, plan) + gate = vpc._fabric_gate(plan) + record = vpc._desired_record( + plan, gate, state="deleting") + retiring_by_host = { + host_id: [dict(member) for member in signed_members + if member["host_id"] == host_id] + for host_id in ("host-a", "host-b") + } + for host in record["hosts"]: + host["retiring_members"] = retiring_by_host[host["host_id"]] + host["pending_member_mutations"] = [ + member["instance_uuid"] + for member in retiring_by_host[host["host_id"]] + ] + record["delete_target"] = { + "operation_id": plan.operation_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "all_host_ids": ["host-a", "host-b"], + "required_host_ids": ["host-a", "host-b"], + "preverified_host_ids": [], + "retiring_members": [dict(member) for member in signed_members], + } + host_a = next( + host for host in record["hosts"] + if host["host_id"] == "host-a") + confirm_delete_hosts(plan, {"hosts": [host_a]}) + host_a["pending_member_mutations"] = [] + vpc._finish(record, gate) + vpc._save("vpc_desired", plan.vpc_id, record) + vpc._save_children(record) + family = retained_multi_host_family( + engine.coordinator, plan, + (("host-b", INSTANCE_B),), + ) + family_resources = { + resource for row in family for resource in row["resources"] + } + assert len(family) == 3 + assert f"vm:host-a:{INSTANCE_A}" not in family_resources + assert f"vm:host-b:{INSTANCE_B}" in family_resources + assert not vpc._retained_family_readback_safe(plan) + readbacks = [] + + def exact_pending_host(plan_value, host, *_args, **_kwargs): + assert plan_value == plan + readbacks.append(host["host_id"]) + assert host["host_id"] == "host-b" + observed = dict(host) + confirm_delete_hosts(plan, {"hosts": [observed]}) + return observed + + monkeypatch.setattr( + vpc, "_run_retained_host_readback", exact_pending_host) + engine.vpc = vpc + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/delete", target) + + assert status == 200, ( + body, fabric.component_requests, readbacks) + assert body["deleted"] is True + assert body["readback"] == "confirmed" + assert fabric.component_requests == [("host-b",)] + assert readbacks == ["host-b"] + assert engine.coordinator.snapshot() == {} + assert store.get("vpc_desired", plan.vpc_id) is None + assert store.get(VPC_DELETE_RECEIPT_KIND, plan.vpc_id) is not None + + asyncio.run(run()) + + +def test_cleanup_marker_requires_vm_fence_only_while_ledger_exists(): + store = FabricStore() + vpc = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) + old_plan = parse_vpc_plan(payload()) + plan = parse_vpc_plan(payload( + desired_revision=2, operation_id="c" * 32, + hosts=[], routes=[], acls=[])) + record = vpc._desired_record( + plan, vpc._fabric_gate(plan), state="applying") + _state, persist, _clear = vpc._guest_exec_context( + old_plan, old_plan.members[0]) + persist({ + "version": 1, "phase": "pending", + "domain_uuid": INSTANCE_A, "domain_runtime_id": 7, + "qemu_runtime_token": "a" * 32, + "guest_network_generation": "old-generation", + "probe_kind": "link", "pid": 41, + }) + ledger_key = f"{plan.vpc_id}.{INSTANCE_A}" + ledger_raw = store.get(VPC_GUEST_EXEC_KIND, ledger_key) + assert isinstance(ledger_raw, str) + encoded = ledger_raw.encode("utf-8") + record["guest_exec_cleanup_recovery"] = { + "version": 1, + "operation_id": plan.operation_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "source_execution_id": "e" * 64, + "source_family_size": 2, + "source_family_sha256": "f" * 64, + "targets": [{ + "host_id": "host-a", "instance_uuid": INSTANCE_A, + "member_id": "member-a", "ledger_key": ledger_key, + "ledger_sha256": hashlib.sha256(encoded).hexdigest(), + "ledger_bytes": len(encoded), + }], + "created_at": 1, + } + family_without_vm = ( + {"kind": "background", "resources": []}, + {"kind": "resource", "resources": ["catalog:network"]}, + ) + family_with_vm = (*family_without_vm, { + "kind": "resource", + "resources": [f"vm:host-a:{INSTANCE_A}"], + }) + + assert not vpc._retained_family_covers_vm_closure( + record, plan, family_without_vm) + assert vpc._retained_family_covers_vm_closure( + record, plan, family_with_vm) + + store.set(VPC_GUEST_EXEC_KIND, ledger_key, None) + assert vpc._retained_family_covers_vm_closure( + record, plan, family_without_vm) + + +def test_retained_detach_clears_exact_guest_ledger_before_new_member_reuses_vm( + tmp_path, monkeypatch): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + old_plan = parse_vpc_plan(payload()) + target = payload( + desired_revision=2, operation_id="b" * 32, + hosts=[], routes=[], acls=[]) + plan = parse_vpc_plan(target) + fabric_plan = FabricPlan( + plan.fabric_id, plan.region, "100.64.0.0/24", + plan.fabric_revision, 51820, + (FabricNode( + "host-a", "100.64.0.1", "192.0.2.1:51820", 51820),), + "f" * 64, + ) + + class CleanupFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, probe_plan): + assert tuple(node.host_id for node in probe_plan.nodes) \ + == ("host-a",) + return {"host-a": ( + SimpleNamespace(manager=object()), + { + "libvirt_uri": "fake:///system", + "storage_pool": "images", "iso_pool": "iso", + "rescue_iso_pool": "rescue", + "default_network": "default", + }, + )} + + lifecycle_store = FabricStore() + vpc = VpcCoordinator( + store, CleanupFabric(hosts=("host-a",)), + registry=SimpleNamespace(components={ + "host-a": SimpleNamespace(store=lifecycle_store), + }), + catalog_lock=engine._network_catalog_mutation_lock, + vm_lock=engine._target_lock, coordinator=engine.coordinator, + ) + vpc._fabric_record = lambda _plan: {} + gate = vpc._fabric_gate(plan) + record = vpc._desired_record(plan, gate, state="applying") + retired_host = { + "host_id": "host-a", "retiring": True, + "pending_member_mutations": [], + } + confirm_delete_hosts(plan, {"hosts": [retired_host]}) + retired_host["retiring"] = True + record["hosts"] = [retired_host] + vpc._finish(record, gate) + + family = retained_multi_host_family( + engine.coordinator, plan, (("host-a", INSTANCE_A),)) + record["claim_family_recovery"] = ( + vpc._retained_family_recovery_marker(plan, family)) + vpc._save("vpc_desired", plan.vpc_id, record) + vpc._save_children(record) + + _state, persist, _clear = vpc._guest_exec_context( + old_plan, old_plan.members[0]) + persist({ + "version": 1, "phase": "pending", + "domain_uuid": INSTANCE_A, "domain_runtime_id": 7, + "qemu_runtime_token": "a" * 32, + "guest_network_generation": "old-generation", + "probe_kind": "link", "pid": 41, + }) + ledger_key = f"{plan.vpc_id}.{INSTANCE_A}" + marker_writes = [] + compare_and_set = store.compare_and_set + + def capture_marker(kind, key, expected, value): + if kind == "vpc_desired" and key == plan.vpc_id: + marker_writes.append(json.loads(value)) + return compare_and_set(kind, key, expected, value) + + class ReadOnlyService: + def __init__(self, *_args, **_kwargs): + pass + + @staticmethod + def read_private_absence_unclaimed( + instance_uuid, *, deadline=None): + assert instance_uuid == INSTANCE_A + assert deadline is not None + return {"removed": True, "running": False, + "readback": "confirmed"} + + monkeypatch.setattr(store, "compare_and_set", capture_marker) + monkeypatch.setattr( + "virsh_panel.libvirt_service.VMService", ReadOnlyService) + monkeypatch.setattr( + vpc, "_reconcile_dataplane_unlocked", + lambda *_args, **_kwargs: pytest.fail( + "exact detach recovery replayed host work")) + engine.vpc = vpc + + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + + assert status == 200, body + assert store.get(VPC_GUEST_EXEC_KIND, ledger_key) is None + assert any("guest_exec_cleanup_recovery" in item + for item in marker_writes) + durable = json.loads(store.get("vpc_desired", plan.vpc_id)) + assert "guest_exec_cleanup_recovery" not in durable + assert engine.coordinator.snapshot() == {} + + new_target = payload( + desired_revision=3, operation_id="c" * 32, + hosts=[{"host_id": "host-a", "members": [{ + "member_id": "member-reused", + "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.20/32", + "subnet_id": "subnet-a", + }]}], + ) + new_plan = parse_vpc_plan(new_target) + accepted = [] + + async def accept_next_revision(plan_value): + accepted.append(plan_value) + return {"state": "active"} + + monkeypatch.setattr(vpc, "_apply_locked", accept_next_revision) + result = await vpc.apply(new_target) + assert result == {"state": "active"} + assert accepted == [new_plan] + state, _persist, _clear = vpc._guest_exec_context( + new_plan, new_plan.members[0]) + assert state == {} + + asyncio.run(run()) + + +def _retired_guest_cleanup_scenario(engine, store): + old_plan = parse_vpc_plan(payload()) + plan = parse_vpc_plan(payload( + desired_revision=2, operation_id="d" * 32, + hosts=[], routes=[], acls=[])) + fabric_plan = FabricPlan( + plan.fabric_id, plan.region, "100.64.0.0/24", + plan.fabric_revision, 51820, + (FabricNode( + "host-a", "100.64.0.1", "192.0.2.1:51820", 51820),), + "f" * 64, + ) + + class CleanupFabric(FakeFabric): + fail_components = False + + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, probe_plan): + if self.fail_components: + raise RuntimeError("component lookup interrupted") + assert tuple(node.host_id for node in probe_plan.nodes) \ + == ("host-a",) + return {"host-a": ( + SimpleNamespace(manager=object()), + { + "libvirt_uri": "fake:///system", + "storage_pool": "images", "iso_pool": "iso", + "rescue_iso_pool": "rescue", + "default_network": "default", + }, + )} + + fabric = CleanupFabric(hosts=("host-a",)) + vpc = VpcCoordinator( + store, fabric, + registry=SimpleNamespace(components={ + "host-a": SimpleNamespace(store=FabricStore()), + }), + catalog_lock=engine._network_catalog_mutation_lock, + vm_lock=engine._target_lock, coordinator=engine.coordinator, + ) + vpc._fabric_record = lambda _plan: {} + gate = vpc._fabric_gate(plan) + record = vpc._desired_record(plan, gate, state="applying") + retired_host = { + "host_id": "host-a", "retiring": True, + "pending_member_mutations": [], + } + confirm_delete_hosts(plan, {"hosts": [retired_host]}) + retired_host["retiring"] = True + record["hosts"] = [retired_host] + vpc._finish(record, gate) + family = retained_multi_host_family( + engine.coordinator, plan, (("host-a", INSTANCE_A),)) + record["claim_family_recovery"] = ( + vpc._retained_family_recovery_marker(plan, family)) + vpc._save("vpc_desired", plan.vpc_id, record) + vpc._save_children(record) + _state, persist, _clear = vpc._guest_exec_context( + old_plan, old_plan.members[0]) + persist({ + "version": 1, "phase": "pending", + "domain_uuid": INSTANCE_A, "domain_runtime_id": 7, + "qemu_runtime_token": "a" * 32, + "guest_network_generation": "old-generation", + "probe_kind": "link", "pid": 41, + }) + engine.vpc = vpc + return vpc, fabric, plan, f"{plan.vpc_id}.{INSTANCE_A}" + + +@pytest.mark.parametrize("failure", ["powered_on", "ledger_changed", "childless"]) +def test_retained_guest_cleanup_failure_keeps_recoverable_root( + tmp_path, monkeypatch, failure): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, fabric, plan, ledger_key = _retired_guest_cleanup_scenario( + engine, store) + if failure == "childless": + fabric.fail_components = True + + class ReadOnlyService: + def __init__(self, *_args, **_kwargs): + pass + + @staticmethod + def read_private_absence_unclaimed( + instance_uuid, *, deadline=None): + assert instance_uuid == INSTANCE_A + if failure == "ledger_changed": + changed = json.loads(store.get( + VPC_GUEST_EXEC_KIND, ledger_key)) + changed["qemu_runtime_token"] = "e" * 32 + store.set(VPC_GUEST_EXEC_KIND, ledger_key, json.dumps( + changed, ensure_ascii=True, separators=(",", ":"))) + return { + "removed": True, + "running": failure == "powered_on", + "readback": "confirmed", + } + + monkeypatch.setattr( + "virsh_panel.libvirt_service.VMService", ReadOnlyService) + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 503, body + assert body["error_code"] == "coordination_uncertain" + durable = json.loads(store.get("vpc_desired", plan.vpc_id)) + assert "guest_exec_cleanup_recovery" in durable + rows = engine.coordinator.snapshot() + assert rows and all(row["state"] == "uncertain" + for row in rows.values()) + assert store.get(VPC_GUEST_EXEC_KIND, ledger_key) is not None + + if failure == "childless": + # The first handoff committed before component resolution and + # before a new cleanup child could complete. A later exact + # reconcile must resume the durable marker from the adopted + # replacement family, including transferred resource fences. + fabric.fail_components = False + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 200, body + assert store.get(VPC_GUEST_EXEC_KIND, ledger_key) is None + durable = json.loads(store.get("vpc_desired", plan.vpc_id)) + assert "guest_exec_cleanup_recovery" not in durable + assert engine.coordinator.snapshot() == {} + + asyncio.run(run()) + + +@pytest.mark.parametrize("lost_reply", ["ledger_delete", "marker_clear"]) +def test_retained_guest_cleanup_classifies_committed_lost_store_reply( + tmp_path, monkeypatch, lost_reply): + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + vpc, _fabric, plan, ledger_key = _retired_guest_cleanup_scenario( + engine, store) + + class ReadOnlyService: + def __init__(self, *_args, **_kwargs): + pass + + @staticmethod + def read_private_absence_unclaimed( + instance_uuid, *, deadline=None): + return {"removed": True, "running": False, + "readback": "confirmed"} + + monkeypatch.setattr( + "virsh_panel.libvirt_service.VMService", ReadOnlyService) + if lost_reply == "ledger_delete": + delete = store.delete + lost = False + + def commit_then_raise(kind, key): + nonlocal lost + delete(kind, key) + if (not lost and kind == VPC_GUEST_EXEC_KIND + and key == ledger_key): + lost = True + raise RuntimeError("ledger delete reply lost") + + monkeypatch.setattr(store, "delete", commit_then_raise) + else: + compare_and_set = store.compare_and_set + lost = False + + def clear_then_raise(kind, key, expected, value): + nonlocal lost + committed = compare_and_set(kind, key, expected, value) + if (committed and not lost and kind == "vpc_desired" + and isinstance(expected, str) + and "guest_exec_cleanup_recovery" in expected + and "guest_exec_cleanup_recovery" not in value): + lost = True + raise RuntimeError("marker clear reply lost") + return committed + + monkeypatch.setattr(store, "compare_and_set", clear_then_raise) + + status, body = await explicit_request( + engine, "POST", + "/internal/v1/vpcs/vpc-hk-1/reconcile", {}) + assert status == 200, body + assert lost + assert store.get(VPC_GUEST_EXEC_KIND, ledger_key) is None + durable = json.loads(store.get("vpc_desired", plan.vpc_id)) + assert "guest_exec_cleanup_recovery" not in durable + assert engine.coordinator.snapshot() == {} + + asyncio.run(run()) diff --git a/engine/tests/test_vpc_confirmed_write_retry.py b/engine/tests/test_vpc_confirmed_write_retry.py new file mode 100644 index 00000000..9b549f0b --- /dev/null +++ b/engine/tests/test_vpc_confirmed_write_retry.py @@ -0,0 +1,351 @@ +"""Confirmed VPC writes must not poison later deterministic retry paths.""" +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace + +import pytest + +from virsh_panel import vpc_dataplane as dataplane +from virsh_panel.coordination import mark_remote_write_started +from virsh_panel.fabric import FabricNode, FabricPlan +from virsh_panel.vpc import VpcCoordinator, parse_vpc_plan +from virsh_panel.vpc_dataplane import ( + DATAPLANE_OWNER_MARKER, + VpcPrivateHostAdapter, + derive_private_mac, +) +from test_engine_coordination import request, running_engine +from test_vpc import FakeFabric, INSTANCE_A, INSTANCE_B, payload + + +FABRIC_DIGEST = "f" * 64 +NFT_RECEIPT = ",".join(character * 64 for character in "abc") +WG_KEYS = ( + "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=", + "AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI=", +) + + +@pytest.fixture(autouse=True) +def empty_apply_receipts(): + with dataplane._APPLY_RECEIPT_LOCK: + dataplane._APPLY_RECEIPTS.clear() + yield + with dataplane._APPLY_RECEIPT_LOCK: + dataplane._APPLY_RECEIPTS.clear() + + +def _target(*, two_hosts: bool = False): + hosts = [{ + "host_id": "host-a", + "members": [{ + "member_id": "member-a", + "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }], + }] + if two_hosts: + hosts.append({ + "host_id": "host-b", + "members": [{ + "member_id": "member-b", + "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }], + }) + return payload(hosts=hosts) + + +def _profile(host_id: str) -> dict[str, object]: + return { + "host_id": host_id, + "host": f"{host_id}.example.test", + "port": 22, + "fingerprint": f"host-key-{host_id}", + "ssh_target": host_id, + "ssh_config_file": f"/test/{host_id}/ssh_config", + "storage_pool": "images", + "iso_pool": "iso", + "rescue_iso_pool": "rescue", + "default_network": "default", + "libvirt_uri": "fake", + } + + +def _fabric_plan(host_ids: tuple[str, ...]) -> FabricPlan: + return FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, + tuple( + FabricNode( + host_id, + f"100.64.0.{index + 1}", + f"192.0.2.{index + 1}:51820", + 51820, + ) + for index, host_id in enumerate(host_ids) + ), + FABRIC_DIGEST, + ) + + +def _confirmed_host(plan, host_id: str) -> dict[str, object]: + member = next(member for member in plan.members + if member.host_id == host_id) + return { + "state": "active", + "transport_verified": True, + "private_dataplane_verified": True, + "config_verified": True, + "host_config_reusable": True, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "member_nics": [{ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "private_ip_cidr": f"{member.private_ip}/32", + "private_mac": derive_private_mac( + plan.vpc_id, member.member_id), + "config_verified": True, + "xml_verified": True, + "seed_verified": True, + "domain_running": False, + "guest_network_verified": False, + "guest_network_readback": "stopped", + "guest_reason_code": "vpc_private_guest_stopped", + "restart_required": False, + }], + "routes": [], + "wg_allowed_ips": [], + "reason_code": None, + } + + +def _durable_vpc(store) -> dict[str, object]: + return json.loads(store.get("vpc_desired", "vpc-hk-1")) + + +def _install_coordinator(engine, store, target, monkeypatch): + plan = parse_vpc_plan(target) + host_ids = tuple(host.host_id for host in plan.hosts) + fabric_plan = _fabric_plan(host_ids) + + class TestFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, selected): + return { + node.host_id: ( + SimpleNamespace(manager=object()), + _profile(node.host_id), + ) + for node in selected.nodes + } + + coordinator = VpcCoordinator( + store, + TestFabric(hosts=host_ids), + registry=object(), + catalog_lock=engine._network_catalog_mutation_lock, + vm_lock=engine._target_lock, + coordinator=engine.coordinator, + ) + coordinator._validate_dataplane_resources = lambda _plan: None + coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False + public_keys = { + host_id: WG_KEYS[index] + for index, host_id in enumerate(host_ids) + } + coordinator._fabric_record = lambda _plan: { + "public_keys": public_keys, + } + coordinator._fabric_peers_for_plan = lambda _plan, _record: [ + { + "host_id": host_id, + "public_key": public_keys[host_id], + "overlay_ip": f"100.64.0.{index + 1}", + "tenant_allowed_ips": sorted( + f"{member.private_ip}/32" + for member in plan.members + if member.host_id == host_id + ), + } + for index, host_id in enumerate(host_ids) + ] + coordinator._require_host_member_mutations_stopped = ( + lambda *_args, **_kwargs: None) + engine.vpc = coordinator + + writes: list[str] = [] + monkeypatch.setattr( + VpcPrivateHostAdapter, + "_deployment_contract", + lambda self: ( + "networkmanager_nmcli", "/usr/sbin/ifup", + "/usr/sbin/ifdown", "/usr/sbin/ifquery", + "native_nftables", + ), + ) + + def strict_receipt(self, script): + assert "__VC_VPC_APPLIED__=yes" in script + host_id = str(self.profile["host_id"]) + writes.append(host_id) + return "\n".join(( + "__VC_VPC_APPLIED__=yes", + f"__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}", + f"__VC_VPC_REVISION__={plan.desired_revision}", + f"__VC_VPC_DIGEST__={plan.desired_digest}", + f"__VC_VPC_NFT_RECEIPT__={NFT_RECEIPT}", + )) + + def confirmed_observation(self, host_plan, *, xml_by_instance=None): + assert xml_by_instance is None or set(xml_by_instance) == { + member.instance_uuid + for member in plan.members + if member.host_id == host_plan.host_id + } + return _confirmed_host(plan, host_plan.host_id), True, NFT_RECEIPT + + monkeypatch.setattr(VpcPrivateHostAdapter, "_run", strict_receipt) + monkeypatch.setattr( + VpcPrivateHostAdapter, "_observe", confirmed_observation) + return coordinator, plan, writes + + +def test_confirmed_ssh_apply_then_deterministic_libvirt_failure_releases_family( + tmp_path, monkeypatch): + from virsh_panel.libvirt_service import VMError, VMService + + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + target = _target() + coordinator, plan, writes = _install_coordinator( + engine, store, target, monkeypatch) + attempts: list[str] = [] + failing = {"host-a"} + + monkeypatch.setattr( + VMService, "require_vpc_members_stopped", + lambda *_args, **_kwargs: None, + ) + + def configure(self, instance_uuid, **_kwargs): + attempts.append(self.execution_scope) + with self._operation_guard( + instance_uuid, storage=True, + tracked_mutations=True): + if self.execution_scope in failing: + raise VMError( + "native snapshot state read failed before libvirt write", + 502, + error_code="vpc_vm_native_snapshot_state_unknown", + ) + mark_remote_write_started() + return { + "xml": f"inactive:{instance_uuid}", + "live": None, + "seed_verified": True, + "restart_required": False, + } + + monkeypatch.setattr( + VMService, "ensure_private_interface", + configure, + ) + status, result = await request( + engine, "POST", "/internal/v1/vpcs/apply", target, + operation_id=target["operation_id"]) + + assert status == 200, result + assert attempts == ["host-a"], json.dumps(result, sort_keys=True) + assert writes == ["host-a"] + durable = _durable_vpc(store) + assert durable["hosts"][0]["last_error_code"] \ + == "vpc_vm_native_snapshot_state_unknown" + assert durable["hosts"][0]["state"] == "uncertain" + assert engine.coordinator.snapshot() == {} + + failing.clear() + attempts.clear() + retry_status, retry_result = await request( + engine, "POST", "/internal/v1/vpcs/apply", target, + operation_id=target["operation_id"]) + assert retry_status == 200, retry_result + assert attempts == ["host-a"] + assert writes == ["host-a"] + assert engine.coordinator.snapshot() == {} + + asyncio.run(run()) + + +def test_confirmed_host_a_then_host_b_prewrite_failure_releases_family( + tmp_path, monkeypatch): + from virsh_panel.libvirt_service import VMError, VMService + + async def run(): + async with running_engine(tmp_path) as (engine, store, _services): + target = _target(two_hosts=True) + coordinator, plan, writes = _install_coordinator( + engine, store, target, monkeypatch) + attempts: list[str] = [] + failing = {"host-b"} + + monkeypatch.setattr( + VMService, "require_vpc_members_stopped", + lambda *_args, **_kwargs: None, + ) + + def configure(self, instance_uuid, **_kwargs): + attempts.append(self.execution_scope) + with self._operation_guard( + instance_uuid, storage=True, + tracked_mutations=True): + if self.execution_scope in failing: + raise VMError( + "host B snapshot state read failed before libvirt write", + 502, + error_code="vpc_vm_native_snapshot_state_unknown", + ) + mark_remote_write_started() + return { + "xml": f"inactive:{instance_uuid}", + "live": None, + "seed_verified": True, + "restart_required": False, + } + + monkeypatch.setattr( + VMService, "ensure_private_interface", configure) + status, result = await request( + engine, "POST", "/internal/v1/vpcs/apply", target, + operation_id=target["operation_id"]) + + assert status == 200, result + assert attempts == ["host-a", "host-b"], json.dumps( + result, sort_keys=True) + assert writes == ["host-a", "host-b"] + durable = _durable_vpc(store) + by_host = {host["host_id"]: host for host in durable["hosts"]} + assert coordinator._confirmed_active_host( + by_host["host-a"], plan, allow_pending=True) + assert by_host["host-b"]["last_error_code"] \ + == "vpc_vm_native_snapshot_state_unknown" + assert engine.coordinator.snapshot() == {} + + failing.clear() + attempts.clear() + retry_status, retry_result = await request( + engine, "POST", "/internal/v1/vpcs/apply", target, + operation_id=target["operation_id"]) + assert retry_status == 200, retry_result + assert attempts == ["host-b"] + assert writes == ["host-a", "host-b"] + assert engine.coordinator.snapshot() == {} + + asyncio.run(run()) diff --git a/engine/tests/test_vpc_dataplane.py b/engine/tests/test_vpc_dataplane.py index 346e1cd4..6906cfbf 100644 --- a/engine/tests/test_vpc_dataplane.py +++ b/engine/tests/test_vpc_dataplane.py @@ -1415,7 +1415,8 @@ def _observe_with_route_backed_wg_markers(plan, *, allowed, key=None, markers.append(item) markers.append(f"__VC_VPC_WG_PEER_SET__={peer_set}") adapter = VpcPrivateHostAdapter(object(), {}) - adapter._deployment_contract = lambda: ("networkmanager", "", "", "") + adapter._deployment_contract = lambda: ( + "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join(markers) + "\n" return adapter.observe(plan, xml_by_instance={ member.instance_uuid: { @@ -1505,7 +1506,8 @@ def test_stopped_host_reports_acl_revision_only_after_exact_configuration_proof( prefix, value = invalid_markers[acl_proof] markers = [prefix + value if item.startswith(prefix) else item for item in markers] adapter = VpcPrivateHostAdapter(object(), {}) - adapter._deployment_contract = lambda: ("networkmanager", "", "", "") + adapter._deployment_contract = lambda: ( + "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join(markers) + "\n" observed = adapter.observe(plan, xml_by_instance={ @@ -1569,7 +1571,7 @@ def test_stopped_linkdown_host_config_remains_reusable_without_becoming_active() )) adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( - "networkmanager", "", "", "", FIREWALL_MODE_NATIVE) + "networkmanager_nmcli", "", "", "", FIREWALL_MODE_NATIVE) adapter._run = lambda _script: "\n".join(markers) + "\n" observed, host_ready, _receipt = adapter._observe( @@ -1626,7 +1628,7 @@ def test_topology_mismatch_revokes_reusable_host_config_proof(): )) adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( - "networkmanager", "", "", "", FIREWALL_MODE_NATIVE) + "networkmanager_nmcli", "", "", "", FIREWALL_MODE_NATIVE) adapter._run = lambda _script: "\n".join(markers) + "\n" observed, host_ready, _receipt = adapter._observe( @@ -1664,7 +1666,8 @@ def test_observer_does_not_activate_unavailable_or_mismatched_guest(): desired_digest=plan.desired_digest, ) adapter = VpcPrivateHostAdapter(object(), {}) - adapter._deployment_contract = lambda: ("networkmanager", "", "", "") + adapter._deployment_contract = lambda: ( + "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join( _active_observation_markers(plan)) + "\n" @@ -1743,7 +1746,7 @@ def test_observer_rejects_shared_fabric_nft_mismatch(): ] adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( - "networkmanager", "", "", "") + "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join(markers) + "\n" observed = adapter.observe(plan, xml_by_instance={ @@ -1828,7 +1831,8 @@ def test_observer_reports_pending_restart_instead_of_active(): )) adapter = VpcPrivateHostAdapter(object(), {}) - adapter._deployment_contract = lambda: ("networkmanager", "", "", "") + adapter._deployment_contract = lambda: ( + "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join(markers) + "\n" observed = adapter.observe(plan, xml_by_instance={ member.instance_uuid: { diff --git a/engine/tests/test_vpc_nft_dataplane.py b/engine/tests/test_vpc_nft_dataplane.py index 2e23ae29..62f33d02 100644 --- a/engine/tests/test_vpc_nft_dataplane.py +++ b/engine/tests/test_vpc_nft_dataplane.py @@ -244,6 +244,8 @@ def test_vpc_observe_strictly_binds_fabric_digest_semantics_and_tenant_set( script = render_observe_script( _plan(), firewall_mode=FIREWALL_MODE_NATIVE) assert NFT_LIVE_SNAPSHOT_SH in script + assert ('__VC_VPC_${artifact_name}_PRESENT__=$artifact_present' + in script) snapshot_functions = NFT_LIVE_SNAPSHOT_SH guard_start = script.index("vc_fabric_nft_strict_shape() {") loader_start = script.index( @@ -251,7 +253,7 @@ def test_vpc_observe_strictly_binds_fabric_digest_semantics_and_tenant_set( guard_functions = script[guard_start:loader_start] loader_end = script.index("FABRIC_NFT_SNAPSHOT=", loader_start) loader_functions = script[loader_start:loader_end] - observe_start = script.index("vc_vpc_fabric_observe_ready() {") + observe_start = script.index("vc_vpc_fabric_closure_ready() {") observe_end = script.index("firewall_mode_state=owned", observe_start) observe_function = script[observe_start:observe_end] @@ -306,27 +308,34 @@ NFT=vc_test_nft guard_functions, loader_functions, 'VPC_TENANT_FILE="$EXPECTED"', - 'VPC_STATE_STATUS=owned', + 'VPC_STATE_STATUS="$STATE_STATUS"', 'FABRIC_NFT_SNAPSHOT="$SNAPSHOT"', "vpc_state_load() { :; }", observe_function, - "vc_vpc_fabric_observe_ready", + "vc_vpc_fabric_closure_ready", )) - def run(table_text, set_text): + def run(table_text, set_text, *, state_status='owned', expected_text=None): live_table.write_text(table_text) live_set.write_text(set_text) + if expected_text is not None: + expected.write_text(expected_text) return subprocess.run( ["/bin/sh", "-eu", "-c", harness], env=dict(os.environ, LIVE_TABLE=str(live_table), LIVE_SET=str(live_set), EXPECTED=str(expected), - SNAPSHOT=str(snapshot)), + SNAPSHOT=str(snapshot), STATE_STATUS=state_status), capture_output=True, text=True, check=False, ) exact = ["10.44.0.10/32", "10.44.0.11/32"] accepted = run(table_with(exact), set_with(exact)) assert accepted.returncode == 0, accepted.stderr + empty = run( + _nft_rules_text(FABRIC_DIGEST), set_with([]), + state_status='empty', expected_text='') + assert empty.returncode == 0, empty.stderr + expected.write_text("10.44.0.10/32\n10.44.0.11/32\n") assert run( table_with(exact, digest="ef" * 32), set_with(exact), ).returncode != 0 @@ -348,9 +357,9 @@ NFT=vc_test_nft owner_at = script.index("nft_plan_owned() {", strict_at) loader_at = script.index( 'VPC_STATE_DIR="/etc/virt-cloud/vpc-dataplane"', owner_at) - ready_at = script.index("vc_vpc_fabric_observe_ready() {", loader_at) - call_at = script.rindex("if vc_vpc_fabric_observe_ready; then") - marker_at = script.index("__VC_VPC_FABRIC_NFT__", call_at) + ready_at = script.index("vc_vpc_fabric_closure_ready() {", loader_at) + call_at = script.rindex("vc_vpc_fabric_closure_ready; then") + marker_at = script.index("__VC_VPC_REMOVAL_FABRIC_NFT__", call_at) assert script.index(expected_assignment) < strict_at < owner_at assert owner_at < loader_at < ready_at < call_at < marker_at assert "VPC_STATE_OBSERVE=\"yes\"" in script diff --git a/engine/tests/test_vpc_observer_optional_results.py b/engine/tests/test_vpc_observer_optional_results.py index 7921fe72..63063750 100644 --- a/engine/tests/test_vpc_observer_optional_results.py +++ b/engine/tests/test_vpc_observer_optional_results.py @@ -44,7 +44,7 @@ def _run(shell: str, script: str, env: dict[str, str]): def test_observer_optional_nmcli_preserves_manager_ownership( tmp_path, shell, scenario, expected): plan = _plan() - body = _fragment("nm_owned=no\n", 'if "$IP" -d link show dev "$BRIDGE"') + body = _fragment("nm_owned=no\n", "for artifact_spec in ") candidates = [tmp_path / f"nmcli-{index}" for index in range(4)] for original, candidate in zip(NM_PATHS, candidates): body = body.replace(original, shlex.quote(str(candidate))) diff --git a/engine/tests/test_vpc_operation_identity.py b/engine/tests/test_vpc_operation_identity.py index c238b177..48723241 100644 --- a/engine/tests/test_vpc_operation_identity.py +++ b/engine/tests/test_vpc_operation_identity.py @@ -19,7 +19,8 @@ from test_durable_fabric_jobs import FabricStore from test_engine_boundary import SUPER, cloud_engine_request from test_engine_coordination import local_backends, request, running_engine from test_vpc import ( - FakeFabric, confirm_delete_hosts, delete_payload, payload, + FakeFabric, INSTANCE_A, INSTANCE_B, confirm_delete_hosts, delete_payload, + payload, ) from test_vpc_admission_coordination import vpc_payload @@ -33,6 +34,29 @@ def identity(plan): "desired_digest": plan.desired_digest} +def assert_retained_claim_family(rows, plan, *, root_kind, member_uuids): + expected_kinds = { + (): root_kind, + ("catalog:network",): "resource", + **{ + (f"vm:host-a:{instance_uuid}",): "resource" + for instance_uuid in member_uuids + }, + } + by_resources = { + tuple(row["resources"]): row + for row in rows.values() + } + assert len(rows) == len(expected_kinds) + assert set(by_resources) == set(expected_kinds) + for resources, expected_kind in expected_kinds.items(): + row = by_resources[resources] + assert row["kind"] == expected_kind + assert row["state"] == "uncertain" + assert row["operation_id"] == plan.operation_id + assert row["metadata"] == identity(plan) + + async def explicit_request(engine, method, target, value=None): actor = {**SUPER, "vpc_reconcile_explicit": True} body = b"" if value is None else json.dumps(value).encode() @@ -1404,7 +1428,8 @@ def test_periodic_unknown_execution_never_returns_success( tmp_path, monkeypatch): async def run(): async with running_engine(tmp_path) as (engine, store, _services): - vpc, _target = await seed(engine, store) + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) async def host(*_args, **_kwargs): mark_execution_unknown() @@ -1414,8 +1439,9 @@ def test_periodic_unknown_execution_never_returns_success( await vpc.reconcile("vpc-hk-1") assert raised.value.error_code == "coordination_uncertain" rows = engine.coordinator.snapshot() - assert len(rows) == 1 - assert next(iter(rows.values()))["state"] == "uncertain" + assert_retained_claim_family( + rows, plan, root_kind="background", + member_uuids=(INSTANCE_A, INSTANCE_B)) asyncio.run(run()) @@ -1513,7 +1539,8 @@ def test_http_unknown_execution_without_residual_returns_503( tmp_path, monkeypatch): async def run(): async with running_engine(tmp_path) as (engine, store, _services): - vpc, _target = await seed(engine, store) + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) async def host(*_args, **_kwargs): mark_execution_unknown() @@ -1524,10 +1551,9 @@ def test_http_unknown_execution_without_residual_returns_503( assert status == 503, result assert result["error_code"] == "coordination_uncertain" rows = engine.coordinator.snapshot() - assert len(rows) == 1 - root = next(iter(rows.values())) - assert root["state"] == "uncertain" - assert root["operation_id"] == OPERATION_ID + assert_retained_claim_family( + rows, plan, root_kind="request", + member_uuids=(INSTANCE_A, INSTANCE_B)) asyncio.run(run()) @@ -1536,7 +1562,8 @@ def test_http_lost_child_release_reply_with_absent_token_returns_503( tmp_path, monkeypatch): async def run(): async with running_engine(tmp_path) as (engine, store, _services): - vpc, _target = await seed(engine, store) + vpc, target = await seed(engine, store) + plan = parse_vpc_plan(target) release = engine.coordinator.release lost_reply = False @@ -1563,10 +1590,12 @@ def test_http_lost_child_release_reply_with_absent_token_returns_503( assert result["error_code"] == "coordination_uncertain" assert lost_reply is True rows = engine.coordinator.snapshot() - assert len(rows) == 1 - root = next(iter(rows.values())) - assert root["kind"] == "request" - assert root["state"] == "uncertain" + assert_retained_claim_family( + rows, plan, root_kind="request", + member_uuids=(INSTANCE_A,)) + assert all( + f"vm:host-a:{INSTANCE_B}" not in row["resources"] + for row in rows.values()) asyncio.run(run()) diff --git a/engine/tests/test_vpc_persistence_owner_gate.py b/engine/tests/test_vpc_persistence_owner_gate.py index 44b2c19c..ba5cd0a8 100644 --- a/engine/tests/test_vpc_persistence_owner_gate.py +++ b/engine/tests/test_vpc_persistence_owner_gate.py @@ -21,6 +21,7 @@ def _persistence_gate(**overrides: str) -> str: ) assert match is not None, "The production persistence gate was not found." values = { + "MODE": "networkmanager_nmcli", "firewall_mode_state": "owned", "NFT_MAIN_COUNT": "1", "persistence_target": "yes", @@ -30,7 +31,11 @@ def _persistence_gate(**overrides: str) -> str: "nft_snapshot_state": "yes", "nm_owned": "yes", "nm_present": "yes", + "nm_profile_query": "ok", + "nm_profile_count": "1", "ifup_owned": "no", + "ifup_profile_query": "ok", + "ifup_profile_count": "0", "persistence": "no", } values.update(overrides) @@ -46,8 +51,20 @@ def _persistence_gate(**overrides: str) -> str: @pytest.mark.parametrize("owners", [ - {"nm_owned": "yes", "nm_present": "yes", "ifup_owned": "no"}, - {"nm_owned": "no", "nm_present": "no", "ifup_owned": "yes"}, + { + "MODE": "networkmanager_nmcli", + "nm_owned": "yes", "nm_present": "yes", + "nm_profile_query": "ok", "nm_profile_count": "1", + "ifup_owned": "no", + "ifup_profile_query": "ok", "ifup_profile_count": "0", + }, + { + "MODE": "ifupdown", + "nm_owned": "no", "nm_present": "no", + "nm_profile_query": "not_applicable", "nm_profile_count": "0", + "ifup_owned": "yes", + "ifup_profile_query": "ok", "ifup_profile_count": "1", + }, ]) def test_persistence_accepts_each_exact_exclusive_owner(owners): assert _persistence_gate(**owners) == "yes" @@ -64,6 +81,17 @@ def test_persistence_rejects_dual_foreign_missing_or_unknown_owners(owners): assert _persistence_gate(**owners) == "no" +@pytest.mark.parametrize("inventory", [ + {"nm_profile_query": "invalid"}, + {"nm_profile_count": "2"}, + {"ifup_profile_query": "invalid"}, + {"ifup_profile_count": "1"}, +]) +def test_networkmanager_persistence_rejects_ambiguous_or_opposite_inventory( + inventory): + assert _persistence_gate(**inventory) == "no" + + @pytest.mark.parametrize("field,value", [ ("firewall_mode_state", "mismatch"), ("NFT_MAIN_COUNT", "0"), @@ -75,8 +103,20 @@ def test_persistence_rejects_dual_foreign_missing_or_unknown_owners(owners): ("nft_snapshot_state", "no"), ]) @pytest.mark.parametrize("owners", [ - {"nm_owned": "yes", "nm_present": "yes", "ifup_owned": "no"}, - {"nm_owned": "no", "nm_present": "no", "ifup_owned": "yes"}, + { + "MODE": "networkmanager_nmcli", + "nm_owned": "yes", "nm_present": "yes", + "nm_profile_query": "ok", "nm_profile_count": "1", + "ifup_owned": "no", + "ifup_profile_query": "ok", "ifup_profile_count": "0", + }, + { + "MODE": "ifupdown", + "nm_owned": "no", "nm_present": "no", + "nm_profile_query": "not_applicable", "nm_profile_count": "0", + "ifup_owned": "yes", + "ifup_profile_query": "ok", "ifup_profile_count": "1", + }, ]) def test_each_persistence_prerequisite_remains_required(field, value, owners): assert _persistence_gate(**owners, **{field: value}) == "no" diff --git a/engine/tests/test_vpc_revision_durability.py b/engine/tests/test_vpc_revision_durability.py new file mode 100644 index 00000000..e54b4bf6 --- /dev/null +++ b/engine/tests/test_vpc_revision_durability.py @@ -0,0 +1,674 @@ +"""Regression contracts for durable VPC revision admission and migration.""" +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest + +from virsh_panel.fabric import FabricNode, FabricPlan +from virsh_panel.vpc import ( + VPC_DELETE_RECEIPT_KIND, + VpcCoordinator, + VpcError, + parse_vpc_delete_plan, + parse_vpc_plan, +) +from virsh_panel.vpc_dataplane import derive_private_mac +from test_vpc import ( + FakeFabric, + INSTANCE_A, + INSTANCE_B, + MemoryStore, + confirm_delete_hosts, + delete_payload, + payload, +) + + +GATE = {"active": True, "available": True} + + +class SimulatedProcessInterruption(BaseException): + """Model a process exit after a host write has crossed its boundary.""" + + +def _two_host_payload(*, desired_revision=1, operation_id="vpc-op-00000001"): + return payload( + desired_revision=desired_revision, + operation_id=operation_id, + hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", + "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-b", "members": [{ + "member_id": "member-b", + "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}, + ], + ) + + +def _exact_host(plan, host_id, *, pending=None): + member = next(item for item in plan.members if item.host_id == host_id) + return { + "host_id": host_id, + "vpc_id": plan.vpc_id, + "member_ids": [member.member_id], + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "observed_revision": plan.desired_revision, + "observed_digest": plan.desired_digest, + "state": "active", + "transport_verified": True, + "private_dataplane_verified": True, + "config_verified": True, + "host_config_reusable": True, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "retiring_members": [], + **({"pending_member_mutations": list(pending)} + if pending is not None else {}), + "member_nics": [{ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "private_ip_cidr": f"{member.private_ip}/32", + "private_mac": derive_private_mac(plan.vpc_id, member.member_id), + "config_verified": True, + "xml_verified": True, + "seed_verified": True, + "domain_running": False, + "guest_network_verified": False, + "guest_network_readback": "stopped", + "guest_reason_code": "vpc_private_guest_stopped", + "restart_required": False, + }], + "routes": [], + "wg_allowed_ips": [], + "readback": {}, + } + + +def _decode(store, kind, key): + raw = store.get(kind, key) + return json.loads(raw) if raw is not None else None + + +def test_new_revision_is_durable_with_children_before_first_host_write(): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a", "host-b"))) + previous_plan = parse_vpc_plan(payload()) + previous = coordinator._desired_record( + previous_plan, GATE, state="applying") + coordinator._save("vpc_desired", previous_plan.vpc_id, previous) + coordinator._save_children(previous) + + plan = parse_vpc_plan(_two_host_payload( + desired_revision=2, operation_id="vpc-op-00000002")) + durable_at_write = {} + host_writes = [] + + async def first_host_write_then_interrupt(*_args, **_kwargs): + host_writes.append("started") + durable_at_write.update({ + "root": _decode(store, "vpc_desired", plan.vpc_id), + "member": _decode( + store, "vpc_member", f"{plan.vpc_id}.member-b"), + "host": _decode(store, "vpc_host", f"{plan.vpc_id}.host-b"), + }) + raise SimulatedProcessInterruption + + coordinator._reconcile_dataplane = first_host_write_then_interrupt + + with pytest.raises(SimulatedProcessInterruption): + asyncio.run(coordinator._apply_locked(plan)) + + assert host_writes == ["started"] + assert durable_at_write["root"]["desired_revision"] == 2 + assert durable_at_write["root"]["desired_digest"] == plan.desired_digest + assert durable_at_write["member"]["desired_revision"] == 2 + assert durable_at_write["member"]["instance_uuid"] == INSTANCE_B + assert durable_at_write["host"]["desired_revision"] == 2 + assert durable_at_write["host"]["pending_member_mutations"] == [INSTANCE_B] + + # A process exit cannot run an in-memory finalizer. The pre-write intent + # must therefore remain sufficient for a fresh process to resume revision 2. + assert _decode(store, "vpc_desired", plan.vpc_id)["desired_revision"] == 2 + assert _decode( + store, "vpc_member", f"{plan.vpc_id}.member-b")[ + "desired_digest"] == plan.desired_digest + + +def test_retry_preflights_and_locks_full_durable_mutation_closure(): + plan = parse_vpc_plan(_two_host_payload()) + fabric_plan = FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, + ( + FabricNode( + "host-a", "100.64.0.1", "192.0.2.1:51820", 51820), + FabricNode( + "host-b", "100.64.0.2", "192.0.2.2:51820", 51820), + ), + "f" * 64, + ) + + class RetryFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, _plan): + return { + host_id: ( + SimpleNamespace(manager=object()), + {"host_id": host_id}, + ) + for host_id in ("host-a", "host-b") + } + + acquired = [] + + @asynccontextmanager + async def vm_lock(host_id, instance_uuid): + acquired.append((host_id, instance_uuid)) + yield + + coordinator = VpcCoordinator( + MemoryStore(), RetryFabric(hosts=("host-a", "host-b")), + registry=object(), vm_lock=vm_lock, + ) + coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False + coordinator._validate_dataplane_resources = lambda _plan: None + coordinator._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, + } + coordinator._fabric_peers_for_plan = lambda _plan, _record: [] + + record = coordinator._desired_record(plan, GATE, state="applying") + by_host = {host["host_id"]: host for host in record["hosts"]} + # Host A has exact persistent configuration, so only host B belongs to the + # next write selection. Its durable marker remains part of this revision's + # stop-before-write closure until every selected host has converged. + by_host["host-a"].update( + _exact_host(plan, "host-a", pending=[INSTANCE_A])) + checked = [] + host_writes = [] + + def require_stopped(_profile, host_id, instance_uuids): + checked.append((host_id, tuple(instance_uuids))) + if host_id == "host-a": + raise VpcError( + "member became running", 409, + "vpc_member_requires_stopped") + + coordinator._require_host_member_mutations_stopped = require_stopped + coordinator._run_host_dataplane = ( + lambda *_args, **_kwargs: + host_writes.append(_args[1]) or {}) + + asyncio.run(coordinator._reconcile_pending_hosts( + plan, record, GATE, plan, + host_selection=({"host-b"}, set()), + )) + + assert acquired == [ + ("host-a", INSTANCE_A), + ("host-b", INSTANCE_B), + ] + assert checked == [ + ("host-a", (INSTANCE_A,)), + ("host-b", (INSTANCE_B,)), + ] + assert host_writes == [] + result_by_host = {host["host_id"]: host for host in record["hosts"]} + assert result_by_host["host-b"]["state"] == "applying" + assert result_by_host["host-b"]["last_error_code"] \ + == "vpc_member_requires_stopped" + assert result_by_host["host-a"]["pending_member_mutations"] == [INSTANCE_A] + + +def test_root_delete_without_prior_record_preflights_full_signed_vm_closure(): + fabric_plan = FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, + ( + FabricNode( + "host-a", "100.64.0.1", "192.0.2.1:51820", 51820), + FabricNode( + "host-b", "100.64.0.2", "192.0.2.2:51820", 51820), + ), + "f" * 64, + ) + + class DeleteFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, _plan): + return { + host_id: ( + SimpleNamespace(manager=object()), + {"host_id": host_id}, + ) + for host_id in ("host-a", "host-b") + } + + acquired = [] + + @asynccontextmanager + async def vm_lock(host_id, instance_uuid): + acquired.append((host_id, instance_uuid)) + yield + + store = MemoryStore() + coordinator = VpcCoordinator( + store, DeleteFabric(hosts=("host-a", "host-b")), + registry=object(), vm_lock=vm_lock, + ) + coordinator._validate_dataplane_resources = lambda _plan: None + coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False + coordinator._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, + } + coordinator._fabric_peers_for_plan = lambda _plan, _record: [] + checked = [] + host_writes = [] + + def require_stopped(_profile, host_id, instance_uuids): + checked.append((host_id, tuple(instance_uuids))) + if host_id == "host-b": + raise VpcError( + "member became running", 409, + "vpc_member_requires_stopped") + + coordinator._require_host_member_mutations_stopped = require_stopped + coordinator._run_host_dataplane = ( + lambda *_args, **_kwargs: + host_writes.append(_args[1]) or {}) + target = delete_payload( + vpc_id="vpc-never-applied", + desired_revision=3, + operation_id="vpc-delete-full-stop-fence-0001", + hosts=("host-a", "host-b"), + cidr="10.21.0.0/24", + retiring_members=( + { + "member_id": "member-a", + "instance_uuid": INSTANCE_A, + "host_id": "host-a", + "private_ip_cidr": "10.21.0.10/32", + "private_mac": derive_private_mac( + "vpc-never-applied", "member-a"), + }, + { + "member_id": "member-b", + "instance_uuid": INSTANCE_B, + "host_id": "host-b", + "private_ip_cidr": "10.21.0.11/32", + "private_mac": derive_private_mac( + "vpc-never-applied", "member-b"), + }, + ), + ) + + with pytest.raises(VpcError) as blocked: + asyncio.run(coordinator.delete("vpc-never-applied", target)) + + assert blocked.value.error_code == "vpc_member_requires_stopped" + assert acquired == [ + ("host-a", INSTANCE_A), + ("host-b", INSTANCE_B), + ] + assert checked == [ + ("host-a", (INSTANCE_A,)), + ("host-b", (INSTANCE_B,)), + ] + assert host_writes == [] + assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-never-applied") is None + durable = _decode(store, "vpc_desired", "vpc-never-applied") + pending = { + host["host_id"]: host["pending_member_mutations"] + for host in durable["hosts"] + } + assert pending == { + "host-a": [INSTANCE_A], + "host-b": [INSTANCE_B], + } + + +def test_root_delete_partial_retry_ignores_exact_offline_host_fence(): + vpc_id = "vpc-partial-delete" + target = delete_payload( + vpc_id=vpc_id, + desired_revision=3, + operation_id="vpc-delete-partial-fence-0001", + hosts=("host-a", "host-b"), + cidr="10.21.0.0/24", + retiring_members=( + { + "member_id": "member-a", + "instance_uuid": INSTANCE_A, + "host_id": "host-a", + "private_ip_cidr": "10.21.0.10/32", + "private_mac": derive_private_mac(vpc_id, "member-a"), + }, + { + "member_id": "member-b", + "instance_uuid": INSTANCE_B, + "host_id": "host-b", + "private_ip_cidr": "10.21.0.11/32", + "private_mac": derive_private_mac(vpc_id, "member-b"), + }, + ), + ) + plan, _manifest = parse_vpc_delete_plan(target) + store = MemoryStore() + first = VpcCoordinator( + store, FakeFabric(hosts=("host-a", "host-b")), registry=object()) + first._validate_dataplane_resources = lambda _plan: None + + async def leave_host_b_pending( + delete_plan, record, _gate, previous_plan=None, **_kwargs): + host_a = next( + host for host in record["hosts"] + if host["host_id"] == "host-a") + confirm_delete_hosts(delete_plan, {"hosts": [host_a]}) + host_b = next( + host for host in record["hosts"] + if host["host_id"] == "host-b") + host_b.update({ + "state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": "vpc_host_unavailable", + }) + + first._reconcile_dataplane = leave_host_b_pending + with pytest.raises(VpcError) as incomplete: + asyncio.run(first.delete(vpc_id, target)) + assert incomplete.value.error_code == "vpc_delete_incomplete" + + durable = _decode(store, "vpc_desired", vpc_id) + durable_hosts = { + host["host_id"]: host for host in durable["hosts"] + } + assert first._confirmed_removal( + durable_hosts["host-a"], plan.desired_revision, + plan.desired_digest) + assert { + host_id: durable_hosts[host_id]["pending_member_mutations"] + for host_id in ("host-a", "host-b") + } == { + "host-a": [INSTANCE_A], + "host-b": [INSTANCE_B], + } + + fabric_plan = FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, + ( + FabricNode( + "host-a", "100.64.0.1", "192.0.2.1:51820", 51820), + FabricNode( + "host-b", "100.64.0.2", "192.0.2.2:51820", 51820), + ), + "f" * 64, + ) + component_requests = [] + + class RetryFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, selected): + requested = [node.host_id for node in selected.nodes] + component_requests.append(requested) + if "host-a" in requested: + raise AssertionError( + "exactly removed offline host must not be requested") + return { + "host-b": ( + SimpleNamespace(manager=object()), + {"host_id": "host-b"}, + ), + } + + acquired = [] + + @asynccontextmanager + async def vm_lock(host_id, instance_uuid): + acquired.append((host_id, instance_uuid)) + yield + + retry = VpcCoordinator( + store, RetryFabric(hosts=("host-a", "host-b")), + registry=object(), vm_lock=vm_lock, + ) + retry._validate_dataplane_resources = lambda _plan: None + retry._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False + retry._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, + } + retry._fabric_peers_for_plan = lambda _plan, _record: [] + stopped = [] + host_runs = [] + + def require_stopped(_profile, host_id, instance_uuids): + stopped.append((host_id, tuple(instance_uuids))) + + def confirm_host(_plan, host_id, *_args, **_kwargs): + host_runs.append(host_id) + assert host_id == "host-b" + return { + "state": "active", + "removed": True, + "transport_verified": True, + "private_dataplane_verified": True, + "config_verified": True, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "member_nics": [], + "routes": [], + "wg_allowed_ips": [], + "proxy_ips": [], + "bridge": None, + "bridge_owner": None, + "nft_table": None, + "anti_spoof": False, + "rp_filter": None, + "proxy_arp": None, + "proxy_arp_pvlan": None, + "readback": "confirmed", + "rollback": "none", + "reason_code": None, + } + + retry._require_host_member_mutations_stopped = require_stopped + retry._run_host_dataplane = confirm_host + result = asyncio.run(retry.delete(vpc_id, target)) + + assert component_requests == [["host-b"]] + assert acquired == [("host-b", INSTANCE_B)] + assert stopped == [("host-b", (INSTANCE_B,))] + assert host_runs == ["host-b"] + assert result["deleted"] is True + assert result["removed_host_ids"] == ["host-a", "host-b"] + assert result["readback"] == "confirmed" + receipt = _decode(store, VPC_DELETE_RECEIPT_KIND, vpc_id) + assert receipt["operation_id"] == plan.operation_id + assert receipt["desired_digest"] == plan.desired_digest + + +def test_legacy_exact_host_migrates_empty_pending_without_host_write(): + store = MemoryStore() + coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) + plan = parse_vpc_plan(payload()) + record = coordinator._desired_record(plan, GATE, state="applying") + record["hosts"] = [_exact_host(plan, "host-a")] + coordinator._finish(record, GATE) + coordinator._save("vpc_desired", plan.vpc_id, record) + coordinator._save_children(record) + host_writes = [] + + async def unexpected_host_write(*_args, **_kwargs): + host_writes.append(True) + + coordinator._reconcile_dataplane = unexpected_host_write + asyncio.run(coordinator.reconcile(plan.vpc_id)) + + durable = _decode(store, "vpc_desired", plan.vpc_id) + durable_host = _decode( + store, "vpc_host", f"{plan.vpc_id}.host-a") + assert host_writes == [] + assert durable["hosts"][0]["pending_member_mutations"] == [] + assert durable_host["pending_member_mutations"] == [] + + +def test_explicit_empty_pending_fence_is_not_rederived_from_current_members(): + coordinator = VpcCoordinator( + MemoryStore(), FakeFabric(hosts=("host-a",))) + plan = parse_vpc_plan(payload()) + record = coordinator._desired_record(plan, GATE, state="applying") + host = record["hosts"][0] + host["pending_member_mutations"] = [] + host["config_verified"] = False + host["config_observed_revision"] = 0 + host["config_observed_digest"] = None + + assert coordinator._migrate_legacy_pending_member_mutations( + record, plan) is False + assert host["pending_member_mutations"] == [] + + +def test_legacy_incomplete_current_and_retiring_hosts_derive_stop_closure(): + coordinator = VpcCoordinator( + MemoryStore(), FakeFabric(hosts=("host-a", "host-b"))) + plan = parse_vpc_plan(_two_host_payload()) + + current = _exact_host(plan, "host-a") + current.update({ + "state": "applying", + "config_verified": False, + "host_config_reusable": False, + "config_observed_revision": 0, + "config_observed_digest": None, + "member_nics": [], + }) + current_record = coordinator._desired_record( + plan, GATE, state="applying") + current_record["hosts"] = [ + current if host["host_id"] == "host-a" else host + for host in current_record["hosts"] + ] + assert coordinator._migrate_legacy_pending_member_mutations( + current_record, plan) is True + current_by_host = { + host["host_id"]: host for host in current_record["hosts"]} + assert current_by_host["host-a"]["pending_member_mutations"] == [INSTANCE_A] + + retiring = { + "host_id": "host-a", + "vpc_id": plan.vpc_id, + "member_ids": ["member-a"], + "retiring": True, + "retiring_members": [{ + "member_id": "member-a", + "instance_uuid": INSTANCE_A, + "host_id": "host-a", + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + "private_mac": derive_private_mac(plan.vpc_id, "member-a"), + }], + } + host_b_only = parse_vpc_plan(payload( + hosts=[{"host_id": "host-b", "members": [{ + "member_id": "member-b", + "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}], + )) + retiring_record = coordinator._desired_record( + host_b_only, GATE, state="applying") + retiring_record["hosts"].append(retiring) + assert coordinator._migrate_legacy_pending_member_mutations( + retiring_record, host_b_only) is True + retiring_by_host = { + host["host_id"]: host for host in retiring_record["hosts"]} + assert retiring_by_host["host-a"]["pending_member_mutations"] == [INSTANCE_A] + + +def test_legacy_retiring_host_with_malformed_identity_fails_closed(): + coordinator = VpcCoordinator( + MemoryStore(), FakeFabric(hosts=("host-a", "host-b"))) + plan = parse_vpc_plan(payload( + hosts=[{"host_id": "host-b", "members": [{ + "member_id": "member-b", + "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}], + )) + malformed = { + "host_id": "host-a", + "vpc_id": plan.vpc_id, + "member_ids": ["member-a"], + "retiring": True, + "retiring_members": [{ + "member_id": "member-a", + "instance_uuid": "not-a-domain-uuid", + "host_id": "host-a", + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + "private_mac": derive_private_mac(plan.vpc_id, "member-a"), + }], + } + + record = coordinator._desired_record(plan, GATE, state="applying") + record["hosts"].append(malformed) + + with pytest.raises(VpcError) as error: + coordinator._migrate_legacy_pending_member_mutations(record, plan) + + assert error.value.error_code == "vpc_record_corrupt" + + +@pytest.mark.parametrize("retiring_members", [None, []]) +def test_legacy_retiring_host_requires_complete_member_closure( + retiring_members): + coordinator = VpcCoordinator( + MemoryStore(), FakeFabric(hosts=("host-a", "host-b"))) + plan = parse_vpc_plan(payload( + hosts=[{"host_id": "host-b", "members": [{ + "member_id": "member-b", + "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}], + )) + retiring = { + "host_id": "host-a", + "vpc_id": plan.vpc_id, + "member_ids": ["member-a"], + "retiring": True, + } + if retiring_members is not None: + retiring["retiring_members"] = retiring_members + + record = coordinator._desired_record(plan, GATE, state="applying") + record["hosts"].append(retiring) + + with pytest.raises(VpcError) as error: + coordinator._migrate_legacy_pending_member_mutations(record, plan) + + assert error.value.error_code == "vpc_record_corrupt" diff --git a/engine/tests/test_vpc_route_readback.py b/engine/tests/test_vpc_route_readback.py index 3dc368fd..cd4535a7 100644 --- a/engine/tests/test_vpc_route_readback.py +++ b/engine/tests/test_vpc_route_readback.py @@ -27,7 +27,10 @@ def _program(phase): end = script.index("bridge_routes6=", start) else: start = script.index('if route_inventory=$("$IP" -o -4 -N route show table "$ROUTE_TABLE"') - end = script.index("while IFS='|' read -r peer_host", start) + # Stop at the end of route/rule readback. The following state and + # shared-Fabric removal checks depend on variables and helpers defined + # by the complete observe program and are outside this fragment test. + end = script.index("vpc_state_loaded=no", start) return helper[0] + "\n" + script[start:end] diff --git a/engine/tests/test_vpc_state_overlay.py b/engine/tests/test_vpc_state_overlay.py index 1ab24422..5039aca1 100644 --- a/engine/tests/test_vpc_state_overlay.py +++ b/engine/tests/test_vpc_state_overlay.py @@ -130,7 +130,8 @@ printf 'state=%s\n' "$VPC_STATE_STATUS" def _run_observe_wg_contract( - tmp_path, plan, texts, actual_by_key, *, peer_read_status=0): + tmp_path, plan, texts, actual_by_key, *, peer_read_status=0, + target_state_present=True): """Run the rendered observer's exact state-union/AllowedIPs block.""" state_dir = tmp_path / 'states' state_dir.mkdir(mode=0o700) @@ -144,7 +145,12 @@ def _run_observe_wg_contract( script = dataplane_module.render_observe_script(plan) block_start = script.index('vpc_state_loaded=no') - block_end = script.index('fabric_nft_state=no', block_start) + block_end = script.index('fabric_nft_closure=no', block_start) + target_state = ( + state_dir / '00000000001.state' + if target_state_present and texts + else tmp_path / 'removed-target.state' + ) harness = '\n'.join(( 'set -eu', _loader_from_script(script, observe=True), @@ -153,6 +159,7 @@ def _run_observe_wg_contract( 'VPC_PEER_FILE=' + shlex.quote(str(tmp_path / 'peers')), 'VPC_ROUTE_FILE=' + shlex.quote(str(tmp_path / 'routes')), 'VPC_TENANT_FILE=' + shlex.quote(str(tmp_path / 'tenants')), + 'STATE=' + shlex.quote(str(target_state)), 'VPC_WG_ROWS=' + shlex.quote(str(wg_rows)), 'VPC_WG_PEER_READ_STATUS=' + str(peer_read_status), 'FIREWALL_MODE=native_nftables', @@ -491,6 +498,8 @@ def test_observer_rejects_stale_tenant_on_authoritative_non_vpc_peer( exact_dir, observed, [_state_text(first), _state_text(second)], rows) assert exact.returncode == 0, exact.stderr assert '__VC_VPC_WG_PEER_SET__=exact\n' in exact.stdout + assert '__VC_VPC_REMOVAL_STATE_CLOSURE__=invalid\n' in exact.stdout + assert '__VC_VPC_REMOVAL_WG_PEER_SET__=exact\n' not in exact.stdout wrong_overlay_dir = tmp_path / 'wrong-overlay-closure' wrong_overlay_dir.mkdir() @@ -533,6 +542,61 @@ def test_observer_empty_authoritative_peer_set_requires_successful_readback( assert '__VC_VPC_WG_PEER_SET__=exact\n' not in result.stdout +def test_removal_wg_closure_accepts_empty_remaining_state(tmp_path): + plan = _plan() + plan = replace( + plan, + fabric_peer_keys=frozenset(peer.public_key for peer in plan.peers), + fabric_peer_overlays=frozenset( + (peer.public_key, peer.overlay_cidr) for peer in plan.peers), + ) + peer = plan.peers[0] + result = _run_observe_wg_contract( + tmp_path, plan, [], {peer.public_key: peer.overlay_cidr}, + target_state_present=False) + assert result.returncode == 0, result.stderr + assert '__VC_VPC_REMOVAL_STATE_CLOSURE__=empty\n' in result.stdout + assert '__VC_VPC_REMOVAL_WG_PEER_SET__=exact\n' in result.stdout + assert '__VC_VPC_WG_PEER_SET__=exact\n' not in result.stdout + + +def test_removal_wg_closure_rejects_stale_tenant_after_last_state(tmp_path): + plan = _plan() + plan = replace( + plan, + fabric_peer_keys=frozenset(peer.public_key for peer in plan.peers), + fabric_peer_overlays=frozenset( + (peer.public_key, peer.overlay_cidr) for peer in plan.peers), + ) + peer = plan.peers[0] + result = _run_observe_wg_contract( + tmp_path, plan, [], { + peer.public_key: peer.overlay_cidr + ',10.44.0.11/32', + }, target_state_present=False) + assert result.returncode == 0, result.stderr + assert '__VC_VPC_REMOVAL_STATE_CLOSURE__=empty\n' in result.stdout + assert '__VC_VPC_REMOVAL_WG_PEER_SET__=missing\n' in result.stdout + assert '__VC_VPC_REMOVAL_WG_PEER_SET__=exact\n' not in result.stdout + + +def test_removal_wg_closure_accepts_other_owned_vpc_state(tmp_path): + first, second = _two_vpc_plans() + first = replace( + first, + fabric_peer_keys=frozenset(peer.public_key for peer in first.peers), + fabric_peer_overlays=frozenset( + (peer.public_key, peer.overlay_cidr) for peer in first.peers), + ) + peer = first.peers[0] + result = _run_observe_wg_contract( + tmp_path, first, [_state_text(second)], { + peer.public_key: peer.overlay_cidr + ',10.45.0.11/32', + }, target_state_present=False) + assert result.returncode == 0, result.stderr + assert '__VC_VPC_REMOVAL_STATE_CLOSURE__=owned\n' in result.stdout + assert '__VC_VPC_REMOVAL_WG_PEER_SET__=exact\n' in result.stdout + + @pytest.mark.parametrize('mode', ['apply', 'observe']) @pytest.mark.parametrize('cidr_after_routes', [False, True]) def test_global_peer_authorization_cannot_authorize_a_foreign_state_route( diff --git a/engine/tests/test_wg_allowed_readback.py b/engine/tests/test_wg_allowed_readback.py index 8b0d484e..d70d5bf8 100644 --- a/engine/tests/test_wg_allowed_readback.py +++ b/engine/tests/test_wg_allowed_readback.py @@ -139,7 +139,8 @@ def test_generated_production_paths_read_complete_unordered_peer_rows( body = ('vpc_state_allowed_for_peer() { printf "10.44.0.11/32,10.44.0.99/32"; }\n' + body) elif operation == 'vpc_observe': - body = ('vpc_state_load() { VPC_STATE_STATUS=owned; }\n' + body = ('STATE="$VC_WG_MUTATIONS.state"\n' + 'vpc_state_load() { VPC_STATE_STATUS=owned; }\n' 'vpc_state_allowed_for_peer() { printf "10.44.0.11/32,10.44.0.99/32"; }\n' + body) result = run_shell(body, rows, mutation_file=mutations) @@ -196,7 +197,8 @@ def test_generated_paths_never_accept_partial_or_failed_readback( script = fabric._observe_script((), expected_digest=FABRIC_DIGEST) start = script.index(' $WG show "$IFACE" peers 2>/dev/null | while') end = script.index(' $IP -N -o route show', start) - body = ('vpc_state_load() { VPC_STATE_STATUS=owned; }\n' + body = ('STATE="$VC_WG_MUTATIONS.state"\n' + 'vpc_state_load() { VPC_STATE_STATUS=owned; }\n' 'vpc_state_allowed_for_peer() { printf "10.44.0.11/32,10.44.0.99/32"; }\n' + script[start:end]) result = run_shell(body, rows, mutation_file=mutations, diff --git a/engine/virsh_panel/coordination.py b/engine/virsh_panel/coordination.py index 708fbd9f..98db20ab 100644 --- a/engine/virsh_panel/coordination.py +++ b/engine/virsh_panel/coordination.py @@ -27,6 +27,7 @@ _TOKEN = re.compile(r"[A-Za-z0-9_-]{16,128}\Z") _KIND = re.compile(r"[a-z][a-z0-9_-]{0,63}\Z") _EXPLICIT_VPC_RECONCILE_METADATA = {"explicit_vpc_reconcile": True} _RESOLVE_AND_BIND_RECEIPT_KIND = "resolve_and_bind_v1" +_RESOLVE_FAMILY_AND_BIND_RECEIPT_KIND = "resolve_family_and_bind_v1" _UNKNOWN_CODES = frozenset({ "libvirt_execution_uncertain", "libvirt_execution_blocked", "coordination_uncertain", "operation_uncertain", @@ -296,6 +297,22 @@ class DurableCoordinator: return True, True return self._change(mutate) + def release_exact(self, claim: Mapping[str, Any]) -> bool: + """Release only an unchanged active durable snapshot.""" + if not isinstance(claim, Mapping): + raise ValueError("Exact coordination release snapshot is required") + expected = dict(claim) + _identity(expected) + + def mutate(document): + current = document["claims"].get(expected["token"]) + if current is None or current != expected or current["state"] != "active": + return False, False + del document["claims"][current["token"]] + return True, True + + return self._change(mutate) + def retain(self, claim: Claim | Mapping[str, Any], reason_code: str = "operation_uncertain") -> bool: if not isinstance(reason_code, str) or not _IDENTIFIER.fullmatch(reason_code): raise ValueError("Invalid coordination reason code") @@ -596,6 +613,9 @@ class DurableCoordinator: target_metadata = json.loads(_encode(dict(metadata))) if len(_encode(target_metadata)) > 8192: raise ValueError("Coordination metadata is too large") + source_resources = _resources(claim.get("resources", ())) + if list(source_resources) != claim.get("resources"): + raise ValueError("Invalid retained reconciliation resources") def replacement_value(name, default=None): if isinstance(replacement, Mapping): @@ -616,7 +636,6 @@ class DurableCoordinator: if (original is None or original != dict(claim) or original["state"] != "uncertain" or original["operation_id"] != operation_id - or original["resources"] != [] or original["maintenance"] or original["kind"] not in {"background", "request"} or original["metadata"] != target_metadata @@ -636,9 +655,12 @@ class DurableCoordinator: conflicts = [ existing for existing in document["claims"].values() if existing["token"] not in {original["token"], current["token"]} - and (existing["execution_id"] in { + and (existing["maintenance"] + or existing["execution_id"] in { original["execution_id"], current["execution_id"]} - or existing["operation_id"] == operation_id) + or existing["operation_id"] == operation_id + or bool(set(source_resources).intersection( + existing["resources"]))) ] if conflicts: unknown = any(existing["state"] == "uncertain" @@ -648,7 +670,7 @@ class DurableCoordinator: 503 if unknown else 409) now = int(time.time()) current.update( - operation_id=operation_id, resources=[], + operation_id=operation_id, resources=list(source_resources), metadata=json.loads(_encode(target_metadata)), updated_at=now) receipt = { "kind": _RESOLVE_AND_BIND_RECEIPT_KIND, @@ -668,11 +690,271 @@ class DurableCoordinator: del document["claims"][original["token"]] return True, Claim( current["token"], current["execution_id"], current["owner_id"], - operation_id, (), "request", + operation_id, source_resources, "request", + json.loads(_encode(target_metadata))) + + return self._change(mutate) + + @staticmethod + def _reconciliation_family( + claims, *, operation_id: str, + metadata: Mapping[str, Any]) -> tuple[dict[str, Any], ...]: + """Validate one exact retained execution family. + + A family contains one request/background root plus only resource + children created beneath that root. A root may already carry the + aggregate resource fence transferred by an earlier handoff. Every row + must carry the same immutable VPC target and already be uncertain. + Callers must still prove from durable host readback that the next pass + is observation-only before using this family handoff. + """ + if isinstance(claims, (str, bytes, Mapping)): + raise ValueError("Reconciliation family must be a sequence") + target_metadata = json.loads(_encode(dict(metadata))) + rows = tuple(dict(claim) for claim in claims) + if not rows: + raise ValueError("Reconciliation family is required") + for row in rows: + _identity(row) + tokens = [row["token"] for row in rows] + if len(tokens) != len(set(tokens)): + raise ValueError("Reconciliation family contains duplicate claims") + execution_ids = {row.get("execution_id") for row in rows} + owner_ids = {row.get("owner_id") for row in rows} + if len(execution_ids) != 1 or len(owner_ids) != 1: + raise ValueError("Reconciliation family identity is inconsistent") + roots = [ + row for row in rows + if row.get("kind") in {"background", "request"} + ] + if (len(roots) != 1 + or any(row.get("state") != "uncertain" + or row.get("operation_id") != operation_id + or row.get("maintenance") is not False + or row.get("metadata") != target_metadata + or not isinstance(row.get("resources"), list) + for row in rows) + or any(row is not roots[0] + and (row.get("kind") != "resource" + or not isinstance(row.get("resources"), list) + or not row["resources"]) + for row in rows)): + raise ValueError("Invalid retained reconciliation family") + return tuple(sorted(rows, key=lambda row: row["token"])) + + def resolve_family_and_bind( + self, claims, replacement: Claim | Mapping[str, Any], + evidence: str, *, operation_id: str, + metadata: Mapping[str, Any]) -> Claim: + """Atomically retire an exact uncertain execution family. + + This method does not itself authorize a retry. The VPC coordinator + calls it only after durable same-target evidence proves that the + immediate host pass can perform read-only observation. Exact family + matching prevents a partial or foreign claim set from being erased. + """ + if not isinstance(evidence, str) or not evidence.strip() or len(evidence) > 1024: + raise ValueError("Explicit reconciliation evidence is required") + if not isinstance(operation_id, str) or not _IDENTIFIER.fullmatch(operation_id): + raise ValueError("Invalid coordination operation") + if not isinstance(metadata, Mapping): + raise ValueError("Invalid coordination metadata") + target_metadata = json.loads(_encode(dict(metadata))) + if len(_encode(target_metadata)) > 8192: + raise ValueError("Coordination metadata is too large") + family = self._reconciliation_family( + claims, operation_id=operation_id, metadata=target_metadata) + replacement_identity = _identity(replacement) + + def replacement_value(name, default=None): + if isinstance(replacement, Mapping): + return replacement.get(name, default) + return getattr(replacement, name, default) + + if (replacement_identity[2] != self.owner_id + or replacement_value("operation_id") is not None + or tuple(replacement_value("resources", ())) != () + or replacement_value("kind") != "request" + or replacement_value("metadata", {}) + != _EXPLICIT_VPC_RECONCILE_METADATA): + raise ValueError("Invalid explicit reconciliation root") + + family_tokens = {row["token"] for row in family} + source_execution_id = family[0]["execution_id"] + source_family_sha256 = _digest(list(family)) + family_resources = _resources( + resource + for row in family + for resource in row["resources"] + ) + root = next(row for row in family + if row["kind"] in {"background", "request"}) + + def mutate(document): + current = document["claims"].get(replacement_identity[0]) + if (current is None + or (current["execution_id"], current["owner_id"]) + != replacement_identity[1:] + or current["owner_id"] != self.owner_id + or current["state"] != "active" + or current["maintenance"] + or current["kind"] != "request" + or current["operation_id"] is not None + or current["resources"] != [] + or current["metadata"] + != _EXPLICIT_VPC_RECONCILE_METADATA): + raise CoordinationClaimRejected("coordination_uncertain", 503) + if any(document["claims"].get(row["token"]) != row + for row in family): + raise CoordinationClaimRejected("coordination_uncertain", 503) + conflicts = [ + existing for existing in document["claims"].values() + if existing["token"] not in family_tokens | {current["token"]} + and (existing["maintenance"] + or existing["execution_id"] in { + source_execution_id, current["execution_id"]} + or existing["operation_id"] == operation_id + or bool(set(family_resources).intersection( + existing["resources"]))) + ] + if conflicts: + unknown = any(existing["state"] == "uncertain" + for existing in conflicts) + raise CoordinationClaimRejected( + "coordination_uncertain" if unknown else "coordination_busy", + 503 if unknown else 409) + now = int(time.time()) + current.update( + operation_id=operation_id, resources=list(family_resources), + metadata=json.loads(_encode(target_metadata)), updated_at=now) + receipt = { + "kind": _RESOLVE_FAMILY_AND_BIND_RECEIPT_KIND, + "token": root["token"], + "operation_id": operation_id, + "resolved_by": self.owner_id, + "resolved_at": now, + "evidence": evidence.strip(), + "source_execution_id": source_execution_id, + "source_family_size": len(family), + "source_family_sha256": source_family_sha256, + "replacement_token": current["token"], + "replacement_identity_sha256": _digest( + _claim_envelope(current)), + } + receipts = document.setdefault("resolutions", []) + receipts.append(receipt) + document["resolutions"] = receipts[-100:] + for token in family_tokens: + del document["claims"][token] + return True, Claim( + current["token"], current["execution_id"], current["owner_id"], + operation_id, family_resources, "request", json.loads(_encode(target_metadata))) return self._change(mutate) + def classify_resolve_family_and_bind( + self, claims, replacement: Claim | Mapping[str, Any], + evidence: str, *, operation_id: str, + metadata: Mapping[str, Any]) -> tuple[str, dict[str, Any] | None]: + """Classify a lost exact-family handoff response.""" + if not isinstance(evidence, str) or not evidence.strip() or len(evidence) > 1024: + raise ValueError("Explicit reconciliation evidence is required") + if not isinstance(operation_id, str) or not _IDENTIFIER.fullmatch(operation_id): + raise ValueError("Invalid coordination operation") + if not isinstance(metadata, Mapping): + raise ValueError("Invalid coordination metadata") + target_metadata = json.loads(_encode(dict(metadata))) + if len(_encode(target_metadata)) > 8192: + raise ValueError("Coordination metadata is too large") + family = self._reconciliation_family( + claims, operation_id=operation_id, metadata=target_metadata) + replacement_identity = _identity(replacement) + family_tokens = {row["token"] for row in family} + source_execution_id = family[0]["execution_id"] + source_family_sha256 = _digest(list(family)) + family_resources = _resources( + resource + for row in family + for resource in row["resources"] + ) + root = next(row for row in family + if row["kind"] in {"background", "request"}) + _raw, document = self._read() + current = document["claims"].get(replacement_identity[0]) + source_rows = { + token: document["claims"].get(token) for token in family_tokens} + anonymous = bool( + current is not None + and (current["execution_id"], current["owner_id"]) + == replacement_identity[1:] + and current["state"] == "active" + and current["maintenance"] is False + and current["operation_id"] is None + and current["resources"] == [] + and current["kind"] == "request" + and current["metadata"] == _EXPLICIT_VPC_RECONCILE_METADATA) + bound = bool( + current is not None + and (current["execution_id"], current["owner_id"]) + == replacement_identity[1:] + and current["state"] in {"active", "uncertain"} + and current["maintenance"] is False + and current["operation_id"] == operation_id + and current["resources"] == list(family_resources) + and current["kind"] == "request" + and current["metadata"] == target_metadata) + other_conflict = any( + existing["token"] not in family_tokens | {replacement_identity[0]} + and (existing["maintenance"] + or existing["execution_id"] in { + source_execution_id, replacement_identity[1]} + or existing["operation_id"] == operation_id + or bool(set(family_resources).intersection( + existing["resources"]))) + for existing in document["claims"].values()) + expected_receipt = { + "kind": _RESOLVE_FAMILY_AND_BIND_RECEIPT_KIND, + "token": root["token"], + "operation_id": operation_id, + "resolved_by": self.owner_id, + "evidence": evidence.strip(), + "source_execution_id": source_execution_id, + "source_family_size": len(family), + "source_family_sha256": source_family_sha256, + "replacement_token": replacement_identity[0], + "replacement_identity_sha256": ( + _digest(_claim_envelope(current)) if bound else None), + } + + def receipt_matches(receipt) -> bool: + return bool( + isinstance(receipt, dict) + and set(receipt) == {*expected_receipt, "resolved_at"} + and isinstance(receipt.get("resolved_at"), int) + and not isinstance(receipt.get("resolved_at"), bool) + and receipt["resolved_at"] > 0 + and all(receipt.get(key) == value + for key, value in expected_receipt.items())) + + related_receipts = [ + receipt for receipt in document.get("resolutions", []) + if (isinstance(receipt, dict) + and (receipt.get("source_execution_id") == source_execution_id + or receipt.get("replacement_token") + == replacement_identity[0])) + ] + matching_receipts = [receipt for receipt in related_receipts + if receipt_matches(receipt)] + if (all(row is None for row in source_rows.values()) + and bound and not other_conflict + and len(matching_receipts) == len(related_receipts) == 1): + return "committed", current + if (all(source_rows[row["token"]] == row for row in family) + and anonymous and not other_conflict and not related_receipts): + return "uncommitted", current + return "mixed", current + def classify_resolve_and_bind( self, claim: Mapping[str, Any], replacement: Claim | Mapping[str, Any], evidence: str, *, operation_id: str, @@ -695,6 +977,9 @@ class DurableCoordinator: target_metadata = json.loads(_encode(dict(metadata))) if len(_encode(target_metadata)) > 8192: raise ValueError("Coordination metadata is too large") + source_resources = _resources(claim.get("resources", ())) + if list(source_resources) != claim.get("resources"): + raise ValueError("Invalid retained reconciliation resources") def replacement_value(name, default=None): if isinstance(replacement, Mapping): @@ -714,9 +999,12 @@ class DurableCoordinator: current = document["claims"].get(replacement_identity[0]) other_conflict = any( existing["token"] not in {claim["token"], replacement_identity[0]} - and (existing["execution_id"] in { + and (existing["maintenance"] + or existing["execution_id"] in { claim["execution_id"], replacement_identity[1]} - or existing["operation_id"] == operation_id) + or existing["operation_id"] == operation_id + or bool(set(source_resources).intersection( + existing["resources"]))) for existing in document["claims"].values() ) anonymous = bool( @@ -736,7 +1024,7 @@ class DurableCoordinator: and current["state"] in {"active", "uncertain"} and current["maintenance"] is False and current["operation_id"] == operation_id - and current["resources"] == [] + and current["resources"] == list(source_resources) and current["kind"] == "request" and current["metadata"] == target_metadata) expected_receipt = { @@ -778,6 +1066,27 @@ class DurableCoordinator: return "uncommitted", current return "mixed", current + def classify_release( + self, claim: Mapping[str, Any]) -> tuple[str, dict[str, Any] | None]: + """Classify a lost exact release response without weakening identity. + + This helper intentionally accepts a complete durable snapshot rather + than a :class:`Claim`: only byte-for-byte survival proves that a failed + release did not commit. Absence proves the idempotent release result; + every changed row is mixed and must remain fenced. + """ + if not isinstance(claim, Mapping): + raise ValueError("Exact coordination release snapshot is required") + expected = dict(claim) + _identity(expected) + _raw, document = self._read() + current = document["claims"].get(expected["token"]) + if current is None: + return "committed", None + if current == expected and current.get("state") == "active": + return "uncommitted", current + return "mixed", current + def configure_default_coordinator(coordinator: DurableCoordinator | None) -> None: global _default_coordinator @@ -813,6 +1122,18 @@ def mark_remote_write_started() -> None: context.remote_write_started.set() +def mark_remote_write_confirmed() -> None: + """Retire confirmed write evidence at a quiescent mutation boundary. + + Callers may use this only after the remote write returned a strict receipt + or equivalent readback and no sibling mutation is still in flight. An + unknown execution remains sticky and can never be made retryable here. + """ + context = current_execution() + if context is not None and not context.execution_unknown.is_set(): + context.remote_write_started.clear() + + def mark_execution_unknown() -> None: """Never allow later no-write evidence to erase an ambiguous execution.""" context = current_execution() @@ -975,6 +1296,13 @@ def resource_guard(resources, timeout: float = 5.0, *, raise CoordinationError("coordination_uncertain", 503) from exc raise else: - if not coordinator.release(claim): + context = current_execution() + settled = ( + coordinator.retain(claim, "operation_uncertain") + if context is not None + and context.execution_unknown.is_set() + else coordinator.release(claim) + ) + if not settled: mark_execution_unknown() raise CoordinationError("coordination_uncertain", 503) diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index ed29d7a9..473b24e1 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -580,6 +580,9 @@ class _DurableAsyncLock(asyncio.Lock): context = current_execution() if context is not None and context.execution_unknown.is_set(): not_started = False + if not await self._retain_expected_claim(): + raise CoordinationError("coordination_unavailable", 503) + return if exc is not None and _uncertain_exception(exc) and not not_started: if not await self._retain_expected_claim(): raise CoordinationError("coordination_unavailable", 503) diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index 81dd2645..059a334a 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -1757,6 +1757,64 @@ class VMService: continue return sorted(rows, key=lambda row: row["name"].casefold()) + def require_vpc_members_stopped(self, vm_ids: Sequence[str]) -> None: + """Prove every VPC member is fully shut off without changing it. + + VPC membership changes rewrite boot-time seed data and persistent + domain XML. They must therefore enter the host transaction only after + a fresh libvirt read proves that every affected domain is in the + ``shutoff`` state. Keeping this check in the host service makes the + Engine boundary authoritative even if a control-plane power cache is + stale or a caller bypasses the PHP admission check. + """ + if (not isinstance(vm_ids, Sequence) + or isinstance(vm_ids, (str, bytes)) + or not 1 <= len(vm_ids) <= 100): + raise VMError("VPC 停机核对数量必须为 1 到 100", 400, + error_code="vpc_member_state_invalid") + canonical_ids: list[str] = [] + seen: set[str] = set() + for value in vm_ids: + canonical = self._uuid(value) + if canonical not in seen: + canonical_ids.append(canonical) + seen.add(canonical) + try: + with self._connect(timeout=LIBVIRT_READ_BUDGET) as conn: + for vm_id in canonical_ids: + domain = self._domain(conn, vm_id) + try: + active = bool(domain.isActive()) + state = STATES.get(domain.info()[0], "unknown") + managed_save = False + managed_save_reader = getattr( + domain, "hasManagedSaveImage", None) + if not callable(managed_save_reader): + raise RuntimeError( + "managed-save state is unavailable") + managed_save = bool(managed_save_reader(0)) + except Exception as exc: + raise VMError( + "无法确认 VPC 成员实例是否已完全关机", 502, + error_code="vpc_member_power_unavailable", + ) from exc + if active or state != "shutoff" or managed_save: + raise VMError( + "请先完全关闭实例,再修改 VPC 成员关系", 409, + error_code="vpc_member_requires_stopped", + ) + except VMError as exc: + if exc.error_code in { + "vpc_member_state_invalid", + "vpc_member_requires_stopped", + "vpc_member_power_unavailable", + }: + raise + raise VMError( + "无法确认 VPC 成员实例是否已完全关机", 502, + error_code="vpc_member_power_unavailable", + ) from exc + def list_vm_metadata(self, vm_ids: list[str]) -> list[dict[str, Any]]: """Fetch static metadata for at most 100 explicitly selected domains.""" if not isinstance(vm_ids, list) or not 1 <= len(vm_ids) <= 100: @@ -2833,6 +2891,228 @@ class VMService: == sorted(set(private_routes)) ) + @staticmethod + def _require_vpc_private_readback_restore_safe( + domain: Any, *, removing: bool, + ) -> None: + """Reject saved domain state that could restore stale private XML. + + Managed-save images and libvirt native snapshots may retain an older + domain definition. Persistent XML/seed readback is therefore not a + terminal configuration or absence proof while either restore source + exists. Panel live snapshots are disk-only backup volumes and do not + enter this libvirt metadata boundary. + """ + managed_save_reader = getattr(domain, "hasManagedSaveImage", None) + if not callable(managed_save_reader): + raise VMError( + "无法确认虚拟机托管保存状态", 502, + error_code="vpc_member_power_unavailable", + ) + try: + managed_save_state = managed_save_reader(0) + except Exception as exc: + raise VMError( + "无法确认虚拟机托管保存状态", 502, + error_code="vpc_member_power_unavailable", + ) from exc + if (isinstance(managed_save_state, bool) + or not isinstance(managed_save_state, int) + or managed_save_state not in {0, 1}): + raise VMError( + "无法确认虚拟机托管保存状态", 502, + error_code="vpc_member_power_unavailable", + ) + if managed_save_state == 1: + if removing: + raise VMError( + "虚拟机存在托管保存状态,无法确认 private0 已永久移除", + 409, + error_code="vpc_private_remove_unsafe", + ) + raise VMError( + "虚拟机存在托管保存状态,不能确认当前私网配置", + 409, + error_code="vpc_vm_managed_save_present", + ) + + snapshot_reader = getattr(domain, "snapshotNum", None) + if not callable(snapshot_reader): + raise VMError( + "无法确认虚拟机原生快照状态", 502, + error_code="vpc_vm_native_snapshot_state_unknown", + ) + try: + snapshot_count = snapshot_reader(0) + except Exception as exc: + raise VMError( + "无法确认虚拟机原生快照状态", 502, + error_code="vpc_vm_native_snapshot_state_unknown", + ) from exc + if (isinstance(snapshot_count, bool) + or not isinstance(snapshot_count, int) + or snapshot_count < 0): + raise VMError( + "无法确认虚拟机原生快照状态", 502, + error_code="vpc_vm_native_snapshot_state_unknown", + ) + if snapshot_count > 0: + if removing: + raise VMError( + "虚拟机存在原生快照,无法确认 private0 已永久移除", + 409, + error_code="vpc_private_remove_unsafe", + ) + raise VMError( + "虚拟机存在原生快照,不能确认当前私网配置", + 409, + error_code="vpc_vm_native_snapshot_present", + ) + + def read_private_configuration_unclaimed( + self, + uuid: str, + *, + private_mac: str, + private_ip_cidr: str, + private_routes: Sequence[str], + private_seed_identity: Mapping[str, Any], + deadline: float | None = None, + ) -> dict[str, Any]: + """Read exact persistent private configuration without a resource claim. + + This narrow observer is reserved for explicit reconciliation while an + older, exact VPC execution still owns the VM/storage claim family. It + performs no libvirt mutation, QGA command, ledger update, or + coordination settlement. The caller must validate the retained + operation and resource closure before invoking it. + """ + vm_uuid = self._uuid(uuid) + timeout = LIBVIRT_READ_BUDGET + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise VMError( + "VPC 持久配置回读超过请求预算", 502, + error_code="vpc_private_guest_probe_deadline_exceeded") + timeout = min(timeout, remaining) + with self._connect(timeout=timeout) as conn: + domain = self._domain(conn, vm_uuid) + if not domain.isPersistent(): + raise VMError("VPC 私网网卡只能接入持久化虚拟机", 409, + error_code="vpc_vm_not_persistent") + self._require_vpc_private_readback_restore_safe( + domain, removing=False) + inactive = domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE) + running = bool(domain.isActive()) + live = domain.XMLDesc(0) if running else None + seed_verified = self._verify_private_seed_readback( + conn, domain, inactive_xml=inactive, + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + identity=private_seed_identity, + ) + return { + "inactive": inactive, + "live": self._without_qga_runtime_metadata(live), + "seed_verified": seed_verified, + "guest_network": { + "state": "unavailable" if running else "stopped", + "verified": False, + "reason_code": ( + "vpc_private_guest_agent_unavailable" if running + else "vpc_private_guest_stopped"), + "observed_at": int(time.time()), + }, + } + + def read_private_absence_unclaimed( + self, + uuid: str, + *, + deadline: float | None = None, + ) -> dict[str, Any]: + """Prove private0, private metadata and seed intent are all absent. + + Like :meth:`read_private_configuration_unclaimed`, this method is a + strict read-only escape hatch for an exact retained VPC claim family. + Unknown or unowned CD-ROM state is not treated as absence. + """ + vm_uuid = self._uuid(uuid) + timeout = LIBVIRT_READ_BUDGET + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise VMError( + "VPC 私网移除回读超过请求预算", 502, + error_code="vpc_private_guest_probe_deadline_exceeded") + timeout = min(timeout, remaining) + + def xml_absent(xml: str) -> tuple[bool, ET.Element]: + try: + root = ET.fromstring(xml) + private = self._private_interface_from_root(root) + except (ET.ParseError, VMError): + return False, ET.Element("invalid") + managed = root.find(f"./metadata/{{{PANEL_NS}}}managed") + private_fields = ( + [key for key in managed.attrib if key.startswith("private-")] + if managed is not None else []) + return private is None and not private_fields, root + + with self._connect(timeout=timeout) as conn: + domain = self._domain(conn, vm_uuid) + if not domain.isPersistent(): + raise VMError("VPC 私网网卡只能接入持久化虚拟机", 409, + error_code="vpc_vm_not_persistent") + self._require_vpc_private_readback_restore_safe( + domain, removing=True) + inactive = domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE) + inactive_absent, inactive_root = xml_absent(inactive) + running = bool(domain.isActive()) + live = domain.XMLDesc(0) if running else None + live_absent = True + if live is not None: + live_absent, _live_root = xml_absent(live) + seed_absent = False + try: + seed = self._owned_seed_volume( + conn, domain, root=inactive_root, + mismatch_error_code="vpc_seed_identity_mismatch") + except VMError as exc: + if exc.error_code == "vpc_seed_identity_mismatch": + seed = None + else: + raise + if seed is None: + # Without a panel-owned seed identity, an arbitrary CD-ROM + # could still carry private0 network intent. Only a domain + # with no CD-ROM is a positive absence proof. + seed_absent = not inactive_root.findall( + "./devices/disk[@device='cdrom']") + else: + try: + network = read_seed_network(self._download_seed_volume( + conn, seed, + invalid_error_code="vpc_seed_readback_invalid")) + except (KeyError, TypeError, ValueError): + network = {} + seed_absent = bool(network) and not any( + key in network for key in ( + "private_ip_cidr", "private_mac", "private_routes")) + seed_absent = seed_absent and ( + "private0" not in network.get("roles", [])) + return { + "removed": bool(inactive_absent and live_absent and seed_absent), + "inactive": inactive, + "live": self._without_qga_runtime_metadata(live), + "seed_verified": seed_absent, + "running": running, + "readback": "confirmed" if ( + inactive_absent and live_absent and seed_absent) else "mismatch", + } + def read_private_domain_pair( self, uuid: str, @@ -3239,23 +3519,21 @@ class VMService: str(exc), exc.status, error_code=exc.error_code) from exc with ( - self._operation_guard(vm_uuid, storage=True), + # This nested VPC mutation has its own precise libvirt write + # marker. A deterministic snapshot/ownership failure before the + # first RPC write must release this VM/storage claim even though + # an earlier host in the same VPC revision already completed a + # separately confirmed SSH transaction. + self._operation_guard( + vm_uuid, storage=True, tracked_mutations=True), self._connect(timeout=LIBVIRT_MUTATION_BUDGET) as conn, ): domain = self._domain(conn, vm_uuid) if not domain.isPersistent(): raise VMError("VPC 私网网卡只能接入持久化虚拟机", 409, error_code="vpc_vm_not_persistent") - if getattr(domain, "hasManagedSaveImage", lambda _flags: 0)(0): - raise VMError("虚拟机存在托管保存状态,请先处理后再接入私网", 409, - error_code="vpc_vm_managed_save_present") - try: - if int(getattr(domain, "snapshotNum", lambda _flags: 0)(0)) > 0: - raise VMError("虚拟机存在原生快照,请先处理后再接入私网", 409, - error_code="vpc_vm_native_snapshot_present") - except (TypeError, ValueError) as exc: - raise VMError("无法确认虚拟机原生快照状态", 502, - error_code="vpc_vm_native_snapshot_state_unknown") from exc + self._require_vpc_private_readback_restore_safe( + domain, removing=False) old_inactive_xml = domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE) old_root = ET.fromstring(old_inactive_xml) @@ -3789,16 +4067,16 @@ class VMService: raise VMError("VPC 成员私网 IP 无效", 422, error_code="vpc_invalid") from exc with ( - self._operation_guard(vm_uuid, storage=True), + self._operation_guard( + vm_uuid, storage=True, tracked_mutations=True), self._connect(timeout=LIBVIRT_MUTATION_BUDGET) as conn, ): domain = self._domain(conn, vm_uuid) - if not domain.isPersistent() or getattr(domain, "hasManagedSaveImage", lambda _flags: 0)(0): + if not domain.isPersistent(): raise VMError("虚拟机状态不允许移除 private0", 409, error_code="vpc_private_remove_unsafe") - if int(getattr(domain, "snapshotNum", lambda _flags: 0)(0)) > 0: - raise VMError("虚拟机存在原生快照,无法移除 private0", 409, - error_code="vpc_private_remove_unsafe") + self._require_vpc_private_readback_restore_safe( + domain, removing=True) if domain.isActive(): return self._confirm_running_private_absence( conn, domain, expected_bridge=expected_bridge, diff --git a/engine/virsh_panel/vpc.py b/engine/virsh_panel/vpc.py index d1b39da2..13f8d4c6 100644 --- a/engine/virsh_panel/vpc.py +++ b/engine/virsh_panel/vpc.py @@ -25,7 +25,8 @@ from typing import Any, Collection, Mapping, Sequence from .coordination import ( CoordinationAdmissionRejected, CoordinationClaimRejected, CoordinationError, - current_execution, execution_context, _uncertain_exception, + current_execution, execution_context, mark_remote_write_confirmed, + resource_guard, _uncertain_exception, ) from .fabric import ( FabricCoordinator, @@ -58,6 +59,8 @@ VPC_GUEST_PROBE_WORKERS = 4 _VPC_GUEST_PROBE_SLOTS = threading.BoundedSemaphore(VPC_GUEST_PROBE_WORKERS) VPC_GUEST_EXEC_KIND = "vpc_guest_exec" VPC_GUEST_EXEC_MAX_BYTES = 1024 +VPC_CLAIM_FAMILY_RECOVERY_VERSION = 1 +VPC_GUEST_EXEC_CLEANUP_VERSION = 1 # PHP gives this endpoint 60 seconds and holds the operation lease for 120 # seconds. This monotonic budget stops admission of further hosts/QGA calls at # 40 seconds, leaving 20 seconds for response/transport and another 60 seconds @@ -68,6 +71,8 @@ VPC_REQUEST_DEADLINE_SECONDS = 40.0 VPC_STATES = frozenset({"applying", "active", "degraded", "uncertain"}) VPC_DELETE_RECEIPT_KIND = "vpc_deleted" VPC_DELETE_DETERMINISTIC_HOST_ERRORS = frozenset({ + "vpc_member_requires_stopped", + "vpc_member_power_unavailable", "vpc_private_remove_requires_stopped", "vpc_private_remove_unsafe", "vpc_private_interface_conflict", @@ -133,6 +138,10 @@ class VpcError(Exception): self.error_code = error_code +class _VpcMemberPreflightError(VpcError): + """A stop-state refusal proven before any VPC host mutation began.""" + + def _reinstall_job_committed(job: Mapping[str, Any]) -> bool: """Whether a durable reinstall row proves the replacement boundary.""" if job.get("reinstall_committed") is True: @@ -1067,6 +1076,573 @@ class VpcCoordinator: and durable.desired_digest == plan.desired_digest and durable.canonical() == plan.canonical()) + @staticmethod + def _retained_family_hash( + family: Sequence[Mapping[str, Any]], + ) -> str: + """Hash the complete canonical retained family, including claim state.""" + canonical = sorted( + (dict(row) for row in family), + key=lambda row: str(row.get("token", "")), + ) + encoded = json.dumps( + canonical, ensure_ascii=True, sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + @classmethod + def _retained_family_recovery_marker( + cls, plan: VpcPlan, family: Sequence[Mapping[str, Any]], + ) -> dict[str, Any]: + execution_ids = { + row.get("execution_id") for row in family + if isinstance(row.get("execution_id"), str) + } + if len(execution_ids) != 1 or len(family) < 2: + raise VpcError("VPC 保留任务闭集无效", 500, + "vpc_record_corrupt") + return { + "version": VPC_CLAIM_FAMILY_RECOVERY_VERSION, + "operation_id": plan.operation_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "source_execution_id": next(iter(execution_ids)), + "source_family_size": len(family), + "source_family_sha256": cls._retained_family_hash(family), + "observed_at": int(time.time()), + } + + @classmethod + def _retained_family_recovery_marker_matches( + cls, plan: VpcPlan, record: Mapping[str, Any], + family: Sequence[Mapping[str, Any]], + ) -> bool: + marker = record.get("claim_family_recovery") + if not isinstance(marker, Mapping) or set(marker) != { + "version", "operation_id", "desired_revision", "desired_digest", + "source_execution_id", "source_family_size", + "source_family_sha256", "observed_at", + }: + return False + execution_ids = { + row.get("execution_id") for row in family + if isinstance(row.get("execution_id"), str) + } + observed_at = marker.get("observed_at") + family_size = marker.get("source_family_size") + return bool( + marker.get("version") == VPC_CLAIM_FAMILY_RECOVERY_VERSION + and marker.get("operation_id") == plan.operation_id + and marker.get("desired_revision") == plan.desired_revision + and marker.get("desired_digest") == plan.desired_digest + and len(execution_ids) == 1 + and marker.get("source_execution_id") + == next(iter(execution_ids)) + and isinstance(family_size, int) + and not isinstance(family_size, bool) + and family_size == len(family) + and 2 <= family_size <= (VPC_MAX_MEMBERS * 2 + VPC_MAX_HOSTS + 2) + and isinstance(marker.get("source_family_sha256"), str) + and _DIGEST.fullmatch(marker["source_family_sha256"]) is not None + and marker["source_family_sha256"] + == cls._retained_family_hash(family) + and isinstance(observed_at, int) + and not isinstance(observed_at, bool) + and observed_at > 0 + ) + + @classmethod + def _guest_exec_cleanup_marker_targets( + cls, plan: VpcPlan, marker: Any, + ) -> tuple[dict[str, Any], ...] | None: + """Validate one durable post-handoff ledger cleanup intent. + + The marker is written while the complete old claim family still owns + every retired VM resource. It deliberately contains only identity and + a bounded digest of the ledger; the ledger is deleted later, under the + replacement root and a fresh VM claim, after another strict stopped + absence readback. + """ + if not isinstance(marker, Mapping) or set(marker) != { + "version", "operation_id", "desired_revision", "desired_digest", + "source_execution_id", "source_family_size", + "source_family_sha256", "targets", "created_at", + }: + return None + family_size = marker.get("source_family_size") + created_at = marker.get("created_at") + if not ( + marker.get("version") == VPC_GUEST_EXEC_CLEANUP_VERSION + and marker.get("operation_id") == plan.operation_id + and marker.get("desired_revision") == plan.desired_revision + and marker.get("desired_digest") == plan.desired_digest + and isinstance(marker.get("source_execution_id"), str) + and _DIGEST.fullmatch(marker["source_execution_id"]) is not None + and isinstance(family_size, int) + and not isinstance(family_size, bool) + and 2 <= family_size <= (VPC_MAX_MEMBERS * 2 + VPC_MAX_HOSTS + 2) + and isinstance(marker.get("source_family_sha256"), str) + and _DIGEST.fullmatch(marker["source_family_sha256"]) is not None + and isinstance(created_at, int) + and not isinstance(created_at, bool) + and created_at > 0 + and isinstance(marker.get("targets"), list) + and 1 <= len(marker["targets"]) <= VPC_MAX_MEMBERS + ): + return None + + active_uuids = {member.instance_uuid for member in plan.members} + normalized: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + try: + for raw in marker["targets"]: + if not isinstance(raw, Mapping) or set(raw) != { + "host_id", "instance_uuid", "member_id", "ledger_key", + "ledger_sha256", "ledger_bytes", + }: + return None + host_id = _host_id(raw.get("host_id")) + instance_uuid = _uuid(raw.get("instance_uuid")) + member_id = _identifier(raw.get("member_id"), "VPC member") + ledger_key = raw.get("ledger_key") + ledger_sha256 = raw.get("ledger_sha256") + ledger_bytes = raw.get("ledger_bytes") + if (instance_uuid in active_uuids + or ledger_key != cls._key(plan.vpc_id, instance_uuid) + or not isinstance(ledger_sha256, str) + or _DIGEST.fullmatch(ledger_sha256) is None + or isinstance(ledger_bytes, bool) + or not isinstance(ledger_bytes, int) + or not 1 <= ledger_bytes <= VPC_GUEST_EXEC_MAX_BYTES + or (host_id, instance_uuid) in seen): + return None + seen.add((host_id, instance_uuid)) + normalized.append({ + "host_id": host_id, + "instance_uuid": instance_uuid, + "member_id": member_id, + "ledger_key": ledger_key, + "ledger_sha256": ledger_sha256, + "ledger_bytes": ledger_bytes, + }) + except VpcError: + return None + normalized.sort(key=lambda item: ( + item["host_id"], item["instance_uuid"], item["member_id"])) + if list(marker["targets"]) != normalized: + return None + return tuple(normalized) + + def _persist_retained_guest_exec_cleanup_marker( + self, plan: VpcPlan, family: Sequence[Mapping[str, Any]], + ) -> bool: + """Capture stale retired-member ledgers before replacing a family.""" + original_raw = self.store.get("vpc_desired", plan.vpc_id) + if not isinstance(original_raw, str): + return False + try: + record = json.loads(original_raw) + durable = _plan_from_record(record) + except (TypeError, ValueError, VpcError): + return False + if (not isinstance(record, dict) + or durable.operation_id != plan.operation_id + or durable.desired_revision != plan.desired_revision + or durable.desired_digest != plan.desired_digest + or durable.canonical() != plan.canonical()): + return False + + existing = record.get("guest_exec_cleanup_recovery") + if existing is not None: + return self._guest_exec_cleanup_marker_targets( + plan, existing) is not None + + execution_ids = { + row.get("execution_id") for row in family + if isinstance(row.get("execution_id"), str) + } + if len(family) < 2 or len(execution_ids) != 1: + return False + active_uuids = {member.instance_uuid for member in plan.members} + candidates: set[tuple[str, str]] = set() + try: + for row in family: + resources = row.get("resources") + if not isinstance(resources, list): + return False + for resource in resources: + if (not isinstance(resource, str) + or not resource.startswith("vm:")): + continue + parts = resource.split(":", 2) + if len(parts) != 3: + return False + host_id = _host_id(parts[1]) + instance_uuid = _uuid(parts[2]) + if instance_uuid not in active_uuids: + candidates.add((host_id, instance_uuid)) + except VpcError: + return False + + targets: list[dict[str, Any]] = [] + for host_id, instance_uuid in sorted(candidates): + key = self._key(plan.vpc_id, instance_uuid) + raw = self.store.get(VPC_GUEST_EXEC_KIND, key) + if raw is None: + continue + try: + _state, base = self._decode_guest_exec_raw( + raw, expected_vpc_id=plan.vpc_id, + expected_host_id=host_id, + expected_instance_uuid=instance_uuid, + ) + except VpcError: + return False + encoded = raw.encode("utf-8") + targets.append({ + "host_id": host_id, + "instance_uuid": instance_uuid, + "member_id": base["member_id"], + "ledger_key": key, + "ledger_sha256": hashlib.sha256(encoded).hexdigest(), + "ledger_bytes": len(encoded), + }) + if not targets: + return True + + marker = { + "version": VPC_GUEST_EXEC_CLEANUP_VERSION, + "operation_id": plan.operation_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "source_execution_id": next(iter(execution_ids)), + "source_family_size": len(family), + "source_family_sha256": self._retained_family_hash(family), + "targets": targets, + "created_at": int(time.time()), + } + if self._guest_exec_cleanup_marker_targets(plan, marker) is None: + return False + candidate = dict(record) + candidate["guest_exec_cleanup_recovery"] = marker + candidate_raw = json.dumps( + candidate, ensure_ascii=True, separators=(",", ":")) + compare_and_set = getattr(self.store, "compare_and_set", None) + if not callable(compare_and_set): + return False + try: + committed = compare_and_set( + "vpc_desired", plan.vpc_id, original_raw, candidate_raw) + except BaseException: + if self.store.get("vpc_desired", plan.vpc_id) != candidate_raw: + raise + else: + if not committed: + return False + return self.store.get("vpc_desired", plan.vpc_id) == candidate_raw + + def _retained_family_covers_vm_closure( + self, record: Mapping[str, Any], plan: VpcPlan, + family: Sequence[Mapping[str, Any]], + ) -> bool: + """Require VM fences for every identity this recovery may still touch.""" + covered: set[str] = set() + for row in family: + resources = row.get("resources") + if not isinstance(resources, list): + return False + for resource in resources: + if isinstance(resource, str) and resource.startswith("vm:"): + covered.add(resource) + required: set[str] = set() + try: + hosts = self._host_closure(record) + wanted = { + host_id for host_id, host in hosts.items() + if not ( + self._confirmed_removal( + host, plan.desired_revision, plan.desired_digest) + or (host.get("retiring") is not True + and self._confirmed_active_host( + host, plan, allow_pending=True)) + ) + } + required.update( + f"vm:{member.host_id}:{member.instance_uuid}" + for member in plan.members if member.host_id in wanted + ) + for host_id, host in hosts.items(): + raw_retiring = host.get("retiring_members", []) + if not isinstance(raw_retiring, list): + return False + for raw in raw_retiring: + if not isinstance(raw, Mapping): + return False + instance_uuid = _uuid(raw.get("instance_uuid")) + if host_id in wanted: + required.add(f"vm:{host_id}:{instance_uuid}") + marker = self._delete_marker(record) + if marker is not None: + for raw in marker["retiring_members"]: + if not isinstance(raw, Mapping): + return False + host_id = _host_id(raw.get("host_id")) + instance_uuid = _uuid(raw.get("instance_uuid")) + if host_id in wanted: + required.add(f"vm:{host_id}:{instance_uuid}") + cleanup_marker = record.get("guest_exec_cleanup_recovery") + if cleanup_marker is not None: + cleanup_targets = self._guest_exec_cleanup_marker_targets( + plan, cleanup_marker) + if cleanup_targets is None: + return False + for item in cleanup_targets: + if self.store.get( + VPC_GUEST_EXEC_KIND, + item["ledger_key"]) is not None: + required.add( + f"vm:{item['host_id']}:" + f"{item['instance_uuid']}") + except (KeyError, TypeError, VpcError): + return False + return required.issubset(covered) + + def _retained_family_readback_safe(self, plan: VpcPlan) -> bool: + """Prove that an uncertain claim family can enter a read-only pass. + + Removing resource children is safe only when the durable VPC record + already carries exact same-target configuration or absence evidence. + The normal retry will then take ``_runtime_readback_eligible`` (or + skip an exact removal), so the handoff cannot replay a possibly + accepted host/libvirt write. + """ + receipt = self._load_delete_receipt(plan.vpc_id) + if receipt is not None: + return bool( + receipt["operation_id"] == plan.operation_id + and receipt["desired_revision"] == plan.desired_revision + and receipt["desired_digest"] == plan.desired_digest) + record = self._load("vpc_desired", plan.vpc_id) + if record is None: + return False + return self._retained_record_readback_safe(plan, record) + + def _retained_record_readback_safe( + self, plan: VpcPlan, record: Mapping[str, Any], + ) -> bool: + """Validate exact configuration/absence proof in one loaded record.""" + durable = _plan_from_record(record) + if not ( + durable.operation_id == plan.operation_id + and durable.desired_revision == plan.desired_revision + and durable.desired_digest == plan.desired_digest + and durable.canonical() == plan.canonical() + ): + return False + hosts = self._host_closure(record) + marker = self._delete_marker(record) + if marker is not None: + required = set(marker["required_host_ids"]) + return bool( + required.issubset(hosts) + and all(self._confirmed_removal( + hosts[host_id], plan.desired_revision, + plan.desired_digest) for host_id in required)) + desired_host_ids = {host.host_id for host in plan.hosts} + member_host_ids = {member.host_id for member in plan.members} + if not desired_host_ids.issubset(hosts): + return False + return all( + self._runtime_readback_eligible(host, plan, host_id) + if host_id in member_host_ids else self._confirmed_removal( + host, plan.desired_revision, plan.desired_digest) + for host_id, host in hosts.items() + ) + + def _retained_family_resources_match( + self, plan: VpcPlan, family: Sequence[Mapping[str, Any]], + ) -> bool: + """Accept only resource children produced by this durable VPC target. + + An execution id and operation id identify one request, but they do not + prove that every child belongs to VPC reconciliation. In particular, + deleting an arbitrary same-execution child while taking over a retained + family could erase the uncertainty fence for another VM or allocator + operation. Derive the narrow resource closure from the durable target + and accept only the lock shapes used by the VPC path today. + """ + receipt = self._load_delete_receipt(plan.vpc_id) + if receipt is not None: + # The terminal receipt is written before the desired row is + # purged. A first family handoff may already have aggregated its + # exact VM/storage/catalog fences onto the request root, so a later + # recovery must accept those inherited fences as well as the one + # outer catalog child that can lose its release response. + if (receipt["operation_id"] != plan.operation_id + or receipt["desired_revision"] != plan.desired_revision + or receipt["desired_digest"] != plan.desired_digest): + return False + roots = [ + row for row in family + if row.get("kind") in {"background", "request"} + ] + if len(roots) != 1: + return False + root = roots[0] + root_resources = root.get("resources") + if (not isinstance(root_resources, list) + or root_resources and root.get("kind") != "request"): + return False + removed_hosts = set(receipt["removed_host_ids"]) + root_vms_by_host: dict[str, int] = {} + for resource in root_resources: + if resource == "catalog:network": + continue + if not isinstance(resource, str): + return False + parts = resource.split(":", 2) + if len(parts) == 3 and parts[0] == "vm": + try: + host_id = _host_id(parts[1]) + _uuid(parts[2]) + except VpcError: + return False + if host_id not in removed_hosts: + return False + root_vms_by_host[host_id] = ( + root_vms_by_host.get(host_id, 0) + 1) + continue + if (len(parts) == 2 and parts[0] == "storage" + and parts[1] in removed_hosts): + continue + return False + if any( + resource.startswith("storage:") + and root_vms_by_host.get(resource.split(":", 1)[1], 0) < 1 + for resource in root_resources + if isinstance(resource, str) + ): + return False + catalog_children = 0 + for row in family: + if row is root: + continue + if (row.get("kind") != "resource" + or row.get("resources") != ["catalog:network"]): + return False + catalog_children += 1 + return bool( + catalog_children <= 1 + and ("catalog:network" in root_resources + or catalog_children == 1)) + record = self._load("vpc_desired", plan.vpc_id) + if record is None: + return False + try: + durable = _plan_from_record(record) + if not ( + durable.operation_id == plan.operation_id + and durable.desired_revision == plan.desired_revision + and durable.desired_digest == plan.desired_digest + and durable.canonical() == plan.canonical() + ): + return False + hosts = self._host_closure(record) + if self._retained_family_recovery_marker_matches( + plan, record, family): + return True + identities = { + (member.host_id, member.instance_uuid) + for member in durable.members + } + for host_id, host in hosts.items(): + raw_retiring = host.get("retiring_members", []) + if (not isinstance(raw_retiring, list) + or len(raw_retiring) > VPC_MAX_MEMBERS): + return False + for raw in raw_retiring: + if not isinstance(raw, Mapping): + return False + member_host_id = _host_id(raw.get("host_id")) + member_uuid = _uuid(raw.get("instance_uuid")) + if member_host_id != host_id: + return False + identities.add((member_host_id, member_uuid)) + marker = self._delete_marker(record) + if marker is not None: + for raw in marker["retiring_members"]: + if not isinstance(raw, Mapping): + return False + member_host_id = _host_id(raw.get("host_id")) + member_uuid = _uuid(raw.get("instance_uuid")) + if member_host_id not in hosts: + return False + identities.add((member_host_id, member_uuid)) + cleanup_marker = record.get("guest_exec_cleanup_recovery") + if cleanup_marker is not None: + cleanup_targets = self._guest_exec_cleanup_marker_targets( + plan, cleanup_marker) + if cleanup_targets is None: + return False + identities.update( + (item["host_id"], item["instance_uuid"]) + for item in cleanup_targets + ) + except (KeyError, TypeError, VpcError): + return False + + allowed = {("catalog:network",)} + for host_id, instance_uuid in identities: + vm_resource = f"vm:{host_id}:{instance_uuid}" + allowed.add((vm_resource,)) + allowed.add((f"storage:{host_id}", vm_resource)) + allowed_resources = { + resource for resources in allowed for resource in resources + } + + roots = [ + row for row in family + if row.get("kind") in {"background", "request"} + ] + if len(roots) != 1: + return False + root = roots[0] + root_resources = root.get("resources") + if (not isinstance(root_resources, list) + or root_resources and root.get("kind") != "request" + or any(not isinstance(resource, str) + or resource not in allowed_resources + for resource in root_resources)): + return False + root_resource_set = set(root_resources) + if any( + resource.startswith("storage:") + and not any( + candidate.startswith(f"vm:{resource.split(':', 1)[1]}:") + for candidate in root_resource_set) + for resource in root_resources + ): + return False + catalog_children = 0 + for row in family: + if row is root: + continue + resources = row.get("resources") + if not isinstance(resources, list): + return False + canonical = tuple(resources) + if canonical not in allowed: + return False + if canonical == ("catalog:network",): + catalog_children += 1 + # The host loop catches a nested VM/seed failure and records it on the + # durable host row. The outer catalog child can therefore release + # normally while the nested VM or VM+storage child stays uncertain. + # At most one catalog child is valid; VM-only resource children may + # legitimately repeat at nested lock boundaries. + return catalog_children <= 1 + @asynccontextmanager async def _operation_scope(self, plan: VpcPlan): """Carry one validated target through HTTP and periodic resource claims.""" @@ -1081,9 +1657,15 @@ class VpcCoordinator: f"vpc-explicit-reconcile-v1 operation={plan.operation_id} " f"vpc={plan.vpc_id} revision={plan.desired_revision} " f"digest={plan.desired_digest}; exact childless retained root") + family_handoff_evidence = ( + f"vpc-explicit-family-reconcile-v1 operation={plan.operation_id} " + f"vpc={plan.vpc_id} revision={plan.desired_revision} " + f"digest={plan.desired_digest}; exact readback-safe retained family") claim = None background_execution_id = None handoff_original = None + handoff_family = None + handoff_resources: tuple[str, ...] = () handoff_attempted = False explicit_bind_attempted = False @@ -1097,19 +1679,22 @@ class VpcCoordinator: return False expected_metadata = metadata if bound else explicit_marker expected_operation = plan.operation_id if bound else None + expected_resources = ( + list(handoff_resources) + if bound and handoff_attempted else []) return bool( record.get("token") == parent.claim.token and record.get("execution_id") == parent.claim.execution_id and record.get("owner_id") == parent.claim.owner_id and record.get("operation_id") == expected_operation - and record.get("resources") == [] + and record.get("resources") == expected_resources and record.get("kind") == "request" and record.get("metadata") == expected_metadata and record.get("maintenance") is False and record.get("state") in {"active", "uncertain"}) async def retained_root_for_explicit_reconcile(): - """Return only one exact, childless VPC root eligible for handoff.""" + """Return one exact VPC root and its canonical retained family.""" if parent is None: return None marker_present = "explicit_vpc_reconcile" in parent.claim.metadata @@ -1135,34 +1720,125 @@ class VpcCoordinator: and row.get("state") == "uncertain" and row.get("kind") in {"background", "request"} and row.get("operation_id") == plan.operation_id - and row.get("resources") == [] + and isinstance(row.get("resources"), list) and row.get("maintenance") is False and row.get("metadata") == metadata): eligible.append(row) if len(eligible) != 1: return None original = eligible[0] - if any( - row.get("token") not in {original["token"], parent.claim.token} - and (row.get("execution_id") in { - original["execution_id"], parent.claim.execution_id} - or row.get("operation_id") == plan.operation_id) - for row in rows.values() - ): + family = tuple(sorted(( + row for row in rows.values() + if row.get("execution_id") == original["execution_id"] + ), key=lambda row: row.get("token", ""))) + family_tokens = {row.get("token") for row in family} + if (not family + or any( + row.get("operation_id") != plan.operation_id + or row.get("metadata") != metadata + or row.get("maintenance") is not False + or row.get("state") != "uncertain" + or (row.get("token") != original["token"] + and (row.get("kind") != "resource" + or not isinstance(row.get("resources"), list) + or not row["resources"])) + for row in family) + or any( + row.get("token") not in ( + family_tokens | {parent.claim.token}) + and (row.get("execution_id") + == parent.claim.execution_id + or row.get("operation_id") == plan.operation_id) + for row in rows.values())): return None - return original + if len(family) > 1 or original.get("resources"): + if not self._retained_family_resources_match(plan, family): + return None + if not self._retained_family_readback_safe(plan): + if len(family) == 1: + return None + original_hash = self._retained_family_hash(family) + if not await self._recover_retained_family_readback( + plan, family): + return None + # The proof and marker live in vpc_desired while the + # claims live in the coordination CAS document. Re-read + # both stores and require the complete old family to be + # byte-for-byte the one that was observed before handing + # its fences to this request. + rows = await _await_mutation_completion( + asyncio.to_thread(coordinator.snapshot)) + current = rows.get(parent.claim.token) + if (not exact_parent(current, bound=False) + or current["state"] != "active"): + raise CoordinationError( + "coordination_unavailable", 503) + refreshed = tuple(sorted(( + row for row in rows.values() + if row.get("execution_id") + == original["execution_id"] + ), key=lambda row: row.get("token", ""))) + if (not refreshed + or self._retained_family_hash(refreshed) + != original_hash): + return None + refreshed_tokens = { + row.get("token") for row in refreshed + } + if any( + row.get("token") not in ( + refreshed_tokens | {parent.claim.token}) + and (row.get("execution_id") + == parent.claim.execution_id + or row.get("operation_id") == plan.operation_id) + for row in rows.values() + ): + return None + committed = self._load("vpc_desired", plan.vpc_id) + if (committed is None + or not self._retained_family_recovery_marker_matches( + plan, committed, refreshed) + or not self._retained_family_resources_match( + plan, refreshed) + or not self._retained_family_readback_safe(plan)): + return None + family = refreshed + original = next( + row for row in family + if row.get("token") == original["token"]) + if (len(family) > 1 + and self._load_delete_receipt(plan.vpc_id) is None + and not await _await_mutation_completion( + asyncio.to_thread( + self._persist_retained_guest_exec_cleanup_marker, + plan, family))): + return None + return original, family async def admit(): nonlocal claim, background_execution_id, handoff_attempted - nonlocal handoff_original, explicit_bind_attempted + nonlocal handoff_original, handoff_family, explicit_bind_attempted + nonlocal handoff_resources if parent is not None: - handoff_original = await retained_root_for_explicit_reconcile() - if handoff_original is not None: + retained = await retained_root_for_explicit_reconcile() + if retained is not None: + handoff_original, handoff_family = retained + handoff_resources = tuple(sorted({ + resource + for row in handoff_family + for resource in row.get("resources", []) + })) handoff_attempted = True - claim = await asyncio.to_thread( - coordinator.resolve_and_bind, handoff_original, parent.claim, - handoff_evidence, - operation_id=plan.operation_id, metadata=metadata) + if len(handoff_family) == 1: + claim = await asyncio.to_thread( + coordinator.resolve_and_bind, handoff_original, + parent.claim, handoff_evidence, + operation_id=plan.operation_id, metadata=metadata) + else: + claim = await asyncio.to_thread( + coordinator.resolve_family_and_bind, handoff_family, + parent.claim, family_handoff_evidence, + operation_id=plan.operation_id, metadata=metadata) return explicit_bind_attempted = parent.claim.metadata == explicit_marker binder = (coordinator.bind_explicit_operation @@ -1236,6 +1912,34 @@ class VpcCoordinator: return raise CoordinationError("coordination_uncertain", 503) + async def release_anonymous_parent_exact( + expected: Mapping[str, Any]) -> None: + """Remove a proven-unbound explicit root, classifying lost CAS replies.""" + snapshot = dict(expected) + if (not exact_parent(snapshot, bound=False) + or snapshot.get("state") != "active"): + raise CoordinationError("coordination_unavailable", 503) + last_error: BaseException | None = None + attempts = max(2, int(getattr(coordinator, "cas_attempts", 2))) + for _attempt in range(attempts): + try: + if await _await_mutation_completion( + asyncio.to_thread( + coordinator.release_exact, snapshot)): + return + except BaseException as release_error: + last_error = release_error + outcome, current = await _await_mutation_completion( + asyncio.to_thread( + coordinator.classify_release, snapshot)) + if outcome == "committed": + return + if outcome != "uncommitted" or current != snapshot: + raise CoordinationError( + "coordination_unavailable", 503) from last_error + raise CoordinationError( + "coordination_unavailable", 503) from last_error + async def settle_background(keep: bool) -> None: if keep: await retain_root_exact() @@ -1402,28 +2106,38 @@ class VpcCoordinator: async def recover_parent_handoff() -> None: """Fence an atomically rebound request after a lost CAS response.""" nonlocal claim - if parent is None or not handoff_attempted or handoff_original is None: + if (parent is None or not handoff_attempted + or handoff_original is None or handoff_family is None): return + family_handoff = len(handoff_family) > 1 + classify = (coordinator.classify_resolve_family_and_bind + if family_handoff + else coordinator.classify_resolve_and_bind) + resolve = (coordinator.resolve_family_and_bind + if family_handoff else coordinator.resolve_and_bind) + source = handoff_family if family_handoff else handoff_original + evidence = (family_handoff_evidence + if family_handoff else handoff_evidence) outcome, current = await _await_mutation_completion( asyncio.to_thread( - coordinator.classify_resolve_and_bind, - handoff_original, parent.claim, handoff_evidence, + classify, source, parent.claim, evidence, operation_id=plan.operation_id, metadata=metadata)) if outcome == "uncommitted": try: claim = await _await_mutation_completion( asyncio.to_thread( - coordinator.resolve_and_bind, - handoff_original, parent.claim, handoff_evidence, + resolve, source, parent.claim, evidence, operation_id=plan.operation_id, metadata=metadata)) except BaseException as retry_error: outcome, current = await _await_mutation_completion( asyncio.to_thread( - coordinator.classify_resolve_and_bind, - handoff_original, parent.claim, handoff_evidence, + classify, source, parent.claim, evidence, operation_id=plan.operation_id, metadata=metadata)) if outcome != "committed": if (isinstance(current, Mapping) + and exact_parent(current, bound=False)): + await release_anonymous_parent_exact(current) + elif (isinstance(current, Mapping) and current.get("token") == parent.claim.token and current.get("execution_id") == parent.claim.execution_id @@ -1441,6 +2155,9 @@ class VpcCoordinator: await retain_root_exact() return if (isinstance(current, Mapping) + and exact_parent(current, bound=False)): + await release_anonymous_parent_exact(current) + elif (isinstance(current, Mapping) and current.get("token") == parent.claim.token and current.get("execution_id") == parent.claim.execution_id and current.get("owner_id") == parent.claim.owner_id): @@ -1509,10 +2226,17 @@ class VpcCoordinator: await _await_mutation_completion(admit()) except CoordinationClaimRejected as exc: if handoff_attempted: + family_handoff = bool( + handoff_family is not None and len(handoff_family) > 1) + classify = (coordinator.classify_resolve_family_and_bind + if family_handoff + else coordinator.classify_resolve_and_bind) + source = handoff_family if family_handoff else handoff_original + evidence = (family_handoff_evidence + if family_handoff else handoff_evidence) outcome, current = await _await_mutation_completion( asyncio.to_thread( - coordinator.classify_resolve_and_bind, - handoff_original, parent.claim, handoff_evidence, + classify, source, parent.claim, evidence, operation_id=plan.operation_id, metadata=metadata)) if outcome == "committed" and exact_parent(current, bound=True): claim = current @@ -1581,6 +2305,60 @@ class VpcCoordinator: def _key(vpc_id: str, child_id: str) -> str: return f"{vpc_id}.{child_id}" + @staticmethod + def _decode_guest_exec_raw( + raw: Any, *, expected_vpc_id: str, + expected_host_id: str | None = None, + expected_member_id: str | None = None, + expected_instance_uuid: str | None = None, + ) -> tuple[dict[str, Any], dict[str, str]]: + """Decode one bounded ledger while preserving its exact identity.""" + if (not isinstance(raw, str) + or len(raw.encode("utf-8")) > VPC_GUEST_EXEC_MAX_BYTES): + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + try: + value = json.loads(raw) + except (TypeError, ValueError) as exc: + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") from exc + try: + base = { + "vpc_id": _identifier(value.get("vpc_id"), "VPC"), + "host_id": _host_id(value.get("host_id")), + "member_id": _identifier( + value.get("member_id"), "VPC member"), + "instance_uuid": _uuid(value.get("instance_uuid")), + } + except (AttributeError, VpcError) as exc: + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") from exc + expected = { + "vpc_id": expected_vpc_id, + **({"host_id": expected_host_id} + if expected_host_id is not None else {}), + **({"member_id": expected_member_id} + if expected_member_id is not None else {}), + **({"instance_uuid": expected_instance_uuid} + if expected_instance_uuid is not None else {}), + } + if any(base.get(field) != wanted for field, wanted in expected.items()): + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + core_fields = { + "version", "phase", "domain_uuid", "domain_runtime_id", + "qemu_runtime_token", "guest_network_generation", "probe_kind", + } | ({"pid"} if value.get("phase") == "pending" else set()) + if (not isinstance(value, Mapping) + or set(value) != set(base) | core_fields): + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + state = _guest_exec_core({field: value[field] for field in core_fields}) + if state.get("domain_uuid") != base["instance_uuid"]: + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + return state, base + def _lock(self, vpc_id: str): import asyncio return self._locks.setdefault(vpc_id, asyncio.Lock()) @@ -1641,32 +2419,12 @@ class VpcCoordinator: "instance_uuid": member.instance_uuid, } if raw is not None: - if (not isinstance(raw, str) - or len(raw.encode("utf-8")) > VPC_GUEST_EXEC_MAX_BYTES): - raise VpcError("VPC 来宾执行账本损坏", 500, - "vpc_record_corrupt") - try: - value = json.loads(raw) - except (TypeError, ValueError) as exc: - raise VpcError("VPC 来宾执行账本损坏", 500, - "vpc_record_corrupt") from exc - if (not isinstance(value, Mapping) - or set(value) != set(base) | { - "version", "phase", "domain_uuid", - "domain_runtime_id", "qemu_runtime_token", - "guest_network_generation", - "probe_kind", - } | ({"pid"} if value.get("phase") == "pending" else set()) - or any(value.get(field) != expected - for field, expected in base.items())): - raise VpcError("VPC 来宾执行账本损坏", 500, - "vpc_record_corrupt") - state = _guest_exec_core({ - field: value[field] for field in value if field not in base - }) - if state.get("domain_uuid") != member.instance_uuid: - raise VpcError("VPC 来宾执行账本损坏", 500, - "vpc_record_corrupt") + state, _loaded_base = self._decode_guest_exec_raw( + raw, expected_vpc_id=plan.vpc_id, + expected_host_id=member.host_id, + expected_member_id=member.member_id, + expected_instance_uuid=member.instance_uuid, + ) def persist(core: Mapping[str, Any]) -> None: normalized = _guest_exec_core(core) @@ -1687,6 +2445,174 @@ class VpcCoordinator: return state, persist, clear + async def _recover_retained_guest_exec_cleanup( + self, plan: VpcPlan, + ) -> None: + """Finish a retired ledger cleanup only after the family handoff. + + This runs inside the canonical VPC -> operation -> catalog order. It + then obtains the ordinary per-VM lock and a replacement-root VM claim, + repeats the strict stopped/absence readback, and deletes only the byte- + exact ledger captured while the old family was still fenced. + """ + desired_raw = self.store.get("vpc_desired", plan.vpc_id) + if desired_raw is None: + return + try: + record = json.loads(desired_raw) + durable = _plan_from_record(record) + except (TypeError, ValueError, VpcError) as exc: + raise VpcError("VPC 持久化记录损坏", 500, + "vpc_record_corrupt") from exc + marker = record.get("guest_exec_cleanup_recovery") + if marker is None: + return + if (not isinstance(record, Mapping) + or durable.operation_id != plan.operation_id + or durable.desired_revision != plan.desired_revision + or durable.desired_digest != plan.desired_digest + or durable.canonical() != plan.canonical()): + raise VpcError("VPC 清理目标已变化", 409, + "vpc_revision_conflict") + targets = self._guest_exec_cleanup_marker_targets(plan, marker) + if targets is None: + raise VpcError("VPC 来宾执行清理记录损坏", 500, + "vpc_record_corrupt") + if not callable(self.vm_lock): + raise CoordinationError("coordination_unavailable", 503) + + identities = tuple(sorted( + (item["host_id"], item["instance_uuid"]) for item in targets)) + wanted_hosts = {host_id for host_id, _instance_uuid in identities} + request_deadline = time.monotonic() + VPC_REQUEST_DEADLINE_SECONDS + + try: + async with AsyncExitStack() as stack: + for host_id, instance_uuid in identities: + await stack.enter_async_context(_pre_dataplane_lock( + self.vm_lock(host_id, instance_uuid))) + + fabric_record = self._fabric_record(plan) + full_fabric_plan = self.fabric._plan_from_record(fabric_record) + probe_plan = replace( + full_fabric_plan, + nodes=tuple( + node for node in full_fabric_plan.nodes + if node.host_id in wanted_hosts), + ) + if {node.host_id for node in probe_plan.nodes} != wanted_hosts: + raise VpcError("VPC 宿主机不在 Fabric 闭集中", 409, + "vpc_fabric_identity_mismatch") + components = await self.fabric._components(probe_plan) + if set(components) != wanted_hosts: + raise VpcError("VPC 宿主机组件闭集不完整", 502, + "vpc_host_unavailable") + + from .libvirt_service import VMService + + for item in targets: + host_id = item["host_id"] + instance_uuid = item["instance_uuid"] + conflict = self._lifecycle_conflict( + host_id, instance_uuid, plan.operation_id) + if conflict is not None: + raise VpcError("实例生命周期任务正在修改私网配置", 409, + conflict) + vm_resource = f"vm:{host_id}:{instance_uuid}" + with resource_guard( + (vm_resource,), tracked_mutations=False): + # A different revision or a tampered cleanup marker must + # never inherit this destructive local-store action. + if self.store.get("vpc_desired", plan.vpc_id) != desired_raw: + raise VpcError("VPC 清理目标已变化", 409, + "vpc_revision_conflict") + component, profile = components[host_id] + del component # The strict proof is libvirt/seed-only. + service = VMService( + profile["libvirt_uri"], profile["storage_pool"], + profile["iso_pool"], profile["default_network"], + rescue_iso_pool=profile.get("rescue_iso_pool", ""), + execution_scope=host_id, + ) + absent = await asyncio.to_thread( + service.read_private_absence_unclaimed, + instance_uuid, deadline=request_deadline) + if (absent.get("removed") is not True + or absent.get("running") is not False): + raise VpcError( + "退役实例尚未停机或私网残留尚未清除", 503, + "vpc_private_guest_cleanup_pending") + + ledger_raw = self.store.get( + VPC_GUEST_EXEC_KIND, item["ledger_key"]) + if ledger_raw is None: + continue + self._decode_guest_exec_raw( + ledger_raw, expected_vpc_id=plan.vpc_id, + expected_host_id=host_id, + expected_member_id=item["member_id"], + expected_instance_uuid=instance_uuid, + ) + encoded = ledger_raw.encode("utf-8") + if (len(encoded) != item["ledger_bytes"] + or hashlib.sha256(encoded).hexdigest() + != item["ledger_sha256"]): + raise VpcError("VPC 来宾执行账本已变化", 500, + "vpc_record_corrupt") + try: + self.store.delete( + VPC_GUEST_EXEC_KIND, item["ledger_key"]) + except BaseException as delete_error: + after = self.store.get( + VPC_GUEST_EXEC_KIND, item["ledger_key"]) + if after is None: + continue + if after != ledger_raw: + raise VpcError( + "VPC 来宾执行账本在清理期间发生变化", 500, + "vpc_record_corrupt") from delete_error + raise + after = self.store.get( + VPC_GUEST_EXEC_KIND, item["ledger_key"]) + if after is None: + continue + if after != ledger_raw: + raise VpcError( + "VPC 来宾执行账本在清理期间发生变化", 500, + "vpc_record_corrupt") + raise VpcError("VPC 来宾执行账本清理尚未确认", 503, + "vpc_private_guest_cleanup_pending") + + # All targets have a fresh stopped/absence proof and their + # ledgers are absent. Remove the intent with a byte-exact CAS so + # a later desired revision is not blocked by an old cleanup. + cleared = dict(record) + cleared.pop("guest_exec_cleanup_recovery", None) + cleared_raw = json.dumps( + cleared, ensure_ascii=True, separators=(",", ":")) + compare_and_set = getattr(self.store, "compare_and_set", None) + if not callable(compare_and_set): + raise CoordinationError("coordination_unavailable", 503) + try: + committed = compare_and_set( + "vpc_desired", plan.vpc_id, + desired_raw, cleared_raw) + except BaseException: + if self.store.get("vpc_desired", plan.vpc_id) != cleared_raw: + raise + else: + if not committed: + raise CoordinationError("coordination_uncertain", 503) + if self.store.get("vpc_desired", plan.vpc_id) != cleared_raw: + raise CoordinationError("coordination_uncertain", 503) + except asyncio.CancelledError: + raise + except BaseException as exc: + # The old family has already been atomically replaced. Preserve the + # replacement root (and any active VM child) so a later explicit + # reconcile can resume this exact marker instead of losing it. + raise CoordinationError("coordination_uncertain", 503) from exc + @staticmethod def _confirmed_removal( host: Mapping[str, Any], revision: int, digest: str, @@ -1732,14 +2658,18 @@ class VpcCoordinator: ) @staticmethod - def _confirmed_active_host(host: Mapping[str, Any], plan: VpcPlan) -> bool: - """Whether this exact host revision already has complete live proof. + def _confirmed_active_host( + host: Mapping[str, Any], plan: VpcPlan, *, allow_pending: bool = False, + ) -> bool: + """Whether this exact host revision has complete configuration proof. A request-wide probe budget is shared by every host. Retrying an already exact host on every reconcile would let an early slow host - starve later hosts forever. Preserve only the strong same-revision - proof; any tombstone, member-set change, or incomplete guest evidence - puts the host back in the execution closure. + starve later hosts forever. Persistent XML, seed and host-manifest + readback complete the configuration operation even when a member stays + stopped or its QGA is unavailable. Guest runtime health remains in the + observation, but does not put the exact configuration back into the + periodic write/claim closure. """ host_id = host.get("host_id") if not isinstance(host_id, str): @@ -1775,27 +2705,260 @@ class VpcCoordinator: member.private_mac) or item.get("config_verified") is not True or item.get("xml_verified") is not True - or item.get("domain_running") is not True - or item.get("guest_network_verified") is not True - or item.get("restart_required") is not False): + or item.get("seed_verified") is not True): continue proven_ids.add(member_id) return bool( host.get("retiring") is not True and host.get("retiring_members", []) == [] - and host.get("state") == "active" + and (allow_pending + or host.get("pending_member_mutations", []) == []) and host.get("transport_verified") is True - and host.get("private_dataplane_verified") is True and host.get("config_verified") is True and host.get("desired_revision") == plan.desired_revision - and host.get("observed_revision") == plan.desired_revision and host.get("config_observed_revision") == plan.desired_revision and host.get("desired_digest") == plan.desired_digest - and host.get("observed_digest") == plan.desired_digest and host.get("config_observed_digest") == plan.desired_digest and proven_ids == expected_ids ) + @staticmethod + def _pending_member_mutations(host: Mapping[str, Any]) -> tuple[str, ...]: + """Validate the durable stop-before-write member closure.""" + if "pending_member_mutations" not in host: + raise VpcError("VPC 成员停机核对记录缺失", 500, + "vpc_record_corrupt") + raw = host.get("pending_member_mutations") + if (not isinstance(raw, list) or len(raw) > VPC_MAX_MEMBERS + or any(not isinstance(value, str) for value in raw)): + raise VpcError("VPC 成员停机核对记录损坏", 500, + "vpc_record_corrupt") + try: + normalized = tuple(sorted({_uuid(value) for value in raw})) + except VpcError as exc: + raise VpcError("VPC 成员停机核对记录损坏", 500, + "vpc_record_corrupt") from exc + if len(normalized) != len(raw): + raise VpcError("VPC 成员停机核对身份重复", 500, + "vpc_record_corrupt") + return normalized + + @classmethod + def _migrate_legacy_pending_member_mutations( + cls, record: dict[str, Any], plan: VpcPlan, + ) -> bool: + """Upgrade pre-stop-fence host rows without guessing write safety. + + An exact configuration or absence proof needs no future host write, so + its legacy row can be completed with an empty fence. Every incomplete + row instead receives the complete current/retiring VM identity set for + that host. A malformed or ambiguous closure remains corrupt rather + than being interpreted as an empty (and therefore writable) fence. + """ + hosts = cls._host_closure(record) + missing = { + host_id for host_id, host in hosts.items() + if "pending_member_mutations" not in host + } + migrated_identities: dict[str, str] = {} + for host_id, host in hosts.items(): + if host_id not in missing: + for instance_uuid in cls._pending_member_mutations(host): + other_host = migrated_identities.get(instance_uuid) + if other_host is not None and other_host != host_id: + raise VpcError("VPC 成员停机核对身份重复", 500, + "vpc_record_corrupt") + migrated_identities[instance_uuid] = host_id + if not missing: + return False + + current_by_host: dict[str, list[VpcMember]] = {} + for member in plan.members: + current_by_host.setdefault(member.host_id, []).append(member) + + delete_by_host: dict[str, list[VpcDeleteMember]] = {} + marker = cls._delete_marker(record) + if marker is not None: + try: + delete_members = _parse_delete_members( + marker["retiring_members"], plan) + except VpcError as exc: + raise VpcError("VPC 删除成员清单记录无效", 500, + "vpc_record_corrupt") from exc + for member in delete_members: + delete_by_host.setdefault(member.host_id, []).append(member) + + for host_id in sorted(missing): + host = hosts[host_id] + if not isinstance(host, dict): + raise VpcError("VPC 宿主机闭集无效", 500, + "vpc_record_corrupt") + if (cls._confirmed_active_host(host, plan, allow_pending=True) + or cls._confirmed_removal( + host, plan.desired_revision, plan.desired_digest)): + pending: set[str] = set() + else: + current = current_by_host.get(host_id, []) + raw_member_ids = host.get("member_ids") + if (not isinstance(raw_member_ids, list) + or len(raw_member_ids) > VPC_MAX_MEMBERS + or any(not isinstance(value, str) + for value in raw_member_ids)): + raise VpcError("VPC 宿主成员闭集损坏", 500, + "vpc_record_corrupt") + try: + member_ids = [_identifier(value, "VPC member") + for value in raw_member_ids] + except VpcError as exc: + raise VpcError("VPC 宿主成员闭集损坏", 500, + "vpc_record_corrupt") from exc + if len(member_ids) != len(set(member_ids)): + raise VpcError("VPC 宿主成员闭集重复", 500, + "vpc_record_corrupt") + expected_ids = {member.member_id for member in current} + if host.get("retiring") is not True: + if set(member_ids) != expected_ids: + raise VpcError("VPC 宿主成员闭集不一致", 500, + "vpc_record_corrupt") + elif current: + raise VpcError("VPC 退役宿主仍包含当前成员", 500, + "vpc_record_corrupt") + + pending = {member.instance_uuid for member in current} + raw_retiring = host.get("retiring_members") + if (not isinstance(raw_retiring, list) + or len(raw_retiring) > VPC_MAX_MEMBERS): + raise VpcError("VPC 宿主机清理成员证据损坏", 500, + "vpc_record_corrupt") + if marker is None: + retiring_source: Mapping[str, Any] + if host.get("retiring") is True: + retiring_source = host + else: + synthetic = { + "member_ids": [ + row.get("member_id") + for row in raw_retiring + if isinstance(row, Mapping) + ], + "retiring_members": raw_retiring, + } + if len(synthetic["member_ids"]) != len(raw_retiring): + raise VpcError( + "VPC 宿主机清理成员证据损坏", 500, + "vpc_record_corrupt") + retiring_source = synthetic + normalized = cls._durable_retiring_members( + retiring_source, + vpc_id=plan.vpc_id, + host_id=host_id) + pending.update( + row["instance_uuid"] for row in normalized) + else: + signed = { + member.member_id: member + for member in delete_by_host.get(host_id, []) + } + for raw in raw_retiring: + if not isinstance(raw, Mapping): + raise VpcError( + "VPC 删除成员清单记录无效", 500, + "vpc_record_corrupt") + member = signed.get(raw.get("member_id")) + try: + raw_uuid = _uuid(raw.get("instance_uuid")) + raw_host = _host_id(raw.get("host_id")) + raw_ip = _host_address(raw.get("private_ip_cidr")) + raw_mac = _effective_private_mac( + plan.vpc_id, str(raw.get("member_id")), + raw.get("private_mac")) + except (TypeError, ValueError, VpcError) as exc: + raise VpcError( + "VPC 删除成员清单记录无效", 500, + "vpc_record_corrupt") from exc + if (member is None + or raw_uuid != member.instance_uuid + or raw_host != member.host_id + or raw_ip != member.private_ip + or raw_mac != member.private_mac): + raise VpcError( + "VPC 删除成员清单记录不一致", 500, + "vpc_record_corrupt") + pending.update( + member.instance_uuid + for member in delete_by_host.get(host_id, [])) + if len(pending) > VPC_MAX_MEMBERS: + raise VpcError("VPC 成员停机核对记录过大", 500, + "vpc_record_corrupt") + + for instance_uuid in pending: + other_host = migrated_identities.get(instance_uuid) + if other_host is not None and other_host != host_id: + raise VpcError("VPC 成员停机核对身份重复", 500, + "vpc_record_corrupt") + migrated_identities[instance_uuid] = host_id + host["pending_member_mutations"] = sorted(pending) + return True + + @classmethod + def _member_mutation_guard_identities( + cls, record: Mapping[str, Any], + ) -> tuple[tuple[str, str], ...]: + """Return the revision-wide VM fence, including exact skipped hosts.""" + identities: dict[str, str] = {} + for host_id, host in cls._host_closure(record).items(): + for instance_uuid in cls._pending_member_mutations(host): + other_host = identities.get(instance_uuid) + if other_host is not None and other_host != host_id: + raise VpcError("VPC 成员停机核对身份重复", 500, + "vpc_record_corrupt") + identities[instance_uuid] = host_id + return tuple(sorted( + (host_id, instance_uuid) + for instance_uuid, host_id in identities.items() + )) + + @classmethod + def _complete_member_mutation_fence( + cls, record: dict[str, Any], plan: VpcPlan, + ) -> bool: + """Clear stop fences only after the full revision is configuration exact.""" + hosts = cls._host_closure(record) + if not any(cls._pending_member_mutations(host) + for host in hosts.values()): + return False + exact = all( + cls._confirmed_removal( + host, plan.desired_revision, plan.desired_digest) + or (host.get("retiring") is not True + and not host.get("retiring_members") + and cls._confirmed_active_host( + host, plan, allow_pending=True)) + for host in hosts.values() + ) + if not exact: + return False + for host in hosts.values(): + if not isinstance(host, dict): + raise VpcError("VPC 宿主机闭集无效", 500, + "vpc_record_corrupt") + host["pending_member_mutations"] = [] + return True + + @classmethod + def _member_boot_identity( + cls, plan: VpcPlan, member: VpcMember, + ) -> tuple[str, str, str, str, tuple[str, ...]]: + """Return only member fields that change persistent XML or seed data.""" + return ( + member.instance_uuid, + member.host_id, + member.private_ip, + _effective_private_mac( + plan.vpc_id, member.member_id, member.private_mac), + tuple(cls._private_routes(plan, member)), + ) + @staticmethod def _runtime_readback_eligible( host: Mapping[str, Any] | None, @@ -1816,6 +2979,8 @@ class VpcCoordinator: or host.get("host_id") != host_id or host.get("retiring") is True or host.get("retiring_members") != [] + or "pending_member_mutations" not in host + or host.get("pending_member_mutations") != [] or bool(retiring_members) or host.get("config_verified") is not True # Legacy rows predate this bit. Admit one readback-only pass; @@ -2421,6 +3586,360 @@ class VpcCoordinator: return code return "vpc_private_dataplane_apply_failed" + @staticmethod + def _require_host_member_mutations_stopped( + profile: Mapping[str, Any], host_id: str, + instance_uuids: Sequence[str], + ) -> None: + """Perform one fresh read-only libvirt stop proof for a host.""" + from .libvirt_service import VMService + + service = VMService( + profile["libvirt_uri"], profile["storage_pool"], + profile["iso_pool"], profile["default_network"], + rescue_iso_pool=profile.get("rescue_iso_pool", ""), + execution_scope=host_id, + ) + for offset in range(0, len(instance_uuids), 100): + service.require_vpc_members_stopped( + instance_uuids[offset:offset + 100]) + + @staticmethod + def _retained_dataplane_plan( + plan: VpcPlan, host_id: str, + fabric_peers: Sequence[Mapping[str, Any]], + fabric_public_keys: Mapping[str, str], + fabric_plan_digest: str, + fabric_peer_overlays: Mapping[str, str], + ) -> Any: + """Build the same typed host projection used by the write path.""" + from .vpc_dataplane import ( + VpcDataplanePlan, parse_vpc_dataplane_plan, resource_names, + ) + + local_public_key = fabric_public_keys.get(host_id) + if fabric_peer_overlays and not isinstance(local_public_key, str): + raise VpcError("Fabric 本机公钥记录不完整", 500, + "vpc_fabric_record_corrupt") + peer_keys = [ + key for peer_host, key in fabric_public_keys.items() + if peer_host != host_id + ] + peer_overlays = { + key: overlay for key, overlay in fabric_peer_overlays.items() + if key != local_public_key + } + if set(peer_overlays) != set(peer_keys): + raise VpcError("Fabric 公钥与 overlay 闭集不一致", 500, + "vpc_fabric_record_corrupt") + if host_id in plan.host_ids: + payload = plan.canonical() + payload["fabric_peers"] = [ + dict(peer) for peer in fabric_peers + if peer.get("host_id") != host_id + ] + trusted_peer_tenant_ips = { + str(peer["host_id"]): tuple( + peer.get("tenant_allowed_ips", ())) + for peer in payload["fabric_peers"] + if (isinstance(peer, Mapping) + and isinstance(peer.get("host_id"), str)) + } + dataplane = parse_vpc_dataplane_plan( + payload, host_id, + trusted_tenant_allowed_ips=trusted_peer_tenant_ips, + fabric_peer_keys=peer_keys, + fabric_peer_overlays=peer_overlays, + fabric_plan_digest=fabric_plan_digest, + ) + return replace(dataplane, vpc=plan) + return VpcDataplanePlan( + plan.vpc, host_id, tuple(plan.members), tuple(), + resource_names(plan.vpc_id), fabric_plan_digest, + frozenset(peer_keys), frozenset(peer_overlays.items()), + ) + + def _run_retained_host_readback( + self, plan: VpcPlan, host: Mapping[str, Any], component: Any, + profile: Mapping[str, Any], + fabric_peers: Sequence[Mapping[str, Any]], + fabric_public_keys: Mapping[str, str], + fabric_plan_digest: str, + fabric_peer_overlays: Mapping[str, str], + request_deadline: float, + ) -> dict[str, Any]: + """Observe one retained-family host without any apply, QGA or claim.""" + from .libvirt_service import VMService + from .vpc_dataplane import ( + VpcPrivateHostAdapter, private_guest_network_generation, + ) + + host_id = _host_id(host.get("host_id")) + dataplane = self._retained_dataplane_plan( + plan, host_id, fabric_peers, fabric_public_keys, + fabric_plan_digest, fabric_peer_overlays, + ) + service = VMService( + profile["libvirt_uri"], profile["storage_pool"], + profile["iso_pool"], profile["default_network"], + rescue_iso_pool=profile.get("rescue_iso_pool", ""), + execution_scope=host_id, + ) + adapter = VpcPrivateHostAdapter( + component.manager, profile, request_deadline=request_deadline) + + def require_budget() -> None: + if time.monotonic() >= request_deadline: + raise VpcError( + "VPC 保留任务只读核对超过请求预算", 502, + "vpc_private_guest_probe_deadline_exceeded") + + def ledger_present(instance_uuid: str) -> bool: + return self.store.get( + VPC_GUEST_EXEC_KIND, + self._key(plan.vpc_id, instance_uuid), + ) is not None + + retiring: list[VpcMember] = [] + raw_retiring = host.get("retiring_members", []) + if not isinstance(raw_retiring, list): + raise VpcError("VPC 宿主机清理成员证据损坏", 500, + "vpc_record_corrupt") + for raw in raw_retiring: + if not isinstance(raw, Mapping): + raise VpcError("VPC 宿主机清理成员证据损坏", 500, + "vpc_record_corrupt") + member_host_id = _host_id(raw.get("host_id")) + if member_host_id != host_id: + raise VpcError("VPC 宿主机清理成员证据损坏", 500, + "vpc_record_corrupt") + private_ip = str(raw.get("private_ip_cidr", "")) + if not private_ip.endswith("/32"): + raise VpcError("VPC 宿主机清理成员证据损坏", 500, + "vpc_record_corrupt") + retiring.append(VpcMember( + _identifier(raw.get("member_id"), "VPC member"), + _uuid(raw.get("instance_uuid")), member_host_id, + _host_address(private_ip), + _identifier(raw.get("subnet_id", "retiring"), "VPC subnet"), + _effective_private_mac( + plan.vpc_id, + _identifier(raw.get("member_id"), "VPC member"), + raw.get("private_mac"), + ), + )) + + for member in retiring: + require_budget() + absent = service.read_private_absence_unclaimed( + member.instance_uuid, deadline=request_deadline) + if absent.get("removed") is not True: + raise VpcError("私网网卡移除回读尚未确认", 502, + "vpc_private_remove_readback_mismatch") + if absent.get("running") is True and ledger_present( + member.instance_uuid): + raise VpcError("来宾私网探测尚未收敛,不能释放保留任务", 409, + "vpc_private_guest_probe_pending") + + pairs: dict[str, Mapping[str, Any]] = {} + for member in dataplane.local_members: + require_budget() + routes = self._private_routes(plan, member) + pair = service.read_private_configuration_unclaimed( + member.instance_uuid, + private_mac=_effective_private_mac( + plan.vpc_id, member.member_id, member.private_mac), + private_ip_cidr=f"{member.private_ip}/32", + private_routes=routes, + private_seed_identity={ + "vpc_id": plan.vpc_id, + "member_id": member.member_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "guest_network_generation": + private_guest_network_generation(member, routes), + }, + deadline=request_deadline, + ) + if pair.get("live") is not None and ledger_present( + member.instance_uuid): + raise VpcError("来宾私网探测尚未收敛,不能释放保留任务", 409, + "vpc_private_guest_probe_pending") + pairs[member.instance_uuid] = pair + + require_budget() + if not dataplane.local_members: + observed = adapter.observe_removal(dataplane) + else: + observed = adapter.observe( + dataplane, xml_by_instance=pairs) + observed.update({ + "vpc_id": plan.vpc_id, + "host_id": host_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "transport_verified": True, + "private_dataplane_verified": observed.get("state") == "active", + "rollback": observed.get("rollback", "none"), + }) + return observed + + async def _observe_retained_family_hosts( + self, plan: VpcPlan, record: Mapping[str, Any], + ) -> dict[str, Any]: + """Collect a complete all-host proof while the old family stays fenced.""" + candidate = json.loads(json.dumps( + record, ensure_ascii=True, separators=(",", ":"))) + hosts = self._host_closure(candidate) + self._validate_dataplane_resources(plan) + fabric_record = self._fabric_record(plan) + full_fabric_plan = self.fabric._plan_from_record(fabric_record) + raw_public_keys = fabric_record.get("public_keys") + if not isinstance(raw_public_keys, Mapping): + raise VpcError("Fabric 公钥记录不完整", 500, + "vpc_fabric_record_corrupt") + public_keys = { + node.host_id: raw_public_keys.get(node.host_id) + for node in full_fabric_plan.nodes + } + if any(not isinstance(value, str) for value in public_keys.values()): + raise VpcError("Fabric 公钥记录不完整", 500, + "vpc_fabric_record_corrupt") + peer_overlays = { + str(public_keys[node.host_id]): node.overlay_ip + "/32" + for node in full_fabric_plan.nodes + } + if len(peer_overlays) != len(full_fabric_plan.nodes): + raise VpcError("Fabric 公钥与 overlay 闭集重复", 500, + "vpc_fabric_record_corrupt") + wanted = { + host_id for host_id, host in hosts.items() + if not ( + self._confirmed_removal( + host, plan.desired_revision, plan.desired_digest) + or (host.get("retiring") is not True + and self._confirmed_active_host( + host, plan, allow_pending=True)) + ) + } + probe_plan = replace( + full_fabric_plan, + nodes=tuple(node for node in full_fabric_plan.nodes + if node.host_id in wanted), + ) + if {node.host_id for node in probe_plan.nodes} != wanted: + raise VpcError("VPC 宿主机不在 Fabric 闭集中", 409, + "vpc_fabric_identity_mismatch") + components = ( + await self.fabric._components(probe_plan) if wanted else {}) + if set(components) != wanted: + raise VpcError("VPC 宿主机组件闭集不完整", 502, + "vpc_host_unavailable") + fabric_peers = self._fabric_peers_for_plan(plan, fabric_record) + request_deadline = time.monotonic() + VPC_REQUEST_DEADLINE_SECONDS + for host_id in sorted(wanted): + host = hosts[host_id] + identities = { + member.instance_uuid for member in plan.members + if member.host_id == host_id + } + for raw in host.get("retiring_members", []): + if isinstance(raw, Mapping): + identities.add(_uuid(raw.get("instance_uuid"))) + if any(self._lifecycle_conflict( + host_id, instance_uuid, plan.operation_id) is not None + for instance_uuid in identities): + raise VpcError("实例生命周期任务正在修改私网配置", 409, + "vpc_private_lifecycle_conflict") + component, profile = components[host_id] + observed = await asyncio.to_thread( + self._run_retained_host_readback, + plan, host, component, profile, fabric_peers, + public_keys, full_fabric_plan.digest, peer_overlays, + request_deadline, + ) + mutable = candidate["hosts"][ + next(index for index, item in enumerate(candidate["hosts"]) + if item.get("host_id") == host_id) + ] + mutable.update(observed) + if mutable.get("retiring_members"): + mutable["retiring_members"] = [] + if mutable.get("retiring") is True: + mutable["removed"] = observed.get("removed") is True + self._complete_member_mutation_fence(candidate, plan) + gate = self._fabric_gate(plan) + self._finish(candidate, gate) + if not self._retained_record_readback_safe(plan, candidate): + raise VpcError("VPC 保留任务只读核对尚未闭合", 502, + "vpc_retained_family_readback_incomplete") + return candidate + + async def _recover_retained_family_readback( + self, plan: VpcPlan, family: Sequence[Mapping[str, Any]], + ) -> bool: + """Persist a proof marker only after a complete read-only observation.""" + original_raw = self.store.get("vpc_desired", plan.vpc_id) + if not isinstance(original_raw, str): + return False + try: + record = json.loads(original_raw) + if (not isinstance(record, dict) + or record.get("vpc_id") != plan.vpc_id): + return False + durable = _plan_from_record(record) + except (TypeError, ValueError, VpcError): + return False + if (durable.operation_id != plan.operation_id + or durable.desired_revision != plan.desired_revision + or durable.desired_digest != plan.desired_digest + or durable.canonical() != plan.canonical() + or not self._retained_family_covers_vm_closure( + record, plan, family)): + return False + try: + candidate = await self._observe_retained_family_hosts(plan, record) + except Exception: + return False + candidate["claim_family_recovery"] = ( + self._retained_family_recovery_marker(plan, family)) + if (not self._retained_record_readback_safe(plan, candidate) + or not self._retained_family_recovery_marker_matches( + plan, candidate, family)): + return False + candidate_raw = json.dumps( + candidate, ensure_ascii=True, separators=(",", ":")) + compare_and_set = getattr(self.store, "compare_and_set", None) + if not callable(compare_and_set): + # Retained-family recovery is a cross-process fencing operation. + # A process-local get/set fallback could overwrite a newer target. + return False + try: + committed_now = compare_and_set( + "vpc_desired", plan.vpc_id, original_raw, candidate_raw) + except BaseException: + # A CAS response may be lost after commit. Classify only a byte + # exact candidate; a different raw value belongs to another + # writer and must never be overwritten or adopted here. + if self.store.get("vpc_desired", plan.vpc_id) != candidate_raw: + raise + else: + if not committed_now: + return False + committed_raw = self.store.get("vpc_desired", plan.vpc_id) + if committed_raw != candidate_raw: + return False + try: + committed = json.loads(committed_raw) + except (TypeError, ValueError): + return False + return bool( + isinstance(committed, Mapping) + and self._retained_record_readback_safe(plan, committed) + and self._retained_family_recovery_marker_matches( + plan, committed, family) + ) + def _run_host_dataplane( self, plan: VpcPlan, host_id: str, component: Any, profile: Mapping[str, Any], fabric_peers: list[dict[str, Any]], @@ -2432,6 +3951,7 @@ class VpcCoordinator: fabric_peer_overlays: Mapping[str, str] | None = None, request_deadline: float | None = None, previous_host: Mapping[str, Any] | None = None, + pending_member_mutations: Sequence[str] | None = None, ) -> dict[str, Any]: """Apply and read back one host synchronously behind the VPC lock.""" from .libvirt_service import VMService @@ -2525,14 +4045,30 @@ class VpcCoordinator: if request_deadline is None: request_deadline = ( time.monotonic() + VPC_REQUEST_DEADLINE_SECONDS) - adapter = VpcPrivateHostAdapter( - component.manager, profile, request_deadline=request_deadline) service = VMService( profile["libvirt_uri"], profile["storage_pool"], profile["iso_pool"], profile["default_network"], rescue_iso_pool=profile.get("rescue_iso_pool", ""), execution_scope=host_id, ) + mutation_ids = self._pending_member_mutations({ + "pending_member_mutations": list( + pending_member_mutations or ()), + }) + # This is the last boundary before any host, libvirt XML or seed write. + # Recheck libvirt itself instead of trusting the panel's cached power + # state. Chunk only to keep the host service's read request bounded. + try: + for offset in range(0, len(mutation_ids), 100): + service.require_vpc_members_stopped( + mutation_ids[offset:offset + 100]) + except Exception as exc: + raise _VpcMemberPreflightError( + str(exc), int(getattr(exc, "status", 502)), + self._observation_error(exc), + ) from exc + adapter = VpcPrivateHostAdapter( + component.manager, profile, request_deadline=request_deadline) guest_exec_contexts: dict[str, tuple[dict[str, Any], Any, Any]] = {} def guest_exec_context( @@ -2605,6 +4141,7 @@ class VpcCoordinator: old_member.private_mac), expected_private_ip_cidr=f"{old_member.private_ip}/32", ) + mark_remote_write_confirmed() if removed_private.get("running") is True and ledger_state: # XML/seed absence cannot prove that an earlier guest-exec # has ended. Keep its durable launch/PID record and require @@ -2622,6 +4159,7 @@ class VpcCoordinator: if not local_members: require_request_budget() removed = adapter.remove(dataplane) + mark_remote_write_confirmed() removed.update({ "vpc_id": plan.vpc_id, "host_id": host_id, @@ -2640,6 +4178,7 @@ class VpcCoordinator: if not runtime_readback_only: require_request_budget() adapter.apply(dataplane) + mark_remote_write_confirmed() prior_nics = { item.get("instance_uuid"): item for item in ( @@ -2682,6 +4221,7 @@ class VpcCoordinator: link_state=(private_link_states or {}).get( member.instance_uuid, "preserve"), ) + mark_remote_write_confirmed() pair = { "inactive": configured.get("xml"), "live": configured.get("live"), @@ -3053,6 +4593,7 @@ class VpcCoordinator: def _mark_lifecycle_conflicts( self, plan: VpcPlan, previous_plan: VpcPlan | None, record: dict[str, Any], operation_id: str | None = None, + extra_identities: Sequence[tuple[str, str]] | None = None, ) -> bool: # ``record`` is the execution closure for this pass. A pending-host # retry deliberately projects the full VPC plan down to one or more @@ -3074,6 +4615,10 @@ class VpcCoordinator: if (isinstance(host_id, str) and isinstance(row, Mapping) and isinstance(row.get("instance_uuid"), str)): identities[(host_id, row["instance_uuid"])] = None + if extra_identities is not None: + for host_id, instance_uuid in extra_identities: + if isinstance(host_id, str) and isinstance(instance_uuid, str): + identities[(host_id, instance_uuid)] = None conflict = False reasons_by_host: dict[str, str] = {} for (host_id, instance_uuid) in identities: @@ -3084,11 +4629,22 @@ class VpcCoordinator: reasons_by_host[host_id] = reason if not conflict: return False + fallback_reason = ( + reasons_by_host[sorted(reasons_by_host)[0]] + if reasons_by_host else None) for host in record.get("hosts", []): if not isinstance(host, dict): continue + execution = current_execution() + if (execution is not None + and execution.execution_unknown.is_set()): + break host_id = host.get("host_id") - reason = reasons_by_host.get(host_id) + # A revision-wide stop fence may belong to a host whose exact + # configuration row was deliberately omitted from this retry. + # Surface that conflict on every selected row so the operation + # cannot look idle while no write was admitted. + reason = reasons_by_host.get(host_id, fallback_reason) if reason is not None: host.update({ "state": "uncertain", @@ -3105,8 +4661,10 @@ class VpcCoordinator: previous_plan: VpcPlan | None = None, private_link_states: Mapping[str, str] | None = None, extra_identities: Sequence[tuple[str, str]] | None = None, + stop_guard_identities: Sequence[tuple[str, str]] | None = None, operation_id: str | None = None, lock_host_ids: Collection[str] | None = None, + complete_mutation_fence: bool = True, ) -> None: # VPC lock is acquired by each public caller before this method. The # VM locks are acquired in stable order here, so a VPC apply cannot @@ -3128,10 +4686,14 @@ class VpcCoordinator: if (isinstance(row, Mapping) and isinstance(row.get("instance_uuid"), str)): derived.append((host_id, row["instance_uuid"])) + if stop_guard_identities is None: + stop_guard_identities = self._member_mutation_guard_identities( + record) if extra_identities is None: - extra_identities = derived + extra_identities = tuple(derived) + tuple(stop_guard_identities) else: - extra_identities = tuple(extra_identities) + tuple(derived) + extra_identities = (tuple(extra_identities) + tuple(derived) + + tuple(stop_guard_identities)) async with self._instance_locks( plan, previous_plan, extra_identities, host_ids=lock_host_ids): @@ -3140,6 +4702,8 @@ class VpcCoordinator: kwargs["private_link_states"] = private_link_states if operation_id is not None: kwargs["operation_id"] = operation_id + kwargs["stop_guard_identities"] = stop_guard_identities + kwargs["complete_mutation_fence"] = complete_mutation_fence await self._reconcile_dataplane_unlocked( plan, record, gate, previous_plan, request_deadline=request_deadline, **kwargs) @@ -3148,8 +4712,10 @@ class VpcCoordinator: self, plan: VpcPlan, record: dict[str, Any], gate: Mapping[str, Any], previous_plan: VpcPlan | None = None, private_link_states: Mapping[str, str] | None = None, + stop_guard_identities: Sequence[tuple[str, str]] | None = None, operation_id: str | None = None, request_deadline: float | None = None, + complete_mutation_fence: bool = True, ) -> None: """Apply/read back every participating host; stay closed on failure.""" if request_deadline is None: @@ -3175,8 +4741,14 @@ class VpcCoordinator: and not retiring_ids and not member_cleanup): self._finish(record, gate) return + if stop_guard_identities is None: + stop_guard_identities = self._member_mutation_guard_identities( + record) + else: + stop_guard_identities = tuple(sorted(set(stop_guard_identities))) if self._mark_lifecycle_conflicts( - plan, previous_plan, record, operation_id): + plan, previous_plan, record, operation_id, + stop_guard_identities): self._finish(record, gate) return try: @@ -3216,6 +4788,8 @@ class VpcCoordinator: if isinstance(item, Mapping) and isinstance(item.get("host_id"), str) } + wanted.update(host_id for host_id, _instance_uuid + in stop_guard_identities) fabric_plan = replace( fabric_plan, nodes=tuple(node for node in fabric_plan.nodes @@ -3227,13 +4801,57 @@ class VpcCoordinator: code = self._observation_error(exc) for host in record.get("hosts", []): if isinstance(host, dict): - host.update({"state": "uncertain", + # Resource, Fabric and host-component resolution above is + # read-only. Its failure proves no VPC write started, so + # retain a retryable operation instead of manufacturing + # an ambiguous/retained-claim result. + host.update({"state": "applying", "transport_verified": False, "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, "last_error_code": code}) self._finish(record, gate) return + # Membership changes alter the shared host manifest as well as the + # target VM's XML/seed. Prove the complete changed-member closure is + # stopped before the first host pass, so a running VM on a later host + # cannot leave earlier hosts partially updated. The target host repeats + # this proof immediately before its own write to close a power-on race. + preflight_failures: dict[str, str] = {} + guard_by_host: dict[str, list[str]] = {} + for host_id, instance_uuid in stop_guard_identities: + guard_by_host.setdefault(host_id, []).append(instance_uuid) + for host_id in sorted(guard_by_host): + try: + if not isinstance(host_id, str) or host_id not in components: + raise VpcError("VPC 宿主机不可用", 502, + "vpc_host_unavailable") + _component, profile = components[host_id] + await asyncio.to_thread( + self._require_host_member_mutations_stopped, + profile, host_id, tuple(sorted(guard_by_host[host_id]))) + except Exception as exc: + if isinstance(host_id, str): + preflight_failures[host_id] = self._observation_error(exc) + if preflight_failures: + fallback_error = preflight_failures[sorted(preflight_failures)[0]] + for host in record.get("hosts", []): + if not isinstance(host, dict): + continue + error_code = preflight_failures.get( + host.get("host_id"), fallback_error) + host.update({ + "state": "applying", + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": error_code, + }) + self._finish(record, gate) + return + for host in record.get("hosts", []): if not isinstance(host, dict): continue @@ -3268,7 +4886,9 @@ class VpcCoordinator: fabric_plan_digest, private_link_states, fabric_peer_overlays=fabric_peer_overlays, request_deadline=request_deadline, - previous_host=host) + previous_host=host, + pending_member_mutations=( + self._pending_member_mutations(host))) if time.monotonic() >= request_deadline: member_nics = observed.get("member_nics") if isinstance(member_nics, list): @@ -3313,12 +4933,37 @@ class VpcCoordinator: # persistent removal proof. host["retiring_members"] = [] except Exception as exc: - host.update({"state": "uncertain", - "transport_verified": False, - "private_dataplane_verified": False, - "observed_revision": 0, - "observed_digest": None, - "last_error_code": self._observation_error(exc)}) + error_code = self._observation_error(exc) + if isinstance(exc, _VpcMemberPreflightError): + # This read-only admission refusal proves that no host or + # libvirt mutation started. Keep the target retryable as + # applying and retain its stop-before-write marker. + host.update({ + "state": "applying", + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": error_code, + }) + else: + execution = current_execution() + # Each SSH/libvirt mutation classifies its own lost or + # incomplete receipt by setting execution_unknown. A + # previously confirmed host receipt only leaves the + # historical remote_write_started bit set; it must not + # turn a later deterministic validation/admission error + # into an unrecoverable retained claim family. + host.update({"state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": error_code}) + if (execution is not None + and execution.execution_unknown.is_set()): + break + if complete_mutation_fence: + self._complete_member_mutation_fence(record, plan) self._finish(record, gate) @classmethod @@ -3331,6 +4976,28 @@ class VpcCoordinator: now = int(time.time()) if updated_at is None else updated_at previous_hosts = previous_hosts or {} host_records = [] + pending_mutations: dict[str, set[str]] = {} + for host_id, old_host in previous_hosts.items(): + if not isinstance(host_id, str) or not isinstance(old_host, Mapping): + continue + pending_mutations.setdefault(host_id, set()).update( + cls._pending_member_mutations(old_host)) + current_by_id = {member.member_id: member for member in plan.members} + previous_by_id = { + member.member_id: member + for member in (previous_plan.members if previous_plan else ()) + } + for member_id in set(current_by_id) | set(previous_by_id): + current_member = current_by_id.get(member_id) + previous_member = previous_by_id.get(member_id) + if (current_member is not None and previous_member is not None + and cls._member_boot_identity(plan, current_member) + == cls._member_boot_identity(previous_plan, previous_member)): + continue + for member in (current_member, previous_member): + if member is not None: + pending_mutations.setdefault( + member.host_id, set()).add(member.instance_uuid) current_members_by_host: dict[str, set[str]] = {} for member in plan.members: current_members_by_host.setdefault(member.host_id, set()).add( @@ -3417,6 +5084,8 @@ class VpcCoordinator: "wg_allowed_ips": old.get("wg_allowed_ips", []) if isinstance(old, Mapping) else [], "readback": safe_readback, "retiring_members": retiring_members, + "pending_member_mutations": sorted( + pending_mutations.get(host.host_id, set())), }) # Keep host tombstones in the durable engine record until the host-side # removal script returns its typed absence proof. They are excluded @@ -3488,6 +5157,9 @@ class VpcCoordinator: "config_observed_digest": None, "retiring": True, "retiring_members": old_members, + "pending_member_mutations": sorted( + pending_mutations.get(old_host_id, set()) + | {item["instance_uuid"] for item in old_members}), "bridge": None, "bridge_owner": None, "nft_table": None, "anti_spoof": False, "rp_filter": None, "proxy_arp": None, "proxy_arp_pvlan": None, @@ -3893,7 +5565,8 @@ class VpcCoordinator: host_id for host_id, host in durable_hosts.items() if ((host_id in member_host_ids and (host_id in targeted_host_ids - or not self._confirmed_active_host(host, plan))) + or not self._confirmed_active_host( + host, plan, allow_pending=True))) or (host_id not in member_host_ids and not self._confirmed_removal( host, plan.desired_revision, plan.desired_digest))) @@ -3915,11 +5588,14 @@ class VpcCoordinator: ) -> None: """Retry only hosts without an exact durable runtime/removal proof.""" self._finish(record, gate) + stop_guard_identities = self._member_mutation_guard_identities(record) if host_selection is None: host_selection = self._reconcile_host_selection( plan, record, private_link_states, extra_identities) pending_host_ids, targeted_host_ids = host_selection if not pending_host_ids: + self._complete_member_mutation_fence(record, plan) + self._finish(record, gate) return execution_record = dict(record) execution_hosts = [ @@ -3953,6 +5629,8 @@ class VpcCoordinator: host["host_id"] for host in execution_hosts if isinstance(host.get("host_id"), str) } + reconcile_kwargs["stop_guard_identities"] = stop_guard_identities + reconcile_kwargs["complete_mutation_fence"] = False await self._reconcile_dataplane( plan, execution_record, gate, previous_plan, **reconcile_kwargs) observations = { @@ -3968,6 +5646,8 @@ class VpcCoordinator: else host for host in record.get("hosts", []) ] + self._complete_member_mutation_fence(record, plan) + self._finish(record, gate) self._finish(record, gate) async def delete(self, vpc_id: str, value: Mapping[str, Any]) -> dict[str, Any]: @@ -4025,6 +5705,13 @@ class VpcCoordinator: if previous is None: self._validate_global_fabric_endpoints(plan) + if previous is not None: + durable_for_migration = _plan_from_record(previous) + if self._migrate_legacy_pending_member_mutations( + previous, durable_for_migration): + self._save("vpc_desired", vpc_id, previous) + self._save_children(previous) + marker = (self._delete_marker(previous) if previous is not None else None) transition_previous: VpcPlan | None = None @@ -4160,8 +5847,19 @@ class VpcCoordinator: for host in record.get("hosts", []): if (isinstance(host, dict) and isinstance(host.get("host_id"), str)): - host["retiring_members"] = list( + retiring_rows = list( retiring_rows_by_host.get(host["host_id"], [])) + host["retiring_members"] = retiring_rows + # A signed root-delete manifest may be the first + # durable Engine record for this VPC. Fence every VM + # in that manifest before any host/VM removal starts; + # relying only on a prior desired row would allow an + # unknown historical member on a later host to start + # after an earlier host had already been cleaned. + host["pending_member_mutations"] = sorted( + set(self._pending_member_mutations(host)) + | {row["instance_uuid"] + for row in retiring_rows}) marker = { "operation_id": plan.operation_id, "desired_revision": plan.desired_revision, @@ -4218,6 +5916,17 @@ class VpcCoordinator: pending_host_ids = required_host_ids - already_confirmed if pending_host_ids: gate = self._fabric_gate(plan) + # Keep the revision-wide marker durable until every host is + # exact, but do not re-lock, probe, or require connectivity + # from a host whose destructive absence proof is already + # complete. Its member XML/seed and host resources are gone; + # only the still-pending hosts can receive another write. + stop_guard_identities = tuple( + (host_id, instance_uuid) + for host_id, instance_uuid + in self._member_mutation_guard_identities(record) + if host_id in pending_host_ids + ) # The submitted plan may still contain a deployment whose # prior remove response was lost by PHP. Its exact durable # absence proof classifies it as preverified; never contact it @@ -4246,7 +5955,9 @@ class VpcCoordinator: execution_record["hosts"] = execution_hosts await self._reconcile_dataplane( plan, execution_record, gate, transition_previous, - operation_id=plan.operation_id) + operation_id=plan.operation_id, + stop_guard_identities=stop_guard_identities, + complete_mutation_fence=False) observations = { host.get("host_id"): { key: value for key, value in host.items() @@ -4263,6 +5974,8 @@ class VpcCoordinator: else host for host in record.get("hosts", []) ] + self._complete_member_mutation_fence(record, plan) + self._finish(record, gate) # Preserve partial/uncertain evidence before deciding whether # a receipt may be issued. self._save("vpc_desired", vpc_id, record) @@ -4318,14 +6031,21 @@ class VpcCoordinator: # durable receipt commit in flight. async with self._lock(vpc_id), self._operation_scope(plan), _pre_dataplane_lock( self._catalog_mutation_lock()): - return await _await_mutation_completion(delete_locked()) + async def admitted_delete() -> dict[str, Any]: + await self._recover_retained_guest_exec_cleanup(plan) + return await delete_locked() + + return await _await_mutation_completion(admitted_delete()) async def apply(self, value: Mapping[str, Any]) -> dict[str, Any]: plan = parse_vpc_plan(value) async with self._lock(plan.vpc_id), self._operation_scope(plan), _pre_dataplane_lock( self._catalog_mutation_lock()): - return await _await_mutation_completion( - self._apply_locked(plan)) + async def admitted_apply() -> dict[str, Any]: + await self._recover_retained_guest_exec_cleanup(plan) + return await self._apply_locked(plan) + + return await _await_mutation_completion(admitted_apply()) async def _apply_locked(self, plan: VpcPlan) -> dict[str, Any]: if self._load_delete_receipt(plan.vpc_id) is not None: @@ -4344,6 +6064,10 @@ class VpcCoordinator: # overwritten by a tenant retry, erasing the evidence needed # to reconcile an interrupted operation. previous_plan = _plan_from_record(previous) + if self._migrate_legacy_pending_member_mutations( + previous, previous_plan): + self._save("vpc_desired", plan.vpc_id, previous) + self._save_children(previous) old_revision = previous.get("desired_revision") old_digest = previous.get("desired_digest") if (old_revision != previous_plan.desired_revision @@ -4421,6 +6145,12 @@ class VpcCoordinator: previous_plan=previous_plan, ) self._finish(record, gate) + # Commit the immutable new target and its child closure before the + # first SSH/libvirt mutation. If the process or HTTP response dies + # after a host write, restart recovery must see this exact revision + # instead of replaying the previous target against changed hardware. + self._save("vpc_desired", plan.vpc_id, record) + self._save_children(record) await self._reconcile_dataplane( plan, record, gate, previous_plan) self._save("vpc_desired", plan.vpc_id, record) @@ -4939,6 +6669,9 @@ class VpcCoordinator: desired_digest=expected_digest, host=None, member_nic=None, reason_code="vpc_private_expectation_unavailable") plan = _plan_from_record(record) + if self._migrate_legacy_pending_member_mutations(record, plan): + self._save("vpc_desired", key, record) + self._save_children(record) member = next((item for item in plan.members if item.instance_uuid == instance_uuid), None) target_row: Mapping[str, Any] | None = None @@ -5145,6 +6878,9 @@ class VpcCoordinator: if self._delete_marker(current) is not None: return None, self._delete_payload_from_record(current) plan = _plan_from_record(current) + if self._migrate_legacy_pending_member_mutations(current, plan): + self._save("vpc_desired", vpc_id, current) + self._save_children(current) if current.get("desired_digest") != plan.desired_digest: raise VpcError("VPC 计划摘要与持久化记录不一致", 500, "vpc_record_corrupt") @@ -5168,10 +6904,46 @@ class VpcCoordinator: result, delete_payload = None, self._delete_payload_from_record(identity) else: plan = _plan_from_record(identity) - async with self._operation_scope(plan), _pre_dataplane_lock( - self._catalog_mutation_lock()): - result, delete_payload = await _await_mutation_completion( - reconcile_locked()) + if self._migrate_legacy_pending_member_mutations( + identity, plan): + self._save("vpc_desired", vpc_id, identity) + self._save_children(identity) + if identity.get("desired_digest") != plan.desired_digest: + raise VpcError( + "VPC 计划摘要与持久化记录不一致", 500, + "vpc_record_corrupt") + self._check_operation_identity(plan) + gate = self._fabric_gate(plan) + self._finish(identity, gate) + host_selection = self._reconcile_host_selection( + plan, identity) + execution = current_execution() + explicit_reconcile = bool( + execution is not None + and execution.claim.operation_id is None + and tuple(execution.claim.resources) == () + and execution.claim.kind == "request" + and execution.claim.metadata + == {"explicit_vpc_reconcile": True} + ) + if not host_selection[0] and not explicit_reconcile: + # Configuration-complete rows need no operation/catalog/VM + # claim merely to repeat stopped/QGA health observations. + # Lifecycle-triggered reads still use reconcile_instance. + # An explicit request is the exception: it may need to + # atomically close an already observed retained family. + self._save("vpc_desired", vpc_id, identity) + self._save_children(identity) + result, delete_payload = self._public(identity), None + else: + async with self._operation_scope(plan), _pre_dataplane_lock( + self._catalog_mutation_lock()): + async def admitted_reconcile(): + await self._recover_retained_guest_exec_cleanup(plan) + return await reconcile_locked() + + result, delete_payload = await _await_mutation_completion( + admitted_reconcile()) # The locked recheck may observe a delete target that was installed # after the optimistic read. Release the non-reentrant lock before # entering ``delete``, which acquires the same per-VPC and catalog @@ -5203,6 +6975,26 @@ class VpcCoordinator: # Do not move or rewrite children under a different VPC key # when a durable row has been tampered with. continue + try: + recovery_plan = _plan_from_record(value) + self._migrate_legacy_pending_member_mutations( + value, recovery_plan) + except VpcError: + value["state"] = "uncertain" + value["next_stage_allowed"] = False + value["private_dataplane_verified"] = False + value["recovery_pending"] = True + for host in value.get("hosts", []): + if isinstance(host, dict): + host.update({ + "state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "last_error_code": "vpc_record_corrupt", + }) + value["updated_at"] = int(time.time()) + self._save("vpc_desired", key, value) + continue value["state"] = "uncertain" value["next_stage_allowed"] = False value["private_dataplane_verified"] = False diff --git a/engine/virsh_panel/vpc_dataplane.py b/engine/virsh_panel/vpc_dataplane.py index 258c8d13..2ee73d8b 100644 --- a/engine/virsh_panel/vpc_dataplane.py +++ b/engine/virsh_panel/vpc_dataplane.py @@ -20,6 +20,7 @@ from dataclasses import dataclass from threading import Lock from typing import Any, Mapping, Sequence +from .coordination import mark_execution_unknown, mark_remote_write_started from .network_roles import ( PRIVATE_INTERFACE_ROLE, PRIVATE_LIBVIRT_ALIAS, @@ -915,6 +916,223 @@ vc_nm_profile_expect() { '''.strip() +NETWORKMANAGER_VPC_TARGET_INVENTORY_SH = r''' +vc_nm_inventory_capture() { + vc_nm_capture_mode=$1 + vc_nm_capture_field=${2-} + vc_nm_capture_uuid=${3-} + vc_nm_capture_nl=' +' + vc_nm_capture_marker=__VC_NM_INVENTORY_CAPTURE_END__ + if [ "$vc_nm_capture_mode" = list ]; then + vc_nm_capture_raw=$( + if "$NMCLI" -t --escape no -f UUID connection show 2>/dev/null; then + vc_nm_capture_status=0 + else + vc_nm_capture_status=$? + fi + printf '\n%s:%s' "$vc_nm_capture_marker" "$vc_nm_capture_status" + ) + else + vc_nm_capture_raw=$( + if "$NMCLI" -t --escape no -g "$vc_nm_capture_field" \ + connection show uuid "$vc_nm_capture_uuid" 2>/dev/null; then + vc_nm_capture_status=0 + else + vc_nm_capture_status=$? + fi + printf '\n%s:%s' "$vc_nm_capture_marker" "$vc_nm_capture_status" + ) + fi + vc_nm_capture_suffix=${vc_nm_capture_raw##*"$vc_nm_capture_nl$vc_nm_capture_marker:"} + case "$vc_nm_capture_suffix" in + ''|*[!0-9]*) VC_NM_CAPTURE_ERROR=invalid; return 1 ;; + 0) : ;; + *) VC_NM_CAPTURE_ERROR=failed; return 1 ;; + esac + VC_NM_CAPTURE=${vc_nm_capture_raw%"$vc_nm_capture_nl$vc_nm_capture_marker:$vc_nm_capture_suffix"} + VC_NM_CAPTURE_ERROR=invalid + if [ -n "$VC_NM_CAPTURE" ]; then + case "$VC_NM_CAPTURE" in + *"$vc_nm_capture_nl") + VC_NM_CAPTURE=${VC_NM_CAPTURE%"$vc_nm_capture_nl"} + ;; + *) return 1 ;; + esac + elif [ "$vc_nm_capture_mode" != list ]; then + # A scalar query must emit exactly one row. nmcli represents an unset + # field as an empty row or "--"; no output is an ambiguous response. + return 1 + fi + if [ "$vc_nm_capture_mode" != list ]; then + case "$VC_NM_CAPTURE" in *"$vc_nm_capture_nl"*) return 1 ;; esac + fi + VC_NM_CAPTURE_ERROR=none +} +vc_nm_inventory_uuid_set() { + vc_nm_uuid_rows=$1 + VC_NM_UUID_SET= + [ -n "$vc_nm_uuid_rows" ] || return 0 + vc_nm_uuid_rows=$(printf '%s\n' "$vc_nm_uuid_rows" | awk ' + NF != 1 || $0 !~ /^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$/ {bad=1; next} + {key=tolower($0); if (seen[key]++) bad=1; print} + END {if (bad) exit 1} + ') || return 1 + VC_NM_UUID_SET=$(printf '%s\n' "$vc_nm_uuid_rows" | LC_ALL=C sort) || + return 1 +} +vc_nm_target_inventory() { + NM_TARGET_COUNT=invalid + NM_TARGET_QUERY=invalid + NM_TARGET_UUID= + NM_TARGET_ID= + NM_TARGET_IFACE= + NM_TARGET_STABLE= + NM_TARGET_TYPE= + NM_TARGET_EXACT=no + VC_NM_CAPTURE_ERROR=invalid + if ! vc_nm_inventory_capture list; then + NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR + return 1 + fi + vc_nm_inventory_uuid_set "$VC_NM_CAPTURE" || { + NM_TARGET_QUERY=invalid + return 1 + } + vc_nm_inventory_uuids=$VC_NM_UUID_SET + vc_nm_target_count=0 + while IFS= read -r vc_nm_inventory_uuid; do + [ -n "$vc_nm_inventory_uuid" ] || continue + vc_nm_inventory_capture scalar connection.id "$vc_nm_inventory_uuid" || { + NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR + return 1 + } + vc_nm_inventory_id=$VC_NM_CAPTURE + [ -n "$vc_nm_inventory_id" ] && [ "$vc_nm_inventory_id" != -- ] || { + NM_TARGET_QUERY=invalid + return 1 + } + vc_nm_inventory_capture scalar connection.interface-name \ + "$vc_nm_inventory_uuid" || { + NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR + return 1 + } + vc_nm_inventory_iface=$VC_NM_CAPTURE + vc_nm_inventory_capture scalar connection.stable-id \ + "$vc_nm_inventory_uuid" || { + NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR + return 1 + } + vc_nm_inventory_stable=$VC_NM_CAPTURE + vc_nm_inventory_capture scalar connection.type "$vc_nm_inventory_uuid" || { + NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR + return 1 + } + vc_nm_inventory_type=$VC_NM_CAPTURE + if [ "$vc_nm_inventory_id" = "$BRIDGE" ] || + [ "$vc_nm_inventory_iface" = "$BRIDGE" ] || + [ "$vc_nm_inventory_stable" = "$NM_PROFILE_MARKER" ]; then + vc_nm_target_count=$((vc_nm_target_count + 1)) + if [ "$vc_nm_target_count" = 1 ]; then + NM_TARGET_UUID=$vc_nm_inventory_uuid + NM_TARGET_ID=$vc_nm_inventory_id + NM_TARGET_IFACE=$vc_nm_inventory_iface + NM_TARGET_STABLE=$vc_nm_inventory_stable + NM_TARGET_TYPE=$vc_nm_inventory_type + fi + fi + done <<__VC_NM_INVENTORY_UUIDS__ +$vc_nm_inventory_uuids +__VC_NM_INVENTORY_UUIDS__ + # Re-list after reading every profile. A concurrent add/remove/reload must + # not turn a partial inventory into an absence receipt. + if ! vc_nm_inventory_capture list; then + NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR + return 1 + fi + vc_nm_inventory_uuid_set "$VC_NM_CAPTURE" || { + NM_TARGET_QUERY=invalid + return 1 + } + [ "$VC_NM_UUID_SET" = "$vc_nm_inventory_uuids" ] || { + NM_TARGET_QUERY=invalid + return 1 + } + NM_TARGET_COUNT=$vc_nm_target_count + NM_TARGET_QUERY=ok + if [ "$NM_TARGET_COUNT" = 1 ] && + [ "$NM_TARGET_ID" = "$BRIDGE" ] && + [ "$NM_TARGET_IFACE" = "$BRIDGE" ] && + [ "$NM_TARGET_STABLE" = "$NM_PROFILE_MARKER" ] && + [ "$NM_TARGET_TYPE" = bridge ]; then + NM_TARGET_EXACT=yes + fi +} +'''.strip() + + +IFUPDOWN_VPC_TARGET_INVENTORY_SH = r''' +vc_ifupdown_scan_file() { + vc_ifup_scan_file=$1 + [ -f "$vc_ifup_scan_file" ] && [ ! -L "$vc_ifup_scan_file" ] && + [ -r "$vc_ifup_scan_file" ] || return 1 + VC_IFUP_SCAN_COUNT=$(awk -v wanted="$BRIDGE" ' + /^[[:space:]]*#/ || NF == 0 {next} + $1 == "iface" { + if (NF < 4) {bad=1; next} + if ($2 == wanted) found++ + next + } + $1 == "auto" || $1 ~ /^allow-[^[:space:]]+$/ { + if (NF < 2) {bad=1; next} + for (i=2; i<=NF; i++) { + if ($i ~ /^#/) break + if ($i == wanted) found++ + } + } + END {if (bad) exit 1; print found + 0} + ' "$vc_ifup_scan_file") || return 1 + case "$VC_IFUP_SCAN_COUNT" in ''|*[!0-9]*) return 1 ;; esac +} +vc_ifupdown_inventory_once() { + vc_ifup_root=${IFUP_ROOT:-/etc/network} + vc_ifup_main=${IFUP_MAIN:-$vc_ifup_root/interfaces} + vc_ifup_dir=${IFUP_DIR:-$vc_ifup_root/interfaces.d} + if [ -e "$vc_ifup_root" ] || [ -L "$vc_ifup_root" ]; then + [ -d "$vc_ifup_root" ] && [ ! -L "$vc_ifup_root" ] && + [ -r "$vc_ifup_root" ] && [ -x "$vc_ifup_root" ] || return 1 + fi + vc_ifup_total=0 + if [ -e "$vc_ifup_main" ] || [ -L "$vc_ifup_main" ]; then + vc_ifupdown_scan_file "$vc_ifup_main" || return 1 + vc_ifup_total=$((vc_ifup_total + VC_IFUP_SCAN_COUNT)) + fi + if [ -e "$vc_ifup_dir" ] || [ -L "$vc_ifup_dir" ]; then + [ -d "$vc_ifup_dir" ] && [ ! -L "$vc_ifup_dir" ] && + [ -r "$vc_ifup_dir" ] && [ -x "$vc_ifup_dir" ] || return 1 + for vc_ifup_candidate in "$vc_ifup_dir"/*; do + if [ ! -e "$vc_ifup_candidate" ] && [ ! -L "$vc_ifup_candidate" ]; then + continue + fi + vc_ifupdown_scan_file "$vc_ifup_candidate" || return 1 + vc_ifup_total=$((vc_ifup_total + VC_IFUP_SCAN_COUNT)) + done + fi + VC_IFUP_TARGET_COUNT=$vc_ifup_total +} +vc_ifupdown_target_inventory() { + IFUP_TARGET_COUNT=invalid + IFUP_TARGET_QUERY=invalid + vc_ifupdown_inventory_once || return 1 + vc_ifup_first_count=$VC_IFUP_TARGET_COUNT + vc_ifupdown_inventory_once || return 1 + [ "$VC_IFUP_TARGET_COUNT" = "$vc_ifup_first_count" ] || return 1 + IFUP_TARGET_COUNT=$VC_IFUP_TARGET_COUNT + IFUP_TARGET_QUERY=ok +} +'''.strip() + + class VpcDataplaneError(Exception): """Safe, typed error at the VPC host boundary.""" @@ -3422,6 +3640,8 @@ EXPECTED_DIGEST={_q(plan.fabric_plan_digest)} {NFT_LIVE_SNAPSHOT_SH} {_NMCLI_FILENAME_RESOLVER_SH} {NETWORKMANAGER_VPC_PROFILE_SH} +{NETWORKMANAGER_VPC_TARGET_INVENTORY_SH} +{IFUPDOWN_VPC_TARGET_INVENTORY_SH} {_FIREWALL_MODE_GUARD_SH} {_FABRIC_NFT_VALIDATION_SH} {_VPC_SYSCTL_VALIDATION_SH} @@ -3775,35 +3995,52 @@ else [ ! -L "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] || exit 79 done fi +NM_REMOVE_UUID= if [ "$MODE" = networkmanager_nmcli ]; then if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then - if vc_nm_profile_resolve; then - vc_nm_profile_base_validate || exit 71 - vc_nm_profile_empty connection.zone || exit 71 - else - nm_resolve_status=$? - [ "$nm_resolve_status" = 2 ] || exit 71 - if [ "$BRIDGE_PRESENT" = yes ]; then - # An unmanaged live bridge is foreign/ambiguous; do not delete it. - exit 71 - fi - fi + vc_nm_target_inventory || exit 71 + case "$NM_TARGET_COUNT" in + 0) + if [ "$BRIDGE_PRESENT" = yes ]; then + # An unmanaged live bridge is foreign/ambiguous; do not delete it. + exit 71 + fi + ;; + 1) + # Name, interface and owner marker must all identify the same unique + # profile. A renamed or colliding profile remains visible but cannot + # be deleted automatically. + [ "$NM_TARGET_EXACT" = yes ] || exit 71 + vc_nm_profile_resolve || exit 71 + [ "$NM_PROFILE_UUID" = "$NM_TARGET_UUID" ] || exit 71 + vc_nm_profile_base_validate || exit 71 + vc_nm_profile_empty connection.zone || exit 71 + NM_REMOVE_UUID=$NM_PROFILE_UUID + ;; + *) exit 71 ;; + esac else - if "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then exit 79; fi + vc_nm_target_inventory || exit 79 + [ "$NM_TARGET_COUNT" = 0 ] || exit 79 fi else if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then if [ "$BRIDGE_PRESENT" = yes ] || [ -e "$IF_FRAGMENT" ]; then [ -f "$IF_FRAGMENT" ] || exit 71 - [ ! -L /etc/network ] && [ ! -L /etc/network/interfaces ] && [ ! -L /etc/network/interfaces.d ] || exit 71 - if [ -f /etc/network/interfaces ] && - grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+|iface)[[:space:]]+'"$BRIDGE"'([[:space:]]|$)' /etc/network/interfaces; then - exit 71 - fi "$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71 fi fi fi +vc_ifupdown_target_inventory || exit 71 +if [ "$PRE_FILES" = yes ]; then + if [ "$MODE" = ifupdown ] && [ -f "$IF_FRAGMENT" ]; then + [ "$IFUP_TARGET_COUNT" = 1 ] || exit 71 + else + [ "$IFUP_TARGET_COUNT" = 0 ] || exit 71 + fi +else + [ "$IFUP_TARGET_COUNT" = 0 ] || exit 79 +fi if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ] || [ "$PHASE" = manager_removed ]; then if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then [ -f "$OWNER_FILE" ] || exit 71 @@ -3983,12 +4220,26 @@ __VC_REMOVE_LOCAL_NEIGHBORS__ fi # vc-local-neighbor-remove-cleanup-end if [ "$MODE" = networkmanager_nmcli ]; then - if [ -n "$NMCLI" ] && "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then - "$NMCLI" connection delete "$BRIDGE" >/dev/null 2>&1 || exit 72 - if "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then exit 72; fi - fi + vc_nm_target_inventory || exit 71 + case "$NM_TARGET_COUNT" in + 0) : ;; + 1) + [ -n "$NM_REMOVE_UUID" ] && [ "$NM_TARGET_EXACT" = yes ] && + [ "$NM_TARGET_UUID" = "$NM_REMOVE_UUID" ] || exit 71 + vc_nm_profile_resolve || exit 71 + [ "$NM_PROFILE_UUID" = "$NM_REMOVE_UUID" ] || exit 71 + vc_nm_profile_base_validate || exit 71 + vc_nm_profile_empty connection.zone || exit 71 + "$NMCLI" connection delete uuid "$NM_REMOVE_UUID" >/dev/null 2>&1 || exit 72 + vc_nm_target_inventory || exit 72 + [ "$NM_TARGET_COUNT" = 0 ] || exit 72 + ;; + *) exit 71 ;; + esac else if [ -f "$IF_FRAGMENT" ]; then + vc_ifupdown_target_inventory || exit 71 + [ "$IFUP_TARGET_COUNT" = 1 ] || exit 71 "$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71 "$IFDOWN" --force "$BRIDGE" >/dev/null 2>&1 || exit 72 fi @@ -4164,9 +4415,12 @@ for family in inet bridge arp; do "$NFT" list table "$family" "$TABLE" >/dev/null 2>&1 && exit 79 || true done grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 79 || true -if [ "$MODE" = networkmanager_nmcli ] && [ -n "$NMCLI" ]; then - "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1 && exit 79 || true +if [ "$MODE" = networkmanager_nmcli ]; then + vc_nm_target_inventory || exit 79 + [ "$NM_TARGET_COUNT" = 0 ] || exit 79 fi +vc_ifupdown_target_inventory || exit 79 +[ "$IFUP_TARGET_COUNT" = 0 ] || exit 79 [ ! -e "$STATE_FILE" ] && [ ! -e "$OWNER_FILE" ] && [ ! -e "$NFT_SNAPSHOT_PREFIX.inet.live" ] && [ ! -e "$NFT_SNAPSHOT_PREFIX.bridge.live" ] && @@ -4202,8 +4456,12 @@ render_apply_script = _render_apply_script_v2 def render_observe_script( plan: VpcDataplanePlan, *, + deployment_mode: str = "networkmanager_nmcli", firewall_mode: str = FIREWALL_MODE_NATIVE, ) -> str: + if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}: + raise VpcDataplaneError("宿主机网络管理模式不支持", 409, + "vpc_dataplane_network_manager_conflict") firewall_mode = _firewall_mode(firewall_mode) from .fabric import ( _FABRIC_NFT_VALIDATION_SH, @@ -4361,7 +4619,7 @@ for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do done BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER={_q(r.owner_file)}; STATE={_q(r.state_file)}; TXN={_q(r.transaction_dir)}; PROXY_FILE={_q(proxy_file)} NFT_SNAPSHOT_PREFIX={_q(f'/etc/virt-cloud/vpc-dataplane/{r.token}.nft')} -NFT_FILE={_q(nft_file)}; NFT_INCLUDE={_q(nft_include)}; LEGACY_MANIFEST={_q(LEGACY_FIREWALLD_NFT_MANIFEST)}; SYSCTL_FILE={_q(sysctl_file)}; IF_FRAGMENT={_q(ifup_fragment)}; NM_PROFILE_MARKER={_q(nm_profile_marker)}; FIREWALL_MODE={_q(firewall_mode)}; NFT_MAIN=; NFT_MAIN_COUNT=0; NM_PROFILE_UUID=; NM_PROFILE_FILE= +NFT_FILE={_q(nft_file)}; NFT_INCLUDE={_q(nft_include)}; LEGACY_MANIFEST={_q(LEGACY_FIREWALLD_NFT_MANIFEST)}; SYSCTL_FILE={_q(sysctl_file)}; IF_FRAGMENT={_q(ifup_fragment)}; NM_PROFILE_MARKER={_q(nm_profile_marker)}; MODE={_q(deployment_mode)}; FIREWALL_MODE={_q(firewall_mode)}; NFT_MAIN=; NFT_MAIN_COUNT=0; NM_PROFILE_UUID=; NM_PROFILE_FILE= NFT_MARKER={_q(nft_marker)} EXPECTED_DIGEST={_q(plan.fabric_plan_digest)} {NFT_LIVE_SNAPSHOT_SH} @@ -4370,6 +4628,8 @@ EXPECTED_DIGEST={_q(plan.fabric_plan_digest)} SHA256SUM=$(command -v sha256sum || true) {_NMCLI_FILENAME_RESOLVER_SH} {NETWORKMANAGER_VPC_PROFILE_SH} +{NETWORKMANAGER_VPC_TARGET_INVENTORY_SH} +{IFUPDOWN_VPC_TARGET_INVENTORY_SH} {_FIREWALL_MODE_GUARD_SH} {_FABRIC_NFT_VALIDATION_SH} {_VPC_SYSCTL_VALIDATION_SH} @@ -4393,8 +4653,8 @@ vc_vpc_observe_cleanup() {{ }} trap 'vc_vpc_observe_cleanup' EXIT trap 'exit 79' HUP INT TERM -vc_vpc_fabric_observe_ready() {{ - [ "$VPC_STATE_STATUS" = owned ] || return 1 +vc_vpc_fabric_closure_ready() {{ + case "$VPC_STATE_STATUS" in owned|empty) : ;; *) return 1 ;; esac vc_nft_table_snapshot inet virt_cloud_wg_fabric \ "$FABRIC_NFT_SNAPSHOT" || return 1 [ -f "$FABRIC_NFT_SNAPSHOT" ] && @@ -4409,6 +4669,19 @@ vc_vpc_fabric_observe_ready() {{ vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \ "$FABRIC_NFT_SNAPSHOT" }} +vc_vpc_target_nft_absent() {{ + for vc_absent_family in inet bridge arp; do + vc_absent_inventory=$("$NFT" -s -n -y list tables \ + "$vc_absent_family") || return 1 + vc_absent_count=$(printf '%s\n' "$vc_absent_inventory" | + awk -v family="$vc_absent_family" -v wanted="$TABLE" ' + NF==0 {{next}} + NF!=3 || $1!="table" || $2!=family || $3!~/^[A-Za-z0-9_.-]+$/ {{bad=1}} + $3==wanted {{found++}} + END {{if (bad || found>1) exit 1; print found+0}}') || return 1 + [ "$vc_absent_count" = 0 ] || return 1 + done +}} firewall_mode_state=owned vc_firewall_mode_ready || firewall_mode_state=mismatch [ "$FIREWALL_MODE" = native_nftables ] || firewall_mode_state=mismatch @@ -4446,29 +4719,53 @@ fi nm_owned=no nm_zone= nm_present=no -# NetworkManager is optional on ifupdown hosts. An exhausted discovery loop -# must succeed with an empty value under set -e so ownership can be observed. -NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do - if [ -x "$p" ]; then printf '%s' "$p"; break; fi -done) -nm_profile_count=0 -if [ -n "$NMCLI" ]; then - nm_profile_count=$($NMCLI -t --escape no -f NAME connection show 2>/dev/null | - awk -v wanted="$BRIDGE" '$0 == wanted {{count++}} END {{print count + 0}}') || nm_profile_count=invalid - case "$nm_profile_count" in ''|*[!0-9]*) nm_profile_count=invalid; nm_present=yes ;; esac - if [ "$nm_profile_count" != invalid ] && [ "$nm_profile_count" -gt 0 ]; then nm_present=yes; fi -fi -if [ -n "$NMCLI" ] && [ "$nm_profile_count" = 1 ]; then - nm_stable=$($NMCLI -t --escape no -g connection.stable-id connection show "$BRIDGE" 2>/dev/null | sed -n '1p') - nm_type=$($NMCLI -t --escape no -g connection.type connection show "$BRIDGE" 2>/dev/null | sed -n '1p') - nm_iface=$($NMCLI -t --escape no -g connection.interface-name connection show "$BRIDGE" 2>/dev/null | sed -n '1p') - nm_method=$($NMCLI -t --escape no -g ipv4.method connection show "$BRIDGE" 2>/dev/null | sed -n '1p') - nm6_method=$($NMCLI -t --escape no -g ipv6.method connection show "$BRIDGE" 2>/dev/null | sed -n '1p') - nm_zone=$($NMCLI -t --escape no -g connection.zone connection show "$BRIDGE" 2>/dev/null | sed -n '1p') - if [ "$nm_stable" = "$NM_PROFILE_MARKER" ] && [ "$nm_type" = bridge ] && [ "$nm_iface" = "$BRIDGE" ] && [ "$nm_method" = disabled ] && [ "$nm6_method" = disabled ]; then - if vc_nm_profile_resolve && vc_nm_profile_expect active; then nm_owned=yes; fi +# The pinned deployment mode decides which persistence inventory is +# authoritative. In NetworkManager mode, a missing binary or failed daemon +# query is unknown rather than proof that the profile is absent. Do not put +# nmcli on the left of a pipeline: POSIX sh would report awk's success and +# silently turn a failed query into count=0. +nm_profile_count=invalid +nm_profile_query=unavailable +NMCLI= +if [ "$MODE" = networkmanager_nmcli ]; then + NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do + if [ -x "$p" ]; then printf '%s' "$p"; break; fi + done) + if [ -n "$NMCLI" ]; then + if vc_nm_target_inventory; then + nm_profile_query=$NM_TARGET_QUERY + nm_profile_count=$NM_TARGET_COUNT + else + nm_profile_query=$NM_TARGET_QUERY + nm_profile_count=invalid + fi fi + case "$nm_profile_count" in + ''|*[!0-9]*) nm_profile_count=invalid; nm_present=yes ;; + 0) : ;; + *) nm_present=yes ;; + esac +else + # ifupdown persistence is proved independently by IF_FRAGMENT. A host + # pinned to this mode is not required to run a NetworkManager daemon. + nm_profile_count=0 + nm_profile_query=not_applicable +fi +if [ "$nm_profile_query" = ok ] && [ "$nm_profile_count" = 1 ] && + [ "$NM_TARGET_EXACT" = yes ]; then + if vc_nm_profile_resolve && [ "$NM_PROFILE_UUID" = "$NM_TARGET_UUID" ] && + vc_nm_profile_expect active; then nm_owned=yes; fi fi +printf '%s\\n' "__VC_VPC_NM_PROFILE_COUNT__=$nm_profile_count" +printf '%s\\n' "__VC_VPC_NM_PROFILE_QUERY__=$nm_profile_query" +ifup_profile_count=invalid +ifup_profile_query=invalid +if vc_ifupdown_target_inventory; then + ifup_profile_count=$IFUP_TARGET_COUNT + ifup_profile_query=$IFUP_TARGET_QUERY +fi +printf '%s\\n' "__VC_VPC_IFUP_PROFILE_COUNT__=$ifup_profile_count" +printf '%s\\n' "__VC_VPC_IFUP_PROFILE_QUERY__=$ifup_profile_query" ifup_owned=no if [ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] && [ "$(stat -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] && grep -Fqx '# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$IF_FRAGMENT" && @@ -4491,11 +4788,45 @@ if [ "$firewall_mode_state" = owned ] && [ "$NFT_MAIN_COUNT" = 1 ] && [ "$persistence_target" = yes ] && [ "$opposite_clean" = yes ] && [ "$nft_file_ok" = yes ] && [ "$sysctl_file_ok" = yes ] && [ "$nft_snapshot_state" = yes ] && - ( {{ [ "$nm_owned" = yes ] && [ "$ifup_owned" = no ]; }} || - {{ [ "$nm_owned" = no ] && [ "$nm_present" = no ] && [ "$ifup_owned" = yes ]; }} ); then + ( {{ [ "$MODE" = networkmanager_nmcli ] && + [ "$nm_profile_query" = ok ] && [ "$nm_profile_count" = 1 ] && + [ "$nm_owned" = yes ] && [ "$ifup_owned" = no ] && + [ "$ifup_profile_query" = ok ] && [ "$ifup_profile_count" = 0 ]; }} || + {{ [ "$MODE" = ifupdown ] && + [ "$nm_profile_query" = not_applicable ] && [ "$nm_profile_count" = 0 ] && + [ "$nm_owned" = no ] && [ "$nm_present" = no ] && + [ "$ifup_profile_query" = ok ] && [ "$ifup_profile_count" = 1 ] && + [ "$ifup_owned" = yes ]; }} ); then persistence=yes fi printf '%s\\n' "__VC_VPC_PERSISTENCE__=$persistence" +for artifact_spec in \ + "NFT_FILE|$NFT_FILE" "SYSCTL_FILE|$SYSCTL_FILE" \ + "PROXY_FILE|$PROXY_FILE" "IF_FRAGMENT|$IF_FRAGMENT" \ + "OWNER_FILE|$OWNER" "STATE_FILE|$STATE" "TRANSACTION|$TXN"; do + artifact_name=${{artifact_spec%%|*}} + artifact_path=${{artifact_spec#*|}} + artifact_present=no + if [ -e "$artifact_path" ] || [ -L "$artifact_path" ]; then + artifact_present=yes + fi + printf '%s\\n' "__VC_VPC_${{artifact_name}}_PRESENT__=$artifact_present" +done +printf '%s\\n' "__VC_VPC_NFT_MAIN_COUNT__=$NFT_MAIN_COUNT" +nft_include_present=no +if [ "$NFT_MAIN_COUNT" = 1 ] && grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN"; then + nft_include_present=yes +fi +printf '%s\\n' "__VC_VPC_NFT_INCLUDE_PRESENT__=$nft_include_present" +removal_bridge_absent=no +if removal_link_inventory=$("$IP" -o link show 2>/dev/null) && + printf '%s\\n' "$removal_link_inventory" | awk -v wanted="$BRIDGE" ' + NF<2 || $1!~/^[0-9]+:$/ {{bad=1; next}} + {{name=$2; sub(/:$/, "", name); sub(/@.*/, "", name); if (name==wanted) found++}} + END {{exit (bad || found) ? 1 : 0}}'; then + removal_bridge_absent=yes +fi +printf '%s\\n' "__VC_VPC_REMOVAL_BRIDGE_ABSENT__=$removal_bridge_absent" if "$IP" -d link show dev "$BRIDGE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_BRIDGE__='"$BRIDGE"; else printf '%s\\n' '__VC_VPC_BRIDGE__=missing'; fi bridge_boundary=no bridge_shape=no; bridge_addr_ok=no; bridge_ports_ok=no; bridge_routes_ok=no; bridge_routes6_ok=no @@ -4601,6 +4932,13 @@ printf '%s\\n' "__VC_VPC_PERSISTENCE_EXACT__=$persistence_exact" vc_observe_nft_receipt=$(vc_vpc_nft_receipt) || exit 74 printf '%s\\n' "__VC_VPC_NFT_RECEIPT__=$vc_observe_nft_receipt" # vc-apply-receipt-observe-end +nft_table_count=0 +for nft_family in inet bridge arp; do + if "$NFT" list table "$nft_family" "$TABLE" >/dev/null 2>&1; then + nft_table_count=$((nft_table_count + 1)) + fi +done +printf '%s\\n' "__VC_VPC_NFT_TABLE_COUNT__=$nft_table_count" if "$NFT" list table inet "$TABLE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_NFT_TABLE__='"$TABLE"; else printf '%s\\n' '__VC_VPC_NFT_TABLE__=missing'; fi if [ -f "$STATE" ] && grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE"; then printf '%s\\n' '__VC_VPC_STATE_OWNER__=owned' @@ -4643,7 +4981,16 @@ if route_inventory=$("$IP" -o -4 -N route show table "$ROUTE_TABLE" 2>/dev/null) else route_records=invalid fi -rule_records=$("$IP" -o -4 -N rule show 2>/dev/null || true) +route_scope_count=$(printf '%s\\n' "$route_records" | awk 'NF {{count++}} END {{print count + 0}}') +printf '%s\\n' "__VC_VPC_ROUTE_SCOPE_COUNT__=$route_scope_count" +removal_route_absent=no +if [ "$route_scope_count" = 0 ]; then removal_route_absent=yes; fi +printf '%s\\n' "__VC_VPC_REMOVAL_ROUTE_ABSENT__=$removal_route_absent" +rule_inventory_valid=yes +rule_records=$("$IP" -o -4 -N rule show 2>/dev/null) || {{ + rule_inventory_valid=no + rule_records= +}} rule_scope_count=$(printf '%s\\n' "$rule_records" | awk -v expected_to={_q(plan.vpc.cidr)} -v expected_table={_q(str(r.route_table))} '{{ destination=""; table="" for (i=1;i<=NF;i++) {{ @@ -4652,6 +4999,12 @@ rule_scope_count=$(printf '%s\\n' "$rule_records" | awk -v expected_to={_q(plan. }} if (destination == expected_to || table == expected_table) count++ }} END {{print count + 0}}') +printf '%s\\n' "__VC_VPC_RULE_SCOPE_COUNT__=$rule_scope_count" +removal_rule_absent=no +if [ "$rule_inventory_valid" = yes ] && [ "$rule_scope_count" = 0 ]; then + removal_rule_absent=yes +fi +printf '%s\\n' "__VC_VPC_REMOVAL_RULE_ABSENT__=$removal_rule_absent" rule_extra=yes; [ "$rule_scope_count" = 2 ] && rule_extra=no printf '%s\\n' "__VC_VPC_RULE_EXTRA__=$rule_extra" {rule_checks} @@ -4664,9 +5017,25 @@ route_extra=$(printf '%s\\n' "$route_records" | awk -F'|' '{{ printf '%s\\n' "__VC_VPC_ROUTE_EXTRA__=$route_extra" {route_checks} vpc_state_loaded=no -if vpc_state_load && [ "$VPC_STATE_STATUS" = owned ]; then - vpc_state_loaded=yes +removal_state_closure=invalid +removal_target_state_absent_before=no +if [ ! -e "$STATE" ] && [ ! -L "$STATE" ]; then + removal_target_state_absent_before=yes fi +if vpc_state_load; then + if [ "$VPC_STATE_STATUS" = owned ]; then + vpc_state_loaded=yes + fi + case "$VPC_STATE_STATUS" in + owned|empty) + if [ "$removal_target_state_absent_before" = yes ] && + [ ! -e "$STATE" ] && [ ! -L "$STATE" ]; then + removal_state_closure=$VPC_STATE_STATUS + fi + ;; + esac +fi +printf '%s\n' "__VC_VPC_REMOVAL_STATE_CLOSURE__=$removal_state_closure" while IFS='|' read -r peer_host key expected_overlay; do [ -n "$peer_host" ] || continue actual=$(vc_wg_allowed_for_peer "$WG_IFACE" "$key") || actual=invalid @@ -4697,42 +5066,46 @@ done <<'{peer_marker}' {peer_lines} {peer_marker} expected_fabric_peer_keys={fabric_peer_key_arg} -wg_peer_set=unverified +wg_peer_keys=unverified if [ "{fabric_peer_keys_provided}" = yes ]; then - wg_peer_set=exact + wg_peer_keys=exact if live_fabric_peer_keys=$("$WG" show "$WG_IFACE" peers 2>/dev/null); then : else live_fabric_peer_keys= - wg_peer_set=missing + wg_peer_keys=missing fi for live_peer_key in $live_fabric_peer_keys; do case " $expected_fabric_peer_keys " in *" $live_peer_key "*) : ;; - *) wg_peer_set=extra ;; + *) wg_peer_keys=extra ;; esac done for expected_peer_key in $expected_fabric_peer_keys; do - printf '%s\n' "$live_fabric_peer_keys" | grep -Fqx "$expected_peer_key" || wg_peer_set=missing + printf '%s\n' "$live_fabric_peer_keys" | grep -Fqx "$expected_peer_key" || wg_peer_keys=missing done # Key presence alone cannot prove the shared cryptokey-routing boundary. # A VPC revision may have withdrawn the final route for a peer that remains # in the Fabric. Require every authoritative key to carry exactly one live # overlay /32 plus only the route-backed tenant union from durable host # state, including peers outside this VPC's current participant set. - if [ "$wg_peer_set" = exact ] && [ "$vpc_state_loaded" = yes ]; then + wg_peer_closure=$wg_peer_keys + if [ "$wg_peer_keys" = exact ] && + {{ [ "$vpc_state_loaded" = yes ] || + [ "$removal_state_closure" != invalid ]; }}; then + wg_peer_closure=exact while IFS='|' read -r expected_peer_key expected_overlay; do [ -n "$expected_peer_key" ] || continue closure_actual=$(vc_wg_allowed_for_peer "$WG_IFACE" "$expected_peer_key") || {{ - wg_peer_set=missing + wg_peer_closure=missing break }} vpc_state_overlay_from_allowed "$closure_actual" >/dev/null || {{ - wg_peer_set=missing + wg_peer_closure=missing break }} closure_extra=$(vpc_state_allowed_for_peer "$expected_peer_key") || {{ - wg_peer_set=missing + wg_peer_closure=missing break }} closure_expected="$expected_overlay" @@ -4741,20 +5114,55 @@ if [ "{fabric_peer_keys_provided}" = yes ]; then closure_expected_set=$(printf '%s\n' "$closure_expected" | tr ',' '\n' | awk 'NF' | LC_ALL=C sort -u) if [ -z "$closure_actual_set" ] || [ "$closure_actual_set" != "$closure_expected_set" ]; then - wg_peer_set=missing + wg_peer_closure=missing break fi done <<'{fabric_peer_marker}' {fabric_peer_overlay_lines} {fabric_peer_marker} - elif [ "$wg_peer_set" = exact ]; then + fi +else + wg_peer_closure=unverified +fi +wg_peer_set=$wg_peer_keys +if [ "$wg_peer_keys" = exact ]; then + if [ "$vpc_state_loaded" = yes ] && [ "$wg_peer_closure" = exact ]; then + wg_peer_set=exact + else wg_peer_set=missing fi fi +removal_wg_peer_set=$wg_peer_keys +if [ "$wg_peer_keys" = exact ]; then + if [ "$removal_state_closure" != invalid ] && + [ "$wg_peer_closure" = exact ]; then + removal_wg_peer_set=exact + else + removal_wg_peer_set=missing + fi +fi printf '%s\n' "__VC_VPC_WG_PEER_SET__=$wg_peer_set" +printf '%s\n' "__VC_VPC_REMOVAL_WG_PEER_SET__=$removal_wg_peer_set" +fabric_nft_closure=no +if {{ [ "$vpc_state_loaded" = yes ] || + [ "$removal_state_closure" != invalid ]; }} && + vc_vpc_fabric_closure_ready; then + fabric_nft_closure=yes +fi fabric_nft_state=no -if vc_vpc_fabric_observe_ready; then fabric_nft_state=yes; fi +if [ "$VPC_STATE_STATUS" = owned ] && [ "$fabric_nft_closure" = yes ]; then + fabric_nft_state=yes +fi printf '%s\n' "__VC_VPC_FABRIC_NFT__=$fabric_nft_state" +removal_fabric_nft_state=no +if [ "$removal_state_closure" != invalid ] && + [ "$fabric_nft_closure" = yes ]; then + removal_fabric_nft_state=yes +fi +printf '%s\n' "__VC_VPC_REMOVAL_FABRIC_NFT__=$removal_fabric_nft_state" +removal_nft_absent=no +if vc_vpc_target_nft_absent; then removal_nft_absent=yes; fi +printf '%s\n' "__VC_VPC_REMOVAL_NFT_ABSENT__=$removal_nft_absent" exit 0 ''' @@ -5124,6 +5532,17 @@ class VpcPrivateHostAdapter: raise VpcDataplaneError("宿主机 VPC 命令失败", 502, "vpc_remote_command_failed") return result.stdout + def _run_mutation(self, script: str) -> str: + """Dispatch one host write and retain ambiguity on any lost receipt.""" + # Keep _run's long-standing override contract for host/test managers; + # the mutation boundary is an adapter concern, not a transport option. + mark_remote_write_started() + try: + return self._run(script) + except BaseException: + mark_execution_unknown() + raise + def _deployment_contract(self) -> tuple[str, str, str, str, str]: """Validate the same immutable host capability gate as Fabric.""" from .fabric import ( @@ -5227,21 +5646,34 @@ class VpcPrivateHostAdapter: if host_ready and receipt == previous[1]: self._save_receipt(scope, fingerprint, receipt) return {"state": "applying", "readback": "pending", "rollback": "none", "owner": DATAPLANE_OWNER_MARKER} - output = self._run(script) - markers = _markers(output) - if markers.get("applied") != "yes": - raise VpcDataplaneError("宿主机未确认 VPC 应用", 502, "vpc_apply_unconfirmed") - receipt_fields = { - "applied": "yes", "owner": DATAPLANE_OWNER_MARKER, - "revision": str(plan.desired_revision), "digest": plan.desired_digest, - } - receipt_keys = (*receipt_fields, "nft_receipt") - receipt_unique = all(sum( - line.startswith(f"__VC_VPC_{key.upper()}__=") - for line in output.splitlines()) == 1 for key in receipt_keys) - if (plan.local_members and receipt_unique - and all(markers.get(key) == value for key, value in receipt_fields.items())): - self._save_receipt(scope, fingerprint, markers.get("nft_receipt")) + output = self._run_mutation(script) + try: + # The host write has already completed. Parsing and strict + # receipt validation are part of the same uncertainty boundary: + # any exception here means the response cannot authorize replay. + markers = _markers(output) + receipt_fields = { + "applied": "yes", "owner": DATAPLANE_OWNER_MARKER, + "revision": str(plan.desired_revision), + "digest": plan.desired_digest, + } + receipt_keys = (*receipt_fields, "nft_receipt") + receipt_unique = all(sum( + line.startswith(f"__VC_VPC_{key.upper()}__=") + for line in output.splitlines()) == 1 for key in receipt_keys) + receipt = markers.get("nft_receipt") + receipt_valid = bool(isinstance(receipt, str) and re.fullmatch( + r"[0-9a-f]{64}(?:,[0-9a-f]{64}){2}", receipt)) + if (not receipt_unique or not receipt_valid + or any(markers.get(key) != value + for key, value in receipt_fields.items())): + raise VpcDataplaneError( + "宿主机未确认 VPC 应用", 502, "vpc_apply_unconfirmed") + except BaseException: + mark_execution_unknown() + raise + if plan.local_members: + self._save_receipt(scope, fingerprint, receipt) return {"state": "applying", "readback": "pending", "rollback": "none", "owner": DATAPLANE_OWNER_MARKER} def remove(self, plan: VpcDataplanePlan) -> dict[str, Any]: @@ -5257,13 +5689,32 @@ class VpcPrivateHostAdapter: scope = self._receipt_scope(plan) with _APPLY_RECEIPT_LOCK: _APPLY_RECEIPTS.pop(scope, None) - markers = _markers(self._run(render_remove_script( + output = self._run_mutation(render_remove_script( plan, deployment_mode=mode, firewall_mode=firewall_mode, ifup_path=ifup_path, - ifdown_path=ifdown_path, ifquery_path=ifquery_path))) - if markers.get("removed") != "yes": - raise VpcDataplaneError( - "宿主机未确认 VPC 清理", 502, "vpc_remove_unconfirmed") + ifdown_path=ifdown_path, ifquery_path=ifquery_path)) + try: + # A malformed removal response is just as ambiguous as a lost + # response. Keep the durable claim until an absence observer can + # prove this exact revision instead of allowing a second delete. + markers = _markers(output) + receipt_fields = { + "removed": "yes", "owner": DATAPLANE_OWNER_MARKER, + "revision": str(plan.desired_revision), + "digest": plan.desired_digest, "rollback": "none", + } + receipt_unique = all(sum( + line.startswith(f"__VC_VPC_{key.upper()}__=") + for line in output.splitlines()) == 1 for key in receipt_fields) + if (not receipt_unique + or any(markers.get(key) != value + for key, value in receipt_fields.items())): + raise VpcDataplaneError( + "宿主机未确认 VPC 清理", 502, + "vpc_remove_unconfirmed") + except BaseException: + mark_execution_unknown() + raise return { "vpc_id": plan.vpc.vpc_id, "host_id": plan.host_id, @@ -5301,12 +5752,105 @@ class VpcPrivateHostAdapter: plan, xml_by_instance=xml_by_instance) return observed + def observe_removal(self, plan: VpcDataplanePlan) -> dict[str, Any]: + """Read back exact host absence without running the removal script.""" + if plan.local_members: + raise VpcDataplaneError( + "仍有本机私网成员,不能核对宿主机 VPC 清理", 409, + "vpc_remove_has_local_members") + mode, _ifup, _ifdown, _ifquery, firewall_mode = ( + self._render_contract()) + output = self._run( + render_observe_script( + plan, deployment_mode=mode, firewall_mode=firewall_mode)) + parsed = _markers(output) + absence_fields = ( + "nft_file_present", "sysctl_file_present", + "proxy_file_present", "if_fragment_present", + "owner_file_present", "state_file_present", + "transaction_present", "nft_include_present", + ) + required_fields = { + "bridge": "missing", + "bridge_owner": "missing", + "nft_table": "missing", + "nft_table_count": "0", + "route_scope_count": "0", + "rule_scope_count": "0", + "nm_profile_count": "0", + "nm_profile_query": ( + "ok" if mode == "networkmanager_nmcli" + else "not_applicable"), + "ifup_profile_count": "0", + "ifup_profile_query": "ok", + "nft_main_count": "1", + "nft_snapshot": "yes", + "firewall_mode": "native_nftables", + "firewall_state": "owned", + "removal_bridge_absent": "yes", + "removal_route_absent": "yes", + "removal_rule_absent": "yes", + "removal_nft_absent": "yes", + "removal_fabric_nft": "yes", + "removal_wg_peer_set": "exact", + } + required_fields.update({field: "no" for field in absence_fields}) + + def unique_scalar(key: str) -> bool: + prefix = f"__VC_VPC_{key.upper()}__=" + return sum(line.startswith(prefix) + for line in output.splitlines()) == 1 + + removed = bool( + all(unique_scalar(key) and parsed.get(key) == expected + for key, expected in required_fields.items()) + and unique_scalar("removal_state_closure") + and parsed.get("removal_state_closure") in {"owned", "empty"} + and parsed.get("proxy_ips", []) == [] + ) + if not removed: + raise VpcDataplaneError( + "宿主机 VPC 清理回读尚未确认", 502, + "vpc_remove_readback_mismatch") + return { + "vpc_id": plan.vpc.vpc_id, + "host_id": plan.host_id, + "desired_revision": plan.desired_revision, + "observed_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "observed_digest": plan.desired_digest, + "config_verified": True, + "host_config_reusable": False, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "state": "active", + "removed": True, + "bridge": None, + "bridge_owner": None, + "member_nics": [], + "routes": [], + "wg_allowed_ips": [], + "nft_table": None, + "acl_revision": plan.desired_revision, + "anti_spoof": False, + "rp_filter": None, + "proxy_arp": None, + "proxy_arp_pvlan": None, + "proxy_ips": [], + "readback": "confirmed", + "transport_verified": True, + "private_dataplane_verified": True, + "rollback": "none", + "reason_code": None, + } + def _observe( self, plan: VpcDataplanePlan, *, xml_by_instance: Mapping[str, Any] | None = None, ) -> tuple[dict[str, Any], bool, Any]: - _mode, _ifup, _ifdown, _ifquery, firewall_mode = self._render_contract() - output = self._run(render_observe_script(plan, firewall_mode=firewall_mode)) + mode, _ifup, _ifdown, _ifquery, firewall_mode = self._render_contract() + output = self._run(render_observe_script( + plan, deployment_mode=mode, firewall_mode=firewall_mode)) parsed = _markers(output) receipt_count = sum(line.startswith("__VC_VPC_NFT_RECEIPT__=") for line in output.splitlines()) diff --git a/web/app/Services/Cloud/CloudEngineClient.php b/web/app/Services/Cloud/CloudEngineClient.php index 776bb63e..50a47ff7 100644 --- a/web/app/Services/Cloud/CloudEngineClient.php +++ b/web/app/Services/Cloud/CloudEngineClient.php @@ -385,6 +385,12 @@ final class CloudEngineClient 'vpc_private_wireguard_readback_mismatch' => 'VPC WireGuard 私网路由回读不一致,正在核对', 'vpc_private_dataplane_readback_mismatch' => 'VPC 私网数据面回读不一致,正在核对', 'vpc_operation_mismatch' => 'VPC 引擎结果身份不一致,正在核对', + 'vpc_member_requires_stopped' => '加入、退出或删除 VPC 前必须先将相关实例完全关机', + 'vpc_member_power_unavailable' => '尚未确认实例已完全关机,请刷新电源状态后重试', + 'vpc_vm_not_persistent' => '实例缺少可核对的持久化定义,暂不能修改私网,请联系管理员', + 'vpc_vm_managed_save_present' => '实例存在 managed-save 状态,暂不能修改私网;请联系管理员核对并清理保存状态', + 'vpc_vm_native_snapshot_present' => '实例存在原生快照,暂不能修改私网;请联系管理员核对快照后重试', + 'vpc_vm_native_snapshot_state_unknown' => '宿主机无法确认实例原生快照状态,暂不能修改私网,请联系管理员检查 libvirt', 'vpc_delete_manifest_invalid' => 'VPC 删除成员清单无效,请刷新后重试', 'vpc_delete_manifest_mismatch' => 'VPC 删除成员清单与当前目标不一致,正在核对', 'vpc_delete_target_not_empty' => 'VPC 删除目标仍包含活动资源,请刷新后重试', diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php index 73321828..e2b1737e 100644 --- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php +++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php @@ -124,6 +124,13 @@ final class CloudOperationErrorCatalog 'vpc_private_observation_missing' => '实例私网状态尚未完成回读,请稍后刷新,切勿重复提交', 'vpc_private_observation_mismatch' => '实例私网状态与当前 VPC 目标不一致,正在核对', 'vpc_private_dataplane_unverified' => 'VPC 私网数据面尚未完成核验,请稍后刷新', + 'vpc_member_requires_stopped' => '加入、退出或删除 VPC 前必须先将相关实例完全关机', + 'vpc_member_power_unavailable' => '尚未确认实例已完全关机,请刷新电源状态后重试', + 'vpc_vm_not_persistent' => '实例缺少可核对的持久化定义,暂不能修改私网,请联系管理员', + 'vpc_vm_managed_save_present' => '实例存在 managed-save 状态,暂不能修改私网;请联系管理员核对并清理保存状态', + 'vpc_vm_native_snapshot_present' => '实例存在原生快照,暂不能修改私网;请联系管理员核对快照后重试', + 'vpc_vm_native_snapshot_state_unknown' => '宿主机无法确认实例原生快照状态,暂不能修改私网,请联系管理员检查 libvirt', + 'vpc_private_remove_unsafe' => '实例存在 managed-save 或原生快照,暂不能移除私网;请联系管理员核对保存状态和快照', 'vpc_private_guest_restart_required' => '私网配置尚未写入当前实例;请完全关机,等待系统核对写入后再开机', 'vpc_private_guest_agent_unavailable' => 'QEMU Guest Agent 暂不可用,无法确认来宾私网地址和路由', 'vpc_private_guest_probe_unsupported' => '来宾缺少受支持的网络探测能力,无法确认私网地址和路由', diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php index 60d3b04a..16501d4a 100644 --- a/web/app/Services/Cloud/CloudVpcService.php +++ b/web/app/Services/Cloud/CloudVpcService.php @@ -56,6 +56,13 @@ final class CloudVpcService /** Engine reason codes that may cross the host/operation/root boundary. */ private const DEFERRED_EVIDENCE_ERRORS = [ + 'vpc_member_requires_stopped', + 'vpc_member_power_unavailable', + 'vpc_vm_not_persistent', + 'vpc_vm_managed_save_present', + 'vpc_vm_native_snapshot_present', + 'vpc_vm_native_snapshot_state_unknown', + 'vpc_private_remove_unsafe', 'vpc_private_guest_restart_required', 'vpc_private_remove_requires_stopped', 'vpc_private_guest_agent_unavailable', @@ -328,7 +335,7 @@ final class CloudVpcService 'confirmation_name' => $confirmationName, ]); $vpc = self::findOwned($vpcKey, $actor, true); - return $vpc->getConnection()->transaction(function () use ( + $prepared = $vpc->getConnection()->transaction(function () use ( $vpc, $actor, $confirmationName, $key, $requestHash ): array { $locked = self::lockOwnedForDelete((string) $vpc->vpc_key, $actor); @@ -336,18 +343,65 @@ final class CloudVpcService throw new CloudError('确认名称与 VPC 名称不一致'); } if (self::hasOperation($locked, $key, 'vpc_delete', $requestHash)) { - return self::documentView($locked, $actor); + return [ + 'replay' => self::documentView($locked, $actor), + 'member_identities' => [], + 'instances' => [], + ]; } if ((string) $locked->state === 'deleted') { // DELETE remains idempotent after the original operation has // completed, even when the caller generated a fresh request // key after losing the prior HTTP response. + return [ + 'replay' => self::documentView($locked, $actor), + 'member_identities' => [], + 'instances' => [], + ]; + } + if ((string) $locked->state === 'deleting') { + throw new CloudError('VPC 删除正在核对,请勿重复提交', 409); + } + + $snapshot = self::deleteMemberSnapshotLocked($locked); + return [ + 'replay' => null, + 'member_identities' => $snapshot['member_identities'], + 'instances' => $snapshot['instances'], + ]; + }, 3); + if ($prepared['replay'] !== null) { + return $prepared['replay']; + } + + self::assertMemberMutationsStopped($prepared['instances']); + + return $vpc->getConnection()->transaction(function () use ( + $vpc, $actor, $confirmationName, $key, $requestHash, $prepared + ): array { + $locked = self::lockOwnedForDelete((string) $vpc->vpc_key, $actor); + if (!hash_equals((string) $locked->name, $confirmationName)) { + throw new CloudError('确认名称与 VPC 名称不一致'); + } + if (self::hasOperation($locked, $key, 'vpc_delete', $requestHash)) { + return self::documentView($locked, $actor); + } + if ((string) $locked->state === 'deleted') { return self::documentView($locked, $actor); } if ((string) $locked->state === 'deleting') { throw new CloudError('VPC 删除正在核对,请勿重复提交', 409); } + $snapshot = self::deleteMemberSnapshotLocked($locked); + if ($snapshot['member_identities'] !== $prepared['member_identities']) { + throw new CloudError( + 'VPC 成员集合已变化;请重新读取并确认全部实例已关机', + 409, + 'vpc_member_power_unavailable' + ); + } + $now = time(); foreach ([ CloudVpcRoute::class, @@ -465,6 +519,13 @@ final class CloudVpcService if ($prepared['replay'] !== null) { return $prepared['replay']; } + if ($autoPower) { + throw new CloudError( + '自动关机和开机已停用;请先在实例详情将实例完全关机,再重新提交', + 422, + 'vpc_auto_power_retired' + ); + } $requestHash = (string) $prepared['request_hash']; $automaticSubnetKey = (string) $prepared['subnet_key']; $instance = CloudInstance::where('id', $instanceId) @@ -474,9 +535,16 @@ final class CloudVpcService && (string) $instance->status === 'active' && $instance->serviceAvailable()) { self::assertInstanceRegion($instance, (int) $vpc->region_id); - self::releasePersistedDeletedMembershipsBeforeAttach( - $vpc, $instance, $actor - ); + $alreadyAttached = CloudVpcMember::where('vpc_id', (int) $vpc->id) + ->where('instance_id', $instanceId) + ->whereNotIn('state', ['deleted', 'deleting']) + ->exists(); + if (!$alreadyAttached) { + self::assertMemberMutationStopped($instance); + self::releasePersistedDeletedMembershipsBeforeAttach( + $vpc, $instance, $actor + ); + } } return $vpc->getConnection()->transaction(function () use ( $vpc, $actor, $key, $requestHash, $instanceId, $automaticSubnetKey, @@ -639,9 +707,6 @@ final class CloudVpcService $member->private_ip_id = $privateIp->id; $member->save(); $result = self::touchDesiredLocked($vpc, 'member_attach', $key, $requestHash); - if ($autoPower) { - CloudVpcPowerWorkflow::initialize($result['operation'], $instance, $member); - } return self::documentView($result['vpc'], $actor); }, 3); } @@ -655,9 +720,40 @@ final class CloudVpcService self::assertKey($memberKey, '成员'); $requestHash = self::requestHash('member_detach', ['member_key' => $memberKey] + ($autoPower ? ['auto_power' => true] : [])); - $vpc = self::lockOwned($vpcKey, $actor); - return $vpc->getConnection()->transaction(function () use ( + $vpc = self::findOwned($vpcKey, $actor, true); + $prepared = $vpc->getConnection()->transaction(function () use ( $vpc, $memberKey, $actor, $key, $requestHash, $autoPower + ): array { + $locked = self::lockOwned($vpc->vpc_key, $actor); + if (self::hasOperation($locked, $key, 'member_detach', $requestHash)) { + return ['replay' => self::documentView($locked, $actor), 'instance' => null]; + } + if ($autoPower) { + throw new CloudError( + '自动关机和开机已停用;请先在实例详情将实例完全关机,再重新提交', + 422, + 'vpc_auto_power_retired' + ); + } + $member = CloudVpcMember::where('vpc_id', $locked->id) + ->where('member_key', $memberKey)->lockForUpdate()->first(); + if ($member === null + || in_array((string) $member->state, ['deleted', 'deleting'], true)) { + throw new CloudError('VPC 成员不存在', 404); + } + $instance = CloudInstance::where('id', (int) $member->instance_id) + ->lockForUpdate()->first(); + if ($instance === null || (string) $instance->status === 'deleted') { + throw new CloudError('虚拟机不存在', 404); + } + return ['replay' => null, 'instance' => $instance]; + }, 3); + if ($prepared['replay'] !== null) { + return $prepared['replay']; + } + self::assertMemberMutationStopped($prepared['instance']); + return $vpc->getConnection()->transaction(function () use ( + $vpc, $memberKey, $actor, $key, $requestHash ): array { $vpc = self::lockOwned($vpc->vpc_key, $actor); if (self::hasOperation($vpc, $key, 'member_detach', $requestHash)) { @@ -690,9 +786,6 @@ final class CloudVpcService $privateIp->updated_at = time(); $privateIp->save(); $result = self::touchDesiredLocked($vpc, 'member_detach', $key, $requestHash); - if ($autoPower) { - CloudVpcPowerWorkflow::initialize($result['operation'], $instance, $member); - } return self::documentView($result['vpc'], $actor); }, 3); } @@ -1472,9 +1565,11 @@ final class CloudVpcService $payload['desired_digest'], $observed, (string) $operation->operation_id, $lease, true); $operation->refresh(); - // Even a partial current receipt consumes this turn. Falling through - // would POST the same configuration again instead of waiting for its - // durable engine observation to finish. + // A same-revision GET proves that this immutable operation was already + // accepted by the Engine. Exact configuration may advance the legacy + // workflow; partial proof remains in its bounded observation window. + // Consume this turn in both cases: an explicit reconcile may repeat + // the read, but must never replay the uncertain write with a POST. return true; } @@ -3777,6 +3872,126 @@ final class CloudVpcService ? strtolower($value) : null; } + /** + * Admit a new member mutation only from a fresh, identity-bound shutoff + * observation. The Engine repeats this guard immediately before host + * mutation; this cache check keeps running guests out of the durable PHP + * desired-state ledger and gives the tenant an actionable response. + */ + private static function assertMemberMutationStopped(CloudInstance $instance): void + { + self::assertMemberMutationsStopped([$instance]); + } + + /** + * Require one identity-bound, fresh shutoff observation for every member + * participating in a configuration mutation. Root deletion uses one + * batch read so no member can be omitted between per-instance lookups. + */ + private static function assertMemberMutationsStopped(iterable $instances): void + { + $items = []; + foreach ($instances as $instance) { + if ($instance instanceof CloudInstance && (int) $instance->id > 0) { + $items[(int) $instance->id] = $instance; + } + } + if ($items === []) { + return; + } + try { + $states = CloudInstanceRuntimeService::forInstances($items); + } catch (\Throwable) { + throw new CloudError( + '暂时无法确认全部实例已关机;请在实例详情刷新电源状态后重试', + 409, + 'vpc_member_power_unavailable' + ); + } + foreach ($items as $instanceId => $_instance) { + $state = $states[(string) $instanceId] ?? null; + if (!is_array($state) || ($state['state_fresh'] ?? false) !== true) { + throw new CloudError( + '实例电源状态尚未完成最新回读;请在实例详情刷新后重试', + 409, + 'vpc_member_power_unavailable' + ); + } + if ((string) ($state['power_state'] ?? '') !== 'stopped') { + throw new CloudError( + '加入、退出或删除 VPC 前必须先将实例完全关机;平台不再自动操作电源', + 409, + 'vpc_member_requires_stopped' + ); + } + } + } + + /** + * Capture the complete non-terminal member/VM identity set under the VPC + * lock. The caller compares this canonical snapshot after the out-of-lock + * runtime read before it is allowed to persist a root-delete target. + * + * @return array{member_identities: array>, + * instances: array} + */ + private static function deleteMemberSnapshotLocked(CloudVpc $vpc): array + { + $members = CloudVpcMember::where('vpc_id', (int) $vpc->id) + ->where('state', '<>', 'deleted') + ->orderBy('id')->lockForUpdate()->get(); + $instanceIds = $members->map( + static fn ($member): int => (int) $member->instance_id + )->unique()->sort()->values()->all(); + $instances = $instanceIds === [] ? collect() : CloudInstance::whereIn( + 'id', $instanceIds + )->orderBy('id')->lockForUpdate()->get()->keyBy('id'); + $identities = []; + $runtimeInstances = []; + foreach ($members as $member) { + $instanceId = (int) $member->instance_id; + $instance = $instances->get($instanceId); + if (!$instance instanceof CloudInstance + || (string) $instance->status === 'deleted') { + throw new CloudError( + 'VPC 成员实例身份不完整,无法安全删除', + 503, + 'vpc_member_power_unavailable' + ); + } + $uuid = strtolower(trim((string) $instance->uuid)); + if (preg_match( + '/^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/D', + $uuid + ) !== 1 + || (int) $instance->host_id !== (int) $member->host_id) { + throw new CloudError( + 'VPC 成员实例身份已变化,无法安全删除', + 503, + 'vpc_member_power_unavailable' + ); + } + $identities[] = [ + 'member_id' => (int) $member->id, + 'member_key' => (string) $member->member_key, + 'instance_id' => $instanceId, + 'host_id' => (int) $instance->host_id, + 'uuid' => $uuid, + 'private_ip' => (string) $member->private_ip, + 'private_ip_id' => $member->private_ip_id === null + ? null : (int) $member->private_ip_id, + 'subnet_id' => $member->subnet_id === null + ? null : (int) $member->subnet_id, + 'state' => (string) $member->state, + ]; + $runtimeInstances[$instanceId] = $instance; + } + return [ + 'member_identities' => $identities, + 'instances' => $runtimeInstances, + ]; + } + /** * Reap legacy terminal identities before an attach reserves new unique * keys. Source and target roots are locked in numeric order so two diff --git a/web/public/cloud/vpc.js b/web/public/cloud/vpc.js index 8626bcc8..f749300d 100644 --- a/web/public/cloud/vpc.js +++ b/web/public/cloud/vpc.js @@ -206,6 +206,14 @@ function errorLabel(code) { vpc_private_guest_route_mismatch: '实例内的私网路由与当前 VPC 配置不一致,请检查私网路由后重新核对。', vpc_private_guest_link_mismatch: '实例内的私网网卡尚未正常连通,请检查网卡状态后重新核对。', vpc_instance_operation_pending: '成员实例还有未完成或结果待核对的任务,请先在实例详情中核对任务,再修改 VPC。', + vpc_auto_power_retired: '新的 VPC 成员操作不再自动关机或开机;请先将实例完全关机再重试。', + vpc_member_requires_stopped: '加入、退出或删除 VPC 前必须先将相关实例完全关机;平台不会自动操作电源。', + vpc_member_power_unavailable: '尚未读到实例的最新关机状态;请在实例详情刷新电源状态后重试。', + vpc_vm_not_persistent: '实例缺少可核对的持久化定义,暂不能修改私网;请联系管理员。', + vpc_vm_managed_save_present: '实例存在 managed-save 状态,暂不能修改私网;请联系管理员核对并清理保存状态。', + vpc_vm_native_snapshot_present: '实例存在原生快照,暂不能修改私网;请联系管理员核对快照后重试。', + vpc_vm_native_snapshot_state_unknown: '宿主机无法确认实例原生快照状态,暂不能修改私网;请联系管理员检查 libvirt。', + vpc_private_remove_unsafe: '实例存在 managed-save 或原生快照,暂不能移除私网;请联系管理员核对保存状态和快照。', vpc_power_shutdown_timeout: '正常关机等待超时,自动流程已暂停,平台未强制断电。可在实例详情检查电源,必要时手动强制断电,再点击“核对 / 重试”继续。', vpc_power_start_timeout: '启动命令已接受,但等待实例开机超时,自动流程已暂停。请在实例详情核对当前电源状态,再点击“核对 / 重试”;平台不会重复发送启动命令。', vpc_power_result_uncertain: '电源请求的执行结果未确认,请先在实例详情核对原操作;不要重复提交加入或退出。', @@ -759,7 +767,7 @@ function refreshMemberChoices() { } function renderMembers(doc) { - const card = section('成员与私网 IP', '加入或退出时,平台会自动正常关机、修改私网配置,再恢复原本运行的实例;原本已关机的实例保持关机。运行中的实例会短暂中断服务,关机超时不会自动强制断电。', 'vpc-members'); + const card = section('成员与私网 IP', '加入或退出前必须先在实例详情将实例完全关机。平台只修改和核对私网配置,不再自动关机或开机;配置完成后实例仍保持关机。', 'vpc-members'); const form = element('form', 'vpc-inline-form vpc-member-form'); form.dataset.vpcForm = 'member'; const instance = input('选择实例', '', 'select'); instance.control.name = 'instance_id'; instance.control.required = true; instance.control.id = 'vpc-member-instance'; @@ -1209,10 +1217,10 @@ async function submitMember(form) { refreshMemberChoices(); return; } - if (!confirm(`确认将“${instanceName(instanceId)}”加入 VPC?运行中的实例将自动正常关机,写入私网配置后恢复开机,期间服务会中断;原本已关机的实例保持关机。关机超时会暂停,平台不会自动强制断电。`)) return; - const body = { instance_id: instanceId, auto_power: true, request_key: requestKey('member-add') }; + if (!confirm(`确认将“${instanceName(instanceId)}”加入 VPC?请先确认实例已完全关机。平台不会自动关机或开机;写入和核对私网配置后,实例仍保持关机。`)) return; + const body = { instance_id: instanceId, request_key: requestKey('member-add') }; await mutate(`/api/v1/vpcs/${encodeURIComponent(state.selectedKey)}/members`, 'POST', body, - '已提交加入 VPC,平台将自动处理关机、私网配置和开机恢复。可在下方查看进度;私网可用性将单独核验。'); + '已提交加入 VPC,正在写入并核对私网配置;实例会保持关机。'); } function readRoutes() { @@ -1283,16 +1291,16 @@ async function reconcile() { async function removeMember(key) { if (!state.selectedKey || state.busy || !allowSelectedMutation() || !confirmDetailMutation() - || !confirm('确认让这个实例退出 VPC?运行中的实例将自动正常关机,移除私网配置后恢复开机,期间服务会中断;原本已关机的实例保持关机。关机超时会暂停,不会自动强制断电。清理确认前,私网 IP 不会重新分配。')) return; + || !confirm('确认让这个实例退出 VPC?请先确认实例已完全关机。平台不会自动关机或开机;移除并核对私网配置后,实例仍保持关机。清理确认前,私网 IP 不会重新分配。')) return; await mutate(`/api/v1/vpcs/${encodeURIComponent(state.selectedKey)}/members/${encodeURIComponent(key)}`, - 'DELETE', { auto_power: true, request_key: requestKey('member-remove') }, '已提交退出 VPC,平台将自动处理关机、私网清理和开机恢复;清理确认前私网 IP 会继续保留。'); + 'DELETE', { request_key: requestKey('member-remove') }, '已提交退出 VPC,正在移除并核对私网配置;实例会保持关机,清理确认前私网 IP 会继续保留。'); } async function deleteVpc() { if (!state.selectedKey || !state.selected || state.busy || selectedRootDeleting()) return; if (!confirmDetailMutation()) return; const root = state.selected.vpc; - const confirmation = prompt(`永久删除 VPC“${root.name}”并清理其私网网卡、路由和 ACL。宿主机回读完成后 CIDR 将被释放。请输入完整 VPC 名称确认:`); + const confirmation = prompt(`永久删除 VPC“${root.name}”并清理其私网网卡、路由和 ACL。请先将 VPC 内全部实例完全关机;平台不会自动操作电源,实例会保持关机。宿主机回读完成后 CIDR 将被释放。请输入完整 VPC 名称确认:`); if (confirmation === null) return; if (confirmation !== root.name) { toast('VPC 名称不匹配,未执行删除。', 'error'); diff --git a/web/tests/CloudVpcControlPlaneContractTest.php b/web/tests/CloudVpcControlPlaneContractTest.php index 16a2daca..7ad56b35 100644 --- a/web/tests/CloudVpcControlPlaneContractTest.php +++ b/web/tests/CloudVpcControlPlaneContractTest.php @@ -418,6 +418,10 @@ $assert(str_contains($runtimeService, "'shutdown' => 'shutdown'") && str_contains($runtimeService, "'poweroff' => 'stopped'") && str_contains($runtimeService, "'start', 'reboot', 'reset' => 'running'"), 'Runtime cache lost bounded power-action transition semantics.'); +$assert(str_contains($service, "['power_state'] ?? '') !== 'stopped'") + && str_contains($service, "'vpc_member_requires_stopped'") + && str_contains($service, "'vpc_member_power_unavailable'"), + 'Stopped-only VPC admission does not use the normalized runtime vocabulary.'); $assert(str_contains( $operationErrorCatalog, '私网配置尚未写入当前实例;请完全关机,等待系统核对写入后再开机' @@ -426,6 +430,13 @@ $assert(str_contains( '来宾私网探测清理状态未确认;请完全关机再开机后继续核验' ) && !str_contains($operationErrorCatalog, '私网地址或路由已暂存'), 'VPC restart guidance must not claim the pending seed/XML was already staged.'); +$assert(str_contains($operationErrorCatalog, "'vpc_member_requires_stopped'") + && str_contains($operationErrorCatalog, "'vpc_member_power_unavailable'") + && str_contains($operationErrorCatalog, "'vpc_vm_not_persistent'") + && str_contains($operationErrorCatalog, "'vpc_vm_managed_save_present'") + && str_contains($operationErrorCatalog, "'vpc_vm_native_snapshot_present'") + && str_contains($operationErrorCatalog, "'vpc_vm_native_snapshot_state_unknown'"), + 'Stopped-only VPC admission codes lost reviewed user-facing labels.'); $assert(str_contains($vpcWorkerService, "'waiting_observed'") && str_contains($vpcWorkerService, 'allowDeferredRetry') && str_contains($vpcWorkerService, "where('status', 'succeeded')") @@ -453,6 +464,12 @@ $assert(str_contains($engineClient, 'function vpc(') && str_contains($engineClient, '(?:reconcile|delete)') && str_contains($engineClient, 'vpc_delete_manifest_invalid') && str_contains($engineClient, 'vpc_delete_manifest_mismatch') + && str_contains($engineClient, 'vpc_member_requires_stopped') + && str_contains($engineClient, 'vpc_member_power_unavailable') + && str_contains($engineClient, 'vpc_vm_not_persistent') + && str_contains($engineClient, 'vpc_vm_managed_save_present') + && str_contains($engineClient, 'vpc_vm_native_snapshot_present') + && str_contains($engineClient, 'vpc_vm_native_snapshot_state_unknown') && str_contains($engineClient, 'vpc_private_remove_requires_stopped') && str_contains($engineClient, 'vpc_private_remove_unsafe') && str_contains($engineClient, 'vpc_private_interface_conflict') diff --git a/web/tests/CloudVpcMemberChoicesTest.mjs b/web/tests/CloudVpcMemberChoicesTest.mjs index 4e6d83bb..d3c5888b 100644 --- a/web/tests/CloudVpcMemberChoicesTest.mjs +++ b/web/tests/CloudVpcMemberChoicesTest.mjs @@ -149,20 +149,20 @@ function harness() { setFetch(nextHandler) { handler = nextHandler; } }; } -// Both explicit member mutations disclose the power cycle before a write. -// Cancelling the dialog must not request shutdown, mutate membership or refresh -// a form in progress. The browser delegates accepted work to the VPC workflow. +// Both explicit member mutations disclose the stopped prerequisite before a +// write. Cancelling the dialog must not mutate membership or refresh a form in +// progress. Accepted work contains no browser or server auto-power opt-in. for (const action of ['attach', 'detach']) { const h = harness(); h.setConfirmResult(false); const submit = () => action === 'attach' ? h.submitMember({ instance_id: { value: '11' } }) : h.removeMember('member-11'); await submit(); - assert.equal(h.requests.length, 0, 'Declining a power cycle must make no request.'); + assert.equal(h.requests.length, 0, 'Declining a stopped-only mutation must make no request.'); assert.equal(h.confirmations.length, 1); - assert.match(h.confirmations[0], /正常关机.*配置.*恢复开机.*服务会中断/u); - assert.match(h.confirmations[0], /原本已关机的实例保持关机/u); - assert.match(h.confirmations[0], /关机超时.*暂停.*不会自动强制断电/u); + assert.match(h.confirmations[0], /实例已完全关机/u); + assert.match(h.confirmations[0], /不会自动关机或开机/u); + assert.match(h.confirmations[0], /实例仍保持关机/u); if (action === 'attach') assert.match(h.confirmations[0], /vm-11/u); else assert.match(h.confirmations[0], /清理确认前.*IP 不会重新分配/u); h.setConfirmResult(true); @@ -173,10 +173,10 @@ for (const action of ['attach', 'detach']) { action === 'attach' ? 'POST /api/v1/vpcs/new-vpc/members' : 'DELETE /api/v1/vpcs/new-vpc/members/member-11', 'GET /user/vpcs', - ], 'Accepted automatic work must stay in the durable VPC API, without browser power calls.'); - assert.equal(JSON.parse(h.requests[0].body).auto_power, true, - 'The explicit confirmation must opt into automatic power handling.'); - assert.match(h.nodes['vpc-page-notice'].textContent, /自动处理关机.*开机恢复/u); + ], 'Accepted stopped-only work must stay in the durable VPC API.'); + assert.equal(Object.hasOwn(JSON.parse(h.requests[0].body), 'auto_power'), false, + 'New member mutations must not opt into legacy automatic power handling.'); + assert.match(h.nodes['vpc-page-notice'].textContent, /实例会保持关机/u); } // Network readiness or an empty VPC must not hide a pending power restore. diff --git a/web/tests/CloudVpcMemberReuseTest.php b/web/tests/CloudVpcMemberReuseTest.php index 8e38cfd7..d9cb88ec 100644 --- a/web/tests/CloudVpcMemberReuseTest.php +++ b/web/tests/CloudVpcMemberReuseTest.php @@ -470,7 +470,10 @@ $seedInstance = static function (int $id, int $hostId, int $userId = 9) use ( ]); Capsule::table('cloud_instance_runtime')->insert([ 'instance_id' => $id, 'host_id' => $hostId, 'uuid' => $uuidFor($id), - 'power_state' => 'running', 'sync_state' => 'ok', + // New VPC member mutations are admitted only from a fresh, stopped + // observation. Individual tests opt back into running when verifying + // the fail-closed prerequisite. + 'power_state' => 'shutoff', 'sync_state' => 'ok', 'last_success_at' => $now, 'last_attempt_at' => $now, 'revision' => 1, 'last_error_code' => '', 'created_at' => $now, 'updated_at' => $now, @@ -1030,6 +1033,77 @@ $assert(count($boundedObserved['hosts'] ?? []) === 257 === CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'Observed proof persistence still truncates hosts, routes or peers at 256.'); +// New member mutations require one fresh stopped observation before any VPC, +// member or address ledger write. They never create the retired automatic +// power workflow, and a running rejection leaves the original revision exact. +$stoppedOnly = $seedVpc(20, '10.61.0.0/24', '10.61.0.0/29', 10); +$seedInstance(20001, 10); +Capsule::table('cloud_instance_runtime')->where('instance_id', 20001)->update([ + 'power_state' => 'running', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +$stoppedInput = [ + 'instance_id' => 20001, 'request_key' => 'stopped-only-attach-20001', +]; +$runningAttach = $expectCloudError(static fn () => CloudVpcService::attachMember( + $stoppedOnly['vpc_key'], $actor, $stoppedInput +), 409); +$assert($runningAttach->errorCode === 'vpc_member_requires_stopped' + && CloudVpcMember::where('instance_id', 20001)->count() === 0 + && Capsule::table('cloud_vpc_operations')->where('vpc_id', 20) + ->where('idempotency_key', 'stopped-only-attach-20001')->count() === 0 + && (int) Capsule::table('cloud_vpcs')->where('id', 20) + ->value('desired_revision') === 1, + 'A running attach changed the durable VPC target before rejecting it.'); +$retiredPower = $expectCloudError(static fn () => CloudVpcService::attachMember( + $stoppedOnly['vpc_key'], $actor, [ + 'instance_id' => 20001, 'request_key' => 'retired-auto-power-20001', + 'auto_power' => true, + ] +), 422); +$assert($retiredPower->errorCode === 'vpc_auto_power_retired' + && CloudVpcMember::where('instance_id', 20001)->count() === 0, + 'A new auto-power attach was not rejected before membership mutation.'); +Capsule::table('cloud_instance_runtime')->where('instance_id', 20001)->update([ + 'power_state' => 'shutoff', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +CloudVpcService::attachMember($stoppedOnly['vpc_key'], $actor, $stoppedInput); +$stoppedMember = CloudVpcMember::where('instance_id', 20001)->first(); +$stoppedAttach = CloudVpcOperation::where('vpc_id', 20) + ->where('idempotency_key', 'stopped-only-attach-20001')->first(); +$assert($stoppedMember !== null && $stoppedAttach !== null + && $stoppedAttach->power_workflow_json === null + && $stoppedAttach->power_instance_id === null, + 'A stopped attach created a legacy automatic power workflow.'); +Capsule::table('cloud_instance_runtime')->where('instance_id', 20001)->update([ + 'power_state' => 'running', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +$runningDetach = $expectCloudError(static fn () => CloudVpcService::detachMember( + $stoppedOnly['vpc_key'], (string) $stoppedMember->member_key, $actor, + ['request_key' => 'stopped-only-detach-20001'] +), 409); +$assert($runningDetach->errorCode === 'vpc_member_requires_stopped' + && (string) $stoppedMember->fresh()->state === 'applying' + && (int) Capsule::table('cloud_vpcs')->where('id', 20) + ->value('desired_revision') === 2, + 'A running detach changed the member tombstone or desired revision.'); +Capsule::table('cloud_instance_runtime')->where('instance_id', 20001)->update([ + 'power_state' => 'shutoff', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +CloudVpcService::detachMember( + $stoppedOnly['vpc_key'], (string) $stoppedMember->member_key, $actor, + ['request_key' => 'stopped-only-detach-20001'] +); +$stoppedDetach = CloudVpcOperation::where('vpc_id', 20) + ->where('idempotency_key', 'stopped-only-detach-20001')->first(); +$assert($stoppedDetach !== null && $stoppedDetach->power_workflow_json === null + && $stoppedDetach->power_instance_id === null + && (string) $stoppedMember->fresh()->state === 'deleting', + 'A stopped detach did not preserve stopped-only, tombstone-safe behavior.'); + // A fully proven historical pair must release both uniqueness keys before a // same-VPC attach automatically reserves the lowest free address again. The host stays // active for a real sibling, so absence is proven by exact NIC/route/proxy diff --git a/web/tests/CloudVpcMutationAdmissionTest.php b/web/tests/CloudVpcMutationAdmissionTest.php index db7d6496..33764a6e 100644 --- a/web/tests/CloudVpcMutationAdmissionTest.php +++ b/web/tests/CloudVpcMutationAdmissionTest.php @@ -153,4 +153,112 @@ $assert((int) Capsule::table('cloud_vpcs')->where('id', 902) ->value('desired_revision') === 2, 'A rolled-back lock conflict prevented a later safe retry.'); +// Root deletion is also a member mutation. It must capture the complete +// member identity set, prove every runtime freshly stopped in one batch, then +// re-lock and compare the same set before it persists any tombstone. +$deleteVpc = $seedVpc(903, '10.90.3.0/24', '10.90.3.0/24', 10); +$seedInstance(90301, 10); +$seedInstance(90302, 10); +CloudVpcService::attachMember($deleteVpc['vpc_key'], $actor, [ + 'instance_id' => 90301, 'request_key' => 'delete-attach-90301', +]); +CloudVpcService::attachMember($deleteVpc['vpc_key'], $actor, [ + 'instance_id' => 90302, 'request_key' => 'delete-attach-90302', +]); +$deleteInput = [ + 'confirmation_name' => 'reuse-vpc-903', + 'request_key' => 'delete-stopped-members-903', +]; +$deleteRevision = (int) Capsule::table('cloud_vpcs')->where('id', 903) + ->value('desired_revision'); +$deleteOperationCount = Capsule::table('cloud_vpc_operations') + ->where('vpc_id', 903)->count(); +Capsule::table('cloud_instance_runtime')->where('instance_id', 90302)->update([ + 'power_state' => 'running', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +$runningDelete = $expectCloudError(static fn () => CloudVpcService::delete( + $deleteVpc['vpc_key'], $actor, $deleteInput +), 409); +$assert($runningDelete->errorCode === 'vpc_member_requires_stopped' + && (string) Capsule::table('cloud_vpcs')->where('id', 903)->value('state') + !== 'deleting' + && (int) Capsule::table('cloud_vpcs')->where('id', 903) + ->value('desired_revision') === $deleteRevision + && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count() + === $deleteOperationCount + && Capsule::table('cloud_vpc_members')->where('vpc_id', 903) + ->where('state', 'deleting')->count() === 0, + 'A running member allowed root-delete intent to change the durable ledger.'); +Capsule::table('cloud_instance_runtime')->where('instance_id', 90302)->update([ + 'power_state' => 'shutoff', 'sync_state' => 'ok', + 'last_success_at' => time() - 60, 'updated_at' => time(), +]); +$staleDelete = $expectCloudError(static fn () => CloudVpcService::delete( + $deleteVpc['vpc_key'], $actor, $deleteInput +), 409); +$assert($staleDelete->errorCode === 'vpc_member_power_unavailable' + && (int) Capsule::table('cloud_vpcs')->where('id', 903) + ->value('desired_revision') === $deleteRevision + && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count() + === $deleteOperationCount, + 'A stale member power observation allowed root deletion to be persisted.'); +Capsule::table('cloud_instance_runtime')->whereIn('instance_id', [90301, 90302]) + ->update([ + 'power_state' => 'shutoff', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), + ]); +$deleteSnapshotMutationArmed = true; +$deleteSnapshotMutationSeen = false; +$connection->beforeExecuting(static function ($sql) use ( + &$deleteSnapshotMutationArmed, &$deleteSnapshotMutationSeen +): void { + if (!$deleteSnapshotMutationArmed + || !str_starts_with(strtolower(ltrim($sql)), 'select') + || preg_match('/\bfrom\s+["`]?cloud_instance_runtime["`]?(?:\s|$)/i', $sql) !== 1) { + return; + } + // Simulate a concurrent member transition after the first locked closure + // has committed and while the out-of-lock batch power read is starting. + $deleteSnapshotMutationArmed = false; + Capsule::table('cloud_vpc_members')->where('vpc_id', 903) + ->where('instance_id', 90302)->update(['state' => 'active']); + $deleteSnapshotMutationSeen = true; +}); +$changedClosureInput = [ + 'confirmation_name' => 'reuse-vpc-903', + 'request_key' => 'delete-changed-closure-903', +]; +$changedClosure = $expectCloudError(static fn () => CloudVpcService::delete( + $deleteVpc['vpc_key'], $actor, $changedClosureInput +), 409); +$assert($deleteSnapshotMutationSeen + && $changedClosure->errorCode === 'vpc_member_power_unavailable' + && (int) Capsule::table('cloud_vpcs')->where('id', 903) + ->value('desired_revision') === $deleteRevision + && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count() + === $deleteOperationCount + && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903) + ->where('idempotency_key', 'delete-changed-closure-903')->count() === 0, + 'A member identity transition between delete phases created an operation.'); +Capsule::table('cloud_vpc_members')->where('vpc_id', 903) + ->where('instance_id', 90302)->update(['state' => 'applying']); +CloudVpcService::delete($deleteVpc['vpc_key'], $actor, $deleteInput); +$assert((string) Capsule::table('cloud_vpcs')->where('id', 903)->value('state') + === 'deleting' + && (int) Capsule::table('cloud_vpcs')->where('id', 903) + ->value('desired_revision') === $deleteRevision + 1 + && Capsule::table('cloud_vpc_members')->where('vpc_id', 903) + ->where('state', 'deleting')->count() === 2, + 'A fully stopped member closure did not admit one root-delete revision.'); +Capsule::table('cloud_instance_runtime')->where('instance_id', 90301)->update([ + 'power_state' => 'running', 'last_success_at' => time(), 'updated_at' => time(), +]); +CloudVpcService::delete($deleteVpc['vpc_key'], $actor, $deleteInput); +$assert((int) Capsule::table('cloud_vpcs')->where('id', 903) + ->value('desired_revision') === $deleteRevision + 1 + && Capsule::table('cloud_vpc_operations')->where('vpc_id', 903)->count() + === $deleteOperationCount + 1, + 'An exact root-delete replay rechecked power or created a second operation.'); + echo "VPC instance-task admission database regressions passed.\n"; diff --git a/web/tests/CloudVpcPowerWorkflowTest.php b/web/tests/CloudVpcPowerWorkflowTest.php index b45ff772..c2781b9f 100644 --- a/web/tests/CloudVpcPowerWorkflowTest.php +++ b/web/tests/CloudVpcPowerWorkflowTest.php @@ -130,12 +130,32 @@ $make = static function (int $id, string $power = 'running') use ($seedVpc, $see $vpc = $seedVpc($id, '10.121.' . ($id % 250) . '.0/24', '10.121.' . ($id % 250) . '.0/24', 10); $seedInstance($id * 100, 10); - CloudEngineClient::$power = $power; CloudVpcService::attachMember($vpc['vpc_key'], $actor, [ 'instance_id' => $id * 100, 'request_key' => 'auto-member-' . $id, - 'auto_power' => true, ]); - return CloudVpcOperation::where('vpc_id', $id)->orderByDesc('desired_revision')->first(); + $operation = CloudVpcOperation::where('vpc_id', $id) + ->orderByDesc('desired_revision')->first(); + $instance = CloudInstance::find($id * 100); + $member = CloudVpcMember::where('instance_id', $id * 100)->first(); + // Manufacture a pre-refactor row to verify compatibility recovery. New + // public attach/detach requests never initialize this workflow. + Workflow::initialize($operation, $instance, $member); + CloudEngineClient::$power = $power; + return $operation->fresh(); +}; +$legacyDetach = static function (CloudVpc $vpc, CloudVpcMember $member, + string $requestKey) use ($actor): CloudVpcOperation { + CloudVpcService::detachMember($vpc->vpc_key, $member->member_key, $actor, [ + 'request_key' => $requestKey, + ]); + $operation = CloudVpcOperation::where('vpc_id', $vpc->id) + ->orderByDesc('desired_revision')->first(); + Workflow::initialize( + $operation, + CloudInstance::find((int) $member->instance_id), + $member->fresh() + ); + return $operation->fresh(); }; $claim = static function (CloudVpcOperation $operation): array { $operation = $operation->fresh(); @@ -206,14 +226,35 @@ $assert(Workflow::beforeApply($op, $lease, $actor, false) && count(CloudEngineClient::$commands) === $commands, 'Accepted shutdown did not recover after a transient state failure.'); -// Immutable API replay includes the explicit maintenance acknowledgement. +// A historical auto-power request remains exactly replayable during a rolling +// deployment, but a new request can no longer create that workflow. $vpc = CloudVpc::find($completedOp->vpc_id); -$expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [ - 'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => false, -]), 409); +$legacyPayload = json_decode((string) $completedOp->payload_json, true); +$legacySubnetKey = (string) ($legacyPayload['members'][0]['subnet_id'] ?? ''); +$completedOp->request_hash = hash('sha256', $canonicalJson([ + 'kind' => 'member_attach', + 'input' => [ + 'subnet_key' => $legacySubnetKey, + 'instance_id' => 120100, + 'private_ip' => null, + 'auto_power' => true, + ], +])); +$completedOp->save(); CloudVpcService::attachMember($vpc->vpc_key, $actor, [ 'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true, ]); +$expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [ + 'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => false, +]), 409); +$retired = $expectCloudError(static fn () => CloudVpcService::attachMember( + $vpc->vpc_key, $actor, [ + 'instance_id' => 120100, 'request_key' => 'new-auto-member-1201', + 'auto_power' => true, + ] +), 422); +$assert($retired->errorCode === 'vpc_auto_power_retired', + 'A new automatic-power workflow was still admitted.'); $expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [ 'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true, 'subnet_key' => str_repeat('a', 32), @@ -795,10 +836,7 @@ $assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded' && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $commands, 'A guest-only stopped state was not retained without replaying apply.'); -CloudVpcService::detachMember($root->vpc_key, $member->member_key, $actor, [ - 'request_key' => 'auto-detach-1210', 'auto_power' => true, -]); -$detach = CloudVpcOperation::where('vpc_id', $root->id)->orderByDesc('desired_revision')->first(); +$detach = $legacyDetach($root, $member, 'auto-detach-1210'); $tick($detach); CloudEngineClient::$power = 'shutoff'; // Losing the start response keeps the fixed child in automatic receipt/state @@ -810,7 +848,7 @@ $assert($phase($detach) === 'starting' && $detach->fresh()->status === 'waiting_ 'Last-member teardown did not preserve restoration workflow after removal.'); $other = $seedVpc(1211, '10.121.211.0/24', '10.121.211.0/24', 10); $expectCloudError(static fn () => CloudVpcService::attachMember($other['vpc_key'], $actor, [ - 'instance_id' => 121000, 'request_key' => 'cross-attach-121000', 'auto_power' => true, + 'instance_id' => 121000, 'request_key' => 'cross-attach-121000', ]), 409); $count = count(CloudEngineClient::$commands); $tick($detach); $tick($detach); @@ -855,10 +893,15 @@ $seedInstance(128001, 10); CloudEngineClient::$power = 'running'; CloudVpcService::attachMember($multiHostVpc['vpc_key'], $actor, [ 'instance_id' => 128001, 'request_key' => 'multi-host-auto-attach-128001', - 'auto_power' => true, ]); $multiAttach = CloudVpcOperation::where('vpc_id', 1280) ->orderByDesc('desired_revision')->first(); +Workflow::initialize( + $multiAttach, + CloudInstance::find(128001), + CloudVpcMember::where('instance_id', 128001)->first() +); +$multiAttach->refresh(); $tick($multiAttach); CloudEngineClient::$power = 'shutoff'; $multiPayload = json_decode($multiAttach->payload_json, true); @@ -917,7 +960,8 @@ CloudEngineClient::$vpcRespond = $ordinaryVpcRespond; CloudEngineClient::$vpcRead = null; $partialDetach = static function (int $id, bool $sameHost = false, ?string $fault = null) use ( - $make, $tick, $actor, $seedInstance, $ordinaryVpcRespond, $runningVpcProof, $assert + $make, $tick, $actor, $seedInstance, $ordinaryVpcRespond, $runningVpcProof, + $legacyDetach, $assert ): array { Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]); CloudEngineClient::$vpcRespond = $ordinaryVpcRespond; @@ -936,10 +980,7 @@ $partialDetach = static function (int $id, bool $sameHost = false, ?string $faul CloudVpcService::reconcile($vpc->vpc_key, $actor, true, true); } $target = CloudVpcMember::where('instance_id', $id * 100)->first(); - CloudVpcService::detachMember($vpc->vpc_key, $target->member_key, $actor, [ - 'request_key' => 'partial-detach-' . $id, 'auto_power' => true, - ]); - $operation = CloudVpcOperation::where('vpc_id', $vpc->id)->orderByDesc('desired_revision')->first(); + $operation = $legacyDetach($vpc, $target, 'partial-detach-' . $id); $tick($operation); CloudEngineClient::$power = 'shutoff'; $full = $runningVpcProof(json_decode($operation->payload_json, true)); diff --git a/web/tests/CloudVpcTombstoneConvergenceTest.php b/web/tests/CloudVpcTombstoneConvergenceTest.php index 472a853d..563adbb2 100644 --- a/web/tests/CloudVpcTombstoneConvergenceTest.php +++ b/web/tests/CloudVpcTombstoneConvergenceTest.php @@ -2307,4 +2307,71 @@ $assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation ->where('host_id', 180)->value('state') === 'deleted', 'Stopping the VM did not converge the original detach operation.'); +// A stopped-only admission failure is an exact, pre-write result. Preserve +// its reviewed reason so the tenant sees the required action instead of a +// generic readback mismatch; the same immutable operation remains retryable. +foreach ([ + 29 => 'vpc_member_requires_stopped', + 30 => 'vpc_member_power_unavailable', +] as $preflightVpcId => $preflightCode) { + $revision = $preflightVpcId * 2; + $vpcKey = substr(hash('sha256', 'preflight-vpc-' . $preflightVpcId), 0, 32); + $digest = hash('sha256', 'preflight-digest-' . $preflightVpcId); + $operationId = substr(hash('sha256', 'preflight-operation-' . $preflightVpcId), 0, 32); + $memberKey = substr(hash('sha256', 'preflight-member-' . $preflightVpcId), 0, 32); + $subnetId = 3000 + $preflightVpcId; + $memberId = 3000 + $preflightVpcId; + $leaseId = 4000 + $preflightVpcId; + $instanceId = 28000 + $preflightVpcId; + $hostId = 280 + $preflightVpcId; + $uuid = sprintf('00000000-0000-4000-8000-%012d', $preflightVpcId); + + $seedVpc($preflightVpcId, $vpcKey, $revision, $digest, + $operationId, 'member_attach'); + $seedSubnet($subnetId, $preflightVpcId, + substr(hash('sha256', 'preflight-subnet-' . $preflightVpcId), 0, 32), + '10.' . (20 + $preflightVpcId) . '.0.0/25', 'applying', + $revision, $digest); + $seedMember($memberId, $leaseId, $preflightVpcId, $subnetId, + $memberKey, $instanceId, $uuid, $hostId, + '10.' . (20 + $preflightVpcId) . '.0.10', 'applying', + $revision, $digest); + $seedHost($preflightVpcId, $hostId, 'applying', $revision, + $digest, $operationId); + + CloudVpcService::recordObserved($vpcKey, $revision, $digest, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, + 'revision' => 0, 'digest' => null, 'state' => 'applying', + 'next_stage_allowed' => false, + 'private_dataplane_verified' => false, + 'fabric_gate' => ['active' => true], + 'hosts' => [(string) $hostId => [ + 'vpc_id' => $vpcKey, 'host_id' => (string) $hostId, + 'desired_revision' => $revision, 'desired_digest' => $digest, + 'observed_revision' => 0, 'observed_digest' => null, + 'state' => 'applying', 'transport_verified' => true, + 'private_dataplane_verified' => false, + 'config_verified' => false, + 'config_observed_revision' => 0, + 'config_observed_digest' => null, + 'removed' => false, 'last_error_code' => $preflightCode, + ]], + ], $operationId); + + $assert(Capsule::table('cloud_vpc_operations') + ->where('operation_id', $operationId)->value('status') + === 'waiting_observed' + && Capsule::table('cloud_vpc_operations') + ->where('operation_id', $operationId)->value('error_code') + === $preflightCode + && Capsule::table('cloud_vpcs')->where('id', $preflightVpcId) + ->value('sync_state') === 'waiting' + && Capsule::table('cloud_vpcs')->where('id', $preflightVpcId) + ->value('last_error_code') === $preflightCode + && Capsule::table('cloud_vpc_host_deployments') + ->where('vpc_id', $preflightVpcId)->where('host_id', $hostId) + ->value('last_error_code') === $preflightCode, + 'A stopped-only preflight reason was replaced by a generic readback error.'); +} + echo "VPC member/IP/subnet tombstone convergence contracts passed.\n"; diff --git a/web/tests/CloudVpcUserPageContractTest.php b/web/tests/CloudVpcUserPageContractTest.php index 75ef913b..87ea5f9e 100644 --- a/web/tests/CloudVpcUserPageContractTest.php +++ b/web/tests/CloudVpcUserPageContractTest.php @@ -150,11 +150,13 @@ $assert($memberRender !== '' && $memberSubmit !== '' && !str_contains($memberSubmit, 'subnet_key') && !str_contains($memberSubmit, 'private_ip'), 'Tenant member attach must preserve server-assigned subnet and address fields.'); -$assert(str_contains($memberSubmit, 'auto_power: true') - && str_contains($memberSubmit, '关机超时会暂停') +$assert(!str_contains($memberSubmit, 'auto_power') + && str_contains($memberSubmit, '请先确认实例已完全关机') + && str_contains($memberSubmit, '平台不会自动关机或开机') + && str_contains($memberSubmit, '实例仍保持关机') && str_contains($memberSubmit, 'if (!confirm(') && str_contains($template, '默认允许本 /24 网段入站和出站通信'), - 'Automatic member power handling needs an explicit downtime confirmation and own-CIDR defaults.'); + 'Stopped-only member handling needs an explicit prerequisite confirmation and own-CIDR defaults.'); $memberChoiceStart = strpos($script, 'function memberChoice('); $memberChoiceEnd = strpos($script, 'function renderMembers(', $memberChoiceStart ?: 0); $memberChoices = $memberChoiceStart !== false && $memberChoiceEnd !== false