diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index 5d13351d..1524a539 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -2234,6 +2234,8 @@ def test_power_action_has_persistent_idempotency(boundary): assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False saved = json.loads(store.get("operation", "reboot-operation")) assert saved["target_action"] == "reboot" + assert saved["mutation_started"] is True + assert operation["mutation_started"] is True def test_orphaned_power_action_reconciles_only_the_final_target_state(boundary): diff --git a/engine/tests/test_libvirt_service.py b/engine/tests/test_libvirt_service.py index 1cf429f4..647168f4 100644 --- a/engine/tests/test_libvirt_service.py +++ b/engine/tests/test_libvirt_service.py @@ -5396,6 +5396,41 @@ class VMServiceTests(unittest.TestCase): self.assertEqual(result[0].vm_id, "00000000-0000-4000-8000-000000000002") self.assertEqual(self.backend.opened, self.backend.closed) + def test_metric_counters_reject_ambiguous_extra_nic_role(self): + constants = { + "VIR_DOMAIN_STATS_CPU_TOTAL": 2, + "VIR_DOMAIN_STATS_BALLOON": 4, + "VIR_DOMAIN_STATS_BLOCK": 8, + "VIR_DOMAIN_STATS_INTERFACE": 16, + "VIR_CONNECT_GET_ALL_DOMAINS_STATS_RUNNING": 32, + } + for name, value in constants.items(): + setattr(self.service_module.libvirt, name, value) + domain = types.SimpleNamespace( + UUIDString=lambda: "00000000-0000-4000-8000-000000000003", + info=lambda: (1, 2048 * 1024, 2048 * 1024, 2, 0), + ID=lambda: 9, + XMLDesc=lambda _flags: ("" + "" + "" + "" + "" + ""), + ) + + def get_stats(_connection, _groups, _flags): + return [(domain, {"cpu.time": 123, "net.count": 2, + "net.0.name": "vnet9", "net.0.rx.bytes": 456, + "net.0.tx.bytes": 789, + "net.1.name": "vnet10", "net.1.rx.bytes": 12, + "net.1.tx.bytes": 34})] + + with patch.object(FakeConnection, "getAllDomainStats", get_stats, create=True): + result = self.service.metric_counters() + self.assertEqual(len(result), 1) + self.assertEqual(result[0].values["net.0.role"], "public") + self.assertEqual(result[0].values["net.1.role"], "unknown") + if __name__ == "__main__": unittest.main() diff --git a/engine/tests/test_metrics.py b/engine/tests/test_metrics.py index d75ecfe2..b6364371 100644 --- a/engine/tests/test_metrics.py +++ b/engine/tests/test_metrics.py @@ -496,6 +496,34 @@ def test_sampler_prefers_stable_network_identity_over_renamed_host_tap(): } +def test_sampler_billing_frame_only_contains_explicit_public_role(): + sampler = MetricSampler() + point = sampler.observe(host_scope(HOST), raw(extra={ + "net.count": 2, + "net.0.identity": "mac:52:54:00:12:34:56", + "net.0.role": "public", + "net.1.name": "vnet2", + "net.1.identity": "mac:52:54:00:aa:bb:cc", + "net.1.role": "private", + "net.1.rx.bytes": 3000, + "net.1.tx.bytes": 4000, + }), timestamp=1000, monotonic=10) + assert point["network_billing_devices"] == { + "mac:52:54:00:12:34:56": [1000, 2000], + } + + +def test_sampler_billing_rejects_ambiguous_multinic_frame(): + sampler = MetricSampler() + point = sampler.observe(host_scope(HOST), raw(extra={ + "net.count": 2, + "net.1.name": "vnet2", + "net.1.rx.bytes": 3000, + "net.1.tx.bytes": 4000, + }), timestamp=1000, monotonic=10) + assert point["network_billing_devices"] is None + + def test_collector_associates_host_and_current_owner_without_hiding_unowned_vms(): saved = [] diff --git a/engine/tests/test_private_network.py b/engine/tests/test_private_network.py new file mode 100644 index 00000000..9ee7e298 --- /dev/null +++ b/engine/tests/test_private_network.py @@ -0,0 +1,88 @@ +from virsh_panel.private_network import observe_private_state_from_xml + + +def _xml(*, private=True, link="up", private_source="vc-private", + private_ip="10.24.3.7/32", verified="true", private_filter=False, + public_source="br0", private_type="bridge"): + private_block = "" + if private: + filt = '' if private_filter else "" + source = (f'' + if private_type == "bridge" + else f'') + private_block = f''' + + {source} + {filt} + + ''' + return f'''00000000-0000-4000-8000-000000000001 + + + + + {private_block} + ''' + + +def test_private_state_without_private_nic_is_confirmed_disabled(): + state = observe_private_state_from_xml(_xml(private=False), vm_uuid="00000000-0000-4000-8000-000000000001", observed_at=10) + assert state["state"] == "disabled" + assert state["readback"] == "confirmed" + assert state["interface"] is None + + +def test_private_state_requires_verified_independent_bridge_and_ip(): + state = observe_private_state_from_xml(_xml(), observed_at=10) + assert state["state"] == "degraded" + assert state["reason_code"] == "private_source_unverified" + state = observe_private_state_from_xml(_xml(), host_verified=True, observed_at=10) + assert state["state"] == "active" + assert state["readback"] == "confirmed" + assert state["private_ip_cidr"] == "10.24.3.7/32" + + unverified = observe_private_state_from_xml(_xml(verified="false")) + assert unverified["state"] == "degraded" + assert unverified["reason_code"] == "private_source_unverified" + + missing_ip = observe_private_state_from_xml(_xml(private_ip=None)) + assert missing_ip["state"] == "degraded" + assert missing_ip["reason_code"] == "private_ip_missing" + + +def test_private_state_marks_link_down_suspended(): + state = observe_private_state_from_xml(_xml(link="down"), host_verified=True) + assert state["state"] == "suspended" + assert state["link_state"] == "down" + + +def test_private_state_rejects_public_source_or_filter(): + shared = observe_private_state_from_xml(_xml(private_source="br0")) + assert shared["state"] == "degraded" + assert shared["reason_code"] == "private_source_invalid" + + filtered = observe_private_state_from_xml(_xml(private_filter=True), host_verified=True) + assert filtered["state"] == "degraded" + assert filtered["reason_code"] == "private_filter_present" + + +def test_private_network_source_network_is_not_proven_isolated(): + state = observe_private_state_from_xml( + _xml(private_source="default", private_type="network"), host_verified=True + ) + # The helper only accepts an explicit host bridge; a libvirt network name + # does not prove which bridge it resolves to. + assert state["state"] == "degraded" + assert state["reason_code"] == "private_source_invalid" + + +def test_private_state_rejects_ambiguous_extra_nic_without_private_role(): + xml = _xml(private=False).replace( + '', + '' + '', + ) + state = observe_private_state_from_xml(xml) + assert state["state"] == "degraded" + assert state["reason_code"] == "private_interface_role_invalid" diff --git a/engine/tests/test_vpc.py b/engine/tests/test_vpc.py index 00c5e050..f854c8bb 100644 --- a/engine/tests/test_vpc.py +++ b/engine/tests/test_vpc.py @@ -292,6 +292,7 @@ def test_coordinator_requires_exact_fabric_revision_and_all_host_readback(): observed = asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True, + "private_dataplane_verified": True, "readback": {"safe": "ok", "oversized": "x" * 513}}, })) assert observed["state"] == "active" @@ -303,6 +304,25 @@ def test_coordinator_requires_exact_fabric_revision_and_all_host_readback(): assert blocked["next_stage_allowed"] is False +def test_transport_only_observation_never_opens_private_dataplane_gate(): + store = MemoryStore() + fabric = FakeFabric(hosts=("host-a",)) + coordinator = VpcCoordinator(store, fabric) + result = asyncio.run(coordinator.apply(payload())) + digest = result["desired_digest"] + pending = asyncio.run(coordinator.observe("vpc-hk-1", { + "host-a": {"state": "active", "observed_revision": 1, + "observed_digest": digest, "transport_verified": True}, + })) + assert pending["state"] == "applying" + assert pending["next_stage_allowed"] is False + assert pending["private_dataplane_verified"] is False + assert pending["hosts"][0]["state"] == "applying" + assert pending["hosts"][0]["last_error_code"] == ( + "vpc_private_dataplane_unverified" + ) + + def test_partial_host_observation_is_degraded_and_never_active(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a", "host-b")) @@ -322,7 +342,8 @@ def test_partial_host_observation_is_degraded_and_never_active(): digest = result["desired_digest"] observed = asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, - "observed_digest": digest, "transport_verified": True}, + "observed_digest": digest, "transport_verified": True, + "private_dataplane_verified": True}, "host-b": {"state": "degraded", "observed_revision": 0, "observed_digest": None, "transport_verified": False}, })) @@ -331,7 +352,8 @@ def test_partial_host_observation_is_degraded_and_never_active(): with pytest.raises(VpcError) as inconsistent: asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, - "observed_digest": digest, "transport_verified": False}, + "observed_digest": digest, "transport_verified": False, + "private_dataplane_verified": True}, "host-b": {"state": "degraded", "observed_revision": 0, "observed_digest": None, "transport_verified": False}, })) @@ -454,7 +476,8 @@ def test_recover_invalidates_observed_gate_and_record_round_trips(): raw["observed_digest"] = raw["desired_digest"] raw["hosts"][0].update(state="active", observed_revision=1, observed_digest=raw["desired_digest"], - transport_verified=True) + transport_verified=True, + private_dataplane_verified=True) store.set("vpc_desired", "vpc-hk-1", json.dumps(raw)) coordinator.recover() recovered = json.loads(store.get("vpc_desired", "vpc-hk-1")) @@ -492,7 +515,8 @@ def test_new_desired_revision_keeps_last_observed_evidence_until_readback(): digest = first["desired_digest"] asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, - "observed_digest": digest, "transport_verified": True}, + "observed_digest": digest, "transport_verified": True, + "private_dataplane_verified": True}, })) second = asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", diff --git a/engine/virsh_panel/app.py b/engine/virsh_panel/app.py index b5fd5bec..04e44e6e 100644 --- a/engine/virsh_panel/app.py +++ b/engine/virsh_panel/app.py @@ -3862,6 +3862,34 @@ def create_app( require_vm_access(_session, profile, vm_id) return new_service(profile).get_vm_state(vm_id) + @app.get("/api/v1/vms/{vm_id}/private-state") + def get_vm_private_state(vm_id: str, _session: Session = Depends(require_approved)): + """Return the bounded private-NIC readback for one VM. + + This endpoint intentionally reports the domain/XML observation only; + VPC revision, WireGuard peer and nftables evidence are checked by the + PHP control plane before it publishes a mutation as successful. + """ + profile = current_profile() + require_vm_access(_session, profile, vm_id) + service = new_service(profile) + getter = getattr(service, "get_private_state", None) + if callable(getter): + return getter(vm_id) + # Test/rolling-upgrade adapters may only expose get_vm; preserve the + # same bounded contract without making the route a second XML parser. + vm = service.get_vm(vm_id) + return vm.get("private_state", { + "version": 1, "required": False, "vm_uuid": vm.get("uuid"), + "vpc_id": None, "member_id": None, "interface": None, + "state": "uncertain", "desired_revision": 0, + "observed_revision": 0, "desired_digest": None, + "observed_digest": None, "readback": "missing", + "link_state": "absent", "private_ip_cidr": None, + "private_mac": None, "reason_code": "private_state_unavailable", + "observed_at": int(time.time()), + }) + @app.get("/api/v1/vms/{vm_id}") def get_vm(vm_id: str, _session: Session = Depends(require_approved)): profile = current_profile() diff --git a/engine/virsh_panel/database.py b/engine/virsh_panel/database.py index e9866619..a54af544 100644 --- a/engine/virsh_panel/database.py +++ b/engine/virsh_panel/database.py @@ -337,6 +337,11 @@ class DatabaseStore: or user_id > _MAX_UNSIGNED_BIGINT)): raise ValueError("Metric user ID is invalid") raw_devices = point.get("network_devices") + # New collectors provide a role-filtered public0 frame for quota + # accounting. Keep the legacy all-NIC fallback for old callers + # and old hosts that have not emitted explicit role metadata. + if "network_billing_devices" in point: + raw_devices = point.get("network_billing_devices") # None means libvirt did not return a complete NIC counter frame. # Never replace a durable baseline with incomplete data, otherwise # the next healthy sample would look like a brand-new NIC and the diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index e2d31a78..df7ecfbe 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -137,7 +137,7 @@ READ_PATHS = ( r"fabric", r"fabric/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}", r"vpcs", r"vpcs/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}", r"user-isos/[0-9a-f]{32}", - rf"vms/{VM_ID}(?:/state|/metrics|/traffic/daily|/rescue|" + rf"vms/{VM_ID}(?:/state|/private-state|/metrics|/traffic/daily|/rescue|" r"/firewall|/snapshots(?:/[0-9a-f]{32})?)?", r"(?:reinstall-jobs|rescue-jobs|vm-templates/import-jobs|rescue-templates/import-jobs|user-isos/import-jobs)/[A-Za-z0-9_-]+", ) @@ -868,6 +868,16 @@ class EngineApplication: elif self._is_vm_delete_operation(operation): await self._capture_delete_cleanup( host_id, operation, delegated) + # Persist the mutation boundary before invoking the child. + # If this write fails, the child is not called and the + # operation remains a safe prevalidation/coordination failure. + # Network changes have a narrower checkpoint inside the child; + # PHP additionally requires network_mutation_started for that + # resource before gating failed results. + operation["mutation_started"] = True + await asyncio.to_thread( + self.store.set, "operation", operation["operation_id"], + json.dumps(operation)) await app(child_scope, receive, capture) status = next(item["status"] for item in messages if item["type"] == "http.response.start") payload = json.loads(b"".join(item.get("body", b"") for item in messages @@ -914,8 +924,9 @@ class EngineApplication: if operation["status"] in {"succeeded", "pending"}: await json_response(send, status, {**payload, **self._public_operation(operation)}) else: - await json_response(send, status, {**payload, "operation_id": operation["operation_id"], - "status": operation["status"], "safe_to_refund": operation["safe_to_refund"]}) + await json_response(send, status, { + **payload, **self._public_operation(operation), + }) except BaseException as exc: network_preflight_failed = ( self._is_network_change_resource( @@ -1922,7 +1933,11 @@ class EngineApplication: "actor_can_manage": bool(delegated.get("can_manage")), "actor_is_admin": bool(delegated.get("is_admin")), "resource": resource, "method": method, "status": "running", "result": None, - "safe_to_refund": False, "created_at": int(time.time())} + "safe_to_refund": False, "created_at": int(time.time()), + # Set immediately before invoking the child endpoint. This + # lets the PHP worker distinguish an actual mutation boundary + # from a durable prevalidation failure. + "mutation_started": False} if create: # New PHP-backed creates always receive the reserved UUID below. # Keep this durable marker so cleanup reconciliation checks the @@ -3166,6 +3181,11 @@ class EngineApplication: "operation_id", "status", "result", "error", "safe_to_refund", "service_was_running_hint", "created_at", "finished_at") if key in record} + # Expose only bounded lifecycle markers. Do not leak persisted + # request bodies or host diagnostics through the operation API. + for key in ("mutation_started", "network_mutation_started"): + if isinstance(record.get(key), bool): + result[key] = record[key] if ("/snapshots" in record.get("resource", "") and record.get("error_code") in SNAPSHOT_PUBLIC_ERROR_CODES): result["error_code"] = record["error_code"] diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index eb5807bb..9675ab08 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -41,6 +41,7 @@ from virsh_panel.network_roles import ( select_private_interface, select_public_interface, ) +from virsh_panel.private_network import observe_private_state from virsh_panel.platform_network_policy import ( MAX_PLATFORM_POLICY_SNAPSHOT_BYTES, PLATFORM_DEFAULT_NETWORK_POLICY, @@ -1219,6 +1220,13 @@ class VMService: rescue.get("volume") if rescue is not None else inactive_rescue.get("volume") if inactive_rescue is not None else None), "console_port": self._vnc_port(root), + # This is a bounded XML observation only. It deliberately does + # not claim that the host WireGuard Fabric, guest route or nft + # ACL is active; the PHP VPC ledger supplies that expectation at + # each mutation commit boundary. + "private_state": observe_private_state( + root, vm_uuid=domain.UUIDString() + ), } @staticmethod @@ -1465,6 +1473,18 @@ class VMService: with self._connect() as conn: return self._vm_dict(self._domain(conn, uuid)) + def get_private_state(self, uuid: str) -> dict[str, Any]: + """Read one VM's bounded private-NIC observation. + + Keep this as a separate endpoint so lifecycle reconciliation does not + need to load disk capacities or console metadata, and so a host-wide + inventory cannot accidentally execute a per-VM network probe. + """ + with self._connect() as conn: + domain = self._domain(conn, uuid) + root = self._vm_xml(domain) + return observe_private_state(root, vm_uuid=domain.UUIDString()) + def get_vm_state(self, uuid: str) -> dict[str, Any]: """Fetch one domain's power state without loading its XML metadata.""" with self._connect() as conn: @@ -1500,8 +1520,39 @@ class VMService: values = dict(values) try: root = ET.fromstring(domain.XMLDesc(0)) - by_target: dict[str, str] = {} - for interface in root.findall("./devices/interface"): + interfaces = root.findall("./devices/interface") + # Resolve the complete role map before annotating any + # counters. Calling ``interface_role(..., + # legacy_public=True)`` independently would classify an + # extra unaliased NIC as public on a multi-NIC domain and + # could charge private traffic to the public quota lane. + try: + public_interface = select_public_interface(interfaces) + private_interface = select_private_interface(interfaces) + role_by_element = { + id(item): ( + "public" if item is public_interface + else "private" if item is private_interface + else "unknown" + ) + for item in interfaces + } + except (TypeError, ValueError): + # Preserve explicit roles for diagnostics while + # marking the complete frame unsafe. The billing + # sampler rejects any ``unknown`` role, but retaining + # the known public role keeps the observation useful + # for reconciliation and avoids silently relabelling + # a valid public0 interface. + role_by_element = {} + for item in interfaces: + try: + role_by_element[id(item)] = interface_role( + item, legacy_public=len(interfaces) == 1) + except ValueError: + role_by_element[id(item)] = "unknown" + by_target: dict[str, tuple[str, str]] = {} + for interface in interfaces: target = interface.find("target") mac = interface.find("mac") target_name = target.get("dev") if target is not None else None @@ -1510,14 +1561,18 @@ class VMService: and isinstance(address, str) and re.fullmatch(r"[0-9A-Fa-f]{2}(?::[0-9A-Fa-f]{2}){5}", address)): - by_target[target_name] = "mac:" + address.lower() + role = role_by_element.get(id(interface), "unknown") + by_target[target_name] = ( + "mac:" + address.lower(), role + ) count = values.get("net.count") if isinstance(count, int) and not isinstance(count, bool): for number in range(min(max(0, count), 1024)): name = values.get(f"net.{number}.name") identity = by_target.get(name) if isinstance(name, str) else None if identity is not None: - values[f"net.{number}.identity"] = identity + values[f"net.{number}.identity"] = identity[0] + values[f"net.{number}.role"] = identity[1] except (AttributeError, ET.ParseError, libvirt.libvirtError): # Metrics remain useful on older/unusual domains whose # live XML cannot be read; accounting falls back to the diff --git a/engine/virsh_panel/metrics.py b/engine/virsh_panel/metrics.py index 52ad5e46..29b6a51d 100644 --- a/engine/virsh_panel/metrics.py +++ b/engine/virsh_panel/metrics.py @@ -86,7 +86,9 @@ def _device_counters(stats: Mapping[str, Any], group: str, return signature, counters_by_direction -def _network_counter_devices(stats: Mapping[str, Any]) -> dict[str, list[int]] | None: +def _network_counter_devices( + stats: Mapping[str, Any], *, role: str | None = None, +) -> dict[str, list[int]] | None: """Return validated cumulative counters keyed by stable guest NIC identity. libvirt reports rx/tx from the guest's point of view: rx is downloaded by @@ -99,20 +101,52 @@ def _network_counter_devices(stats: Mapping[str, Any]) -> dict[str, list[int]] | if count is None or count > 1024: return None candidates: dict[str, list[int] | None] = {} + seen_identities: set[str] = set() + # A role-filtered frame is safe to use for billing only when every NIC + # declares a role. The one-NIC legacy format has no role field and is + # still unambiguous; a multi-NIC frame without roles is not. + explicit_role_keys = [f"net.{number}.role" in stats for number in range(count)] + has_explicit_roles = any(explicit_role_keys) + if role is not None and not has_explicit_roles and count not in {0, 1}: + return None for number in range(count): name = stats.get(f"net.{number}.name") identity = stats.get(f"net.{number}.identity", name) + observed_role = stats.get(f"net.{number}.role") + if (not isinstance(identity, str) or not 1 <= len(identity) <= 255 + or any(ord(char) < 32 or ord(char) == 127 for char in identity)): + return None + if identity in seen_identities: + return None + seen_identities.add(identity) + if role is not None: + if has_explicit_roles: + if observed_role not in {"public", "private", "unknown"}: + # A partially labelled multi-NIC frame cannot establish + # which interface belongs to the public quota lane. + return None + if observed_role == "unknown": + return None + if observed_role != role: + continue + elif observed_role is not None and observed_role != role: + continue rx = _nonnegative_int(stats.get(f"net.{number}.rx.bytes")) tx = _nonnegative_int(stats.get(f"net.{number}.tx.bytes")) - if (not isinstance(identity, str) or not 1 <= len(identity) <= 255 - or any(ord(char) < 32 or ord(char) == 127 for char in identity) - or rx is None or tx is None + if (rx is None or tx is None or rx > _MAX_UNSIGNED_BIGINT or tx > _MAX_UNSIGNED_BIGINT): return None - # Duplicate names cannot be safely matched to a previous observation. + # Duplicate identities were rejected above, including across + # selected and unselected roles, so a bad private NIC cannot alias a + # billable public NIC. if identity in candidates: return None candidates[identity] = [rx, tx] + if role is None: + return {identity: counters for identity, counters in candidates.items() + if counters is not None} + # ``candidates`` contains only the requested role when explicit roles are + # present; for the legacy one-NIC path it contains the sole public NIC. return {identity: counters for identity, counters in candidates.items() if counters is not None} @@ -210,6 +244,13 @@ class MetricSampler: # totals here. The database store locks and advances its durable # baseline in the same transaction as this sample. "network_devices": _network_counter_devices(values), + # Once explicit public0/private0 roles are available, quota + # accounting must not charge east-west VPC traffic as public + # traffic. Legacy frames without role metadata continue through + # the historical fallback in DatabaseStore. + "network_billing_devices": _network_counter_devices( + values, role="public" + ), } self._previous[key] = _Baseline(monotonic, raw.generation, raw.vcpus, cpu_time, disk_signature, disk, network_signature, network) diff --git a/engine/virsh_panel/private_network.py b/engine/virsh_panel/private_network.py new file mode 100644 index 00000000..13d09d0d --- /dev/null +++ b/engine/virsh_panel/private_network.py @@ -0,0 +1,202 @@ +"""Bounded observation of a VM's private network role. + +This module deliberately only reads the domain definition. It does not run +``ip``, ``wg`` or ``nft`` commands and it never treats the host Fabric status +as proof that a tenant NIC is connected. The PHP control plane supplies the +VPC/member revision when it evaluates this observation at an operation +commit boundary. +""" + +from __future__ import annotations + +import ipaddress +import re +import time +import xml.etree.ElementTree as ET +from typing import Any + +from virsh_panel.network_roles import ( + PRIVATE_INTERFACE_ALIAS, + interface_role, + private_interface_has_public_filter, + select_private_interface, + select_public_interface, +) + + +PRIVATE_STATE_VERSION = 1 +PANEL_NS = "urn:virsh-panel:1" +_MAC_RE = re.compile(r"[0-9a-f]{2}(?::[0-9a-f]{2}){5}\Z") +_DIGEST_RE = re.compile(r"[0-9a-f]{64}\Z") + + +def _source(interface: ET.Element) -> tuple[str | None, str | None]: + source = interface.find("source") + if source is None: + return None, None + for kind in ("bridge", "network", "dev", "name", "path"): + value = source.get(kind) + if isinstance(value, str) and value.strip(): + return kind, value.strip() + return None, None + + +def _managed_metadata(root: ET.Element) -> ET.Element | None: + # Only the panel-owned namespace is trusted for the optional identity + # metadata. Guest/user XML must never become a host-state assertion. + for node in root.findall("./metadata"): + for child in list(node): + if child.tag == f"{{{PANEL_NS}}}managed": + return child + return None + + +def _private_ip(root: ET.Element, managed: ET.Element | None) -> str | None: + values: list[str | None] = [] + if managed is not None: + values.extend((managed.get("private-ip-cidr"), managed.get("private_ip_cidr"))) + for node in root.findall("./metadata/*"): + if node.tag.rsplit("}", 1)[-1] in {"private-network", "private_state"}: + values.extend((node.get("ip-cidr"), node.get("private-ip-cidr"), node.get("cidr"))) + for value in values: + if not isinstance(value, str) or not value: + continue + try: + address = ipaddress.ip_interface(value) + except ValueError: + continue + if isinstance(address, ipaddress.IPv4Interface) and address.network.prefixlen == 32: + return str(address) + return None + + +def _revision(managed: ET.Element | None) -> int: + if managed is None: + return 0 + raw = managed.get("private-network-revision") or managed.get("private_revision") + try: + value = int(raw or 0) + except (TypeError, ValueError): + return 0 + return value if 0 <= value <= (1 << 63) - 1 else 0 + + +def _digest(managed: ET.Element | None) -> str | None: + if managed is None: + return None + value = managed.get("private-network-digest") or managed.get("private_digest") + return value.lower() if isinstance(value, str) and _DIGEST_RE.fullmatch(value.lower()) else None + + +def observe_private_state( + root: ET.Element, + *, + vm_uuid: str | None = None, + host_verified: bool = False, + observed_at: int | None = None, +) -> dict[str, Any]: + """Return a stable, credential-free private NIC observation. + + ``disabled/confirmed`` means that the domain has no private role and no + VPC expectation was supplied to this read. A caller that has a VPC + expectation must independently change that result to ``degraded`` or + ``uncertain`` until the expected member/revision/digest is verified. + """ + now = int(time.time()) if observed_at is None else int(observed_at) + base: dict[str, Any] = { + "version": PRIVATE_STATE_VERSION, + "required": False, + "vm_uuid": vm_uuid, + "vpc_id": None, + "member_id": None, + "interface": None, + "state": "disabled", + "desired_revision": 0, + "observed_revision": 0, + "desired_digest": None, + "observed_digest": None, + "readback": "confirmed", + "link_state": "absent", + "private_ip_cidr": None, + "private_mac": None, + "reason_code": None, + "observed_at": now, + } + interfaces = list(root.findall("./devices/interface")) + # Validate the complete role map even when no private NIC is present. A + # second unlabelled/ambiguous NIC must not disappear behind a successful + # public-only observation and later become an accidental data-plane path. + try: + public = select_public_interface(interfaces) + private = select_private_interface(interfaces) + except (TypeError, ValueError): + base.update(state="degraded", readback="mismatch", + reason_code="private_interface_role_invalid") + return base + if private is None: + return base + + base["interface"] = PRIVATE_INTERFACE_ALIAS + managed = _managed_metadata(root) + base["private_ip_cidr"] = _private_ip(root, managed) + base["observed_revision"] = _revision(managed) + base["observed_digest"] = _digest(managed) + mac_node = private.find("mac") + mac = mac_node.get("address", "").lower() if mac_node is not None else "" + base["private_mac"] = mac or None + link = private.find("link") + link_state = link.get("state", "up").lower() if link is not None else "up" + base["link_state"] = link_state if link_state in {"up", "down"} else "absent" + + private_kind, private_source = _source(private) + public_kind, public_source = _source(public) if public is not None else (None, None) + # The tenant data plane is an independent host bridge. A libvirt + # ``network``/``direct`` source does not let this layer prove which host + # bridge it resolves to, so accepting it would make a public bridge reuse + # indistinguishable from a private network. The future data-plane apply + # path may add an explicit verification marker to the managed metadata. + if (private_kind != "bridge" or private_source is None + or private_kind == public_kind and private_source == public_source): + base.update(state="degraded", readback="mismatch", + reason_code="private_source_invalid") + return base + if private_interface_has_public_filter(private): + base.update(state="degraded", readback="mismatch", + reason_code="private_filter_present") + return base + if not _MAC_RE.fullmatch(mac): + base.update(state="degraded", readback="mismatch", + reason_code="private_mac_missing") + return base + if base["private_ip_cidr"] is None: + base.update(state="degraded", readback="missing", + reason_code="private_ip_missing") + return base + # XML metadata is tenant/domain data and can be edited independently of + # the host. Only a future host adapter probe may set ``host_verified``; + # never use a managed-XML marker as proof of bridge/WireGuard/nft state. + if not host_verified: + base.update(state="degraded", readback="missing", + reason_code="private_source_unverified") + return base + if link_state not in {"up", "down"}: + base.update(state="degraded", readback="mismatch", + reason_code="private_link_state_invalid") + return base + base["state"] = "suspended" if link_state == "down" else "active" + return base + + +def observe_private_state_from_xml( + xml: str | bytes, + *, + vm_uuid: str | None = None, + host_verified: bool = False, + observed_at: int | None = None, +) -> dict[str, Any]: + """Parse a libvirt XML document and return :func:`observe_private_state`.""" + root = ET.fromstring(xml) + return observe_private_state( + root, vm_uuid=vm_uuid, host_verified=host_verified, + observed_at=observed_at, + ) diff --git a/engine/virsh_panel/vpc.py b/engine/virsh_panel/vpc.py index 074dd23d..7ba82d6b 100644 --- a/engine/virsh_panel/vpc.py +++ b/engine/virsh_panel/vpc.py @@ -851,6 +851,10 @@ class VpcCoordinator: "observed_digest": old_digest, "state": "applying" if gate.get("active") else "uncertain", "transport_verified": bool(gate.get("active")), + # Fabric transport is necessary but never sufficient for a + # tenant VPC. Only the future host adapter may set this + # after private0/route/nft/member readback. + "private_dataplane_verified": False, "readback": safe_readback, }) return { @@ -866,6 +870,7 @@ class VpcCoordinator: "operation_id": plan.operation_id, "state": state, "next_stage_allowed": False, + "private_dataplane_verified": False, "fabric_gate": dict(gate), "hosts": host_records, "subnets": [item.__dict__ for item in plan.subnets], @@ -891,7 +896,7 @@ class VpcCoordinator: "vpc_id", "region", "fabric_id", "fabric_revision", "cidr", "desired_revision", "desired_digest", "observed_revision", "observed_digest", "state", "next_stage_allowed", - "operation_id", "updated_at"): + "private_dataplane_verified", "operation_id", "updated_at"): if key in value: result[key] = value[key] gate = value.get("fabric_gate") @@ -927,7 +932,8 @@ class VpcCoordinator: item = {key: raw[key] for key in ( "host_id", "member_ids", "desired_revision", "observed_revision", "observed_digest", "state", - "transport_verified", "last_error_code") if key in raw} + "transport_verified", "private_dataplane_verified", + "last_error_code") if key in raw} readback = raw.get("readback") if isinstance(readback, Mapping): item["readback"] = { @@ -948,12 +954,14 @@ class VpcCoordinator: host_id = raw["host_id"] item: dict[str, Any] = {} for key in ("state", "observed_revision", "observed_digest", - "transport_verified", "last_error_code"): + "transport_verified", "private_dataplane_verified", + "last_error_code"): if key in raw: item[key] = raw[key] item["next_stage_allowed"] = ( raw.get("state") == "active" and raw.get("transport_verified") is True + and raw.get("private_dataplane_verified") is True and raw.get("observed_revision") == value.get("desired_revision") and raw.get("observed_digest") == value.get("desired_digest") ) @@ -963,6 +971,9 @@ class VpcCoordinator: "digest": value.get("observed_digest"), "state": value.get("state"), "next_stage_allowed": value.get("next_stage_allowed") is True, + "private_dataplane_verified": ( + value.get("private_dataplane_verified") is True + ), "hosts": observed_hosts, # PHP consumes this bounded observed projection when deciding # whether the control-plane result is deliberately waiting for @@ -1084,6 +1095,7 @@ class VpcCoordinator: if isinstance(item, dict): item["state"] = "applying" item["transport_verified"] = True + item["private_dataplane_verified"] = False hosts = record.get("hosts") desired_revision = record.get("desired_revision") desired_digest = record.get("desired_digest") @@ -1093,25 +1105,31 @@ class VpcCoordinator: and item.get("observed_revision") == desired_revision and item.get("observed_digest") == desired_digest and item.get("transport_verified") is True + and item.get("private_dataplane_verified") is True for item in hosts) fabric_active = gate.get("active") is True if fabric_active and all_observed: record["state"] = "active" record["next_stage_allowed"] = True + record["private_dataplane_verified"] = True record["observed_revision"] = desired_revision record["observed_digest"] = desired_digest elif not fabric_active: record["state"] = "degraded" if gate.get("available") else "uncertain" record["next_stage_allowed"] = False + record["private_dataplane_verified"] = False elif hosts and any(item.get("state") == "uncertain" for item in hosts if isinstance(item, Mapping)): record["state"] = "uncertain" record["next_stage_allowed"] = False + record["private_dataplane_verified"] = False elif hosts and any(item.get("state") == "degraded" for item in hosts if isinstance(item, Mapping)): record["state"] = "degraded" record["next_stage_allowed"] = False + record["private_dataplane_verified"] = False else: record["state"] = "applying" record["next_stage_allowed"] = False + record["private_dataplane_verified"] = False record["updated_at"] = int(time.time()) async def apply(self, value: Mapping[str, Any]) -> dict[str, Any]: @@ -1226,6 +1244,7 @@ class VpcCoordinator: if raw is None: host["state"] = "uncertain" host["transport_verified"] = False + host["private_dataplane_verified"] = False continue if not isinstance(raw, Mapping): raise VpcError("VPC observed 宿主记录无效") @@ -1233,6 +1252,7 @@ class VpcCoordinator: observed_digest = raw.get("observed_digest") state = raw.get("state") transport = raw.get("transport_verified") + private_dataplane = raw.get("private_dataplane_verified", False) if (isinstance(observed_revision, bool) or not isinstance(observed_revision, int) or observed_revision < 0): @@ -1246,10 +1266,22 @@ class VpcCoordinator: raise VpcError("VPC observed state 无效") if not isinstance(transport, bool): raise VpcError("VPC observed transport_verified 无效") + if not isinstance(private_dataplane, bool): + raise VpcError("VPC observed private_dataplane_verified 无效") if state == "active" and (transport is not True or observed_digest is None): - raise VpcError("VPC active 观测缺少传输或摘要证据", 422, - "vpc_observation_invalid") + raise VpcError( + "VPC active 观测缺少传输或摘要证据", 422, + "vpc_observation_invalid") + # The current Fabric adapter reports transport/readback before + # the tenant private data-plane adapter exists. Preserve that + # observation as an applying gate instead of rejecting the + # whole reconciliation request: transport is useful progress, + # but it must never open the VPC or be reported as active. + observation_error = raw.get("error_code") + if state == "active" and private_dataplane is not True: + state = "applying" + observation_error = "vpc_private_dataplane_unverified" readback = raw.get("readback", {}) if not isinstance(readback, Mapping) or len(readback) > 64: raise VpcError("VPC readback 无效") @@ -1262,10 +1294,11 @@ class VpcCoordinator: "observed_revision": observed_revision, "observed_digest": observed_digest, "transport_verified": transport, + "private_dataplane_verified": private_dataplane, "readback": safe_readback, - "last_error_code": raw.get("error_code") - if isinstance(raw.get("error_code"), str) - and len(raw.get("error_code")) <= 64 else None}) + "last_error_code": observation_error + if isinstance(observation_error, str) + and len(observation_error) <= 64 else None}) gate = self._fabric_gate(plan) self._finish(record, gate) self._save("vpc_desired", vpc_id, record) @@ -1303,6 +1336,7 @@ class VpcCoordinator: continue value["state"] = "uncertain" value["next_stage_allowed"] = False + value["private_dataplane_verified"] = False # A previous Fabric gate may have been active before the process # stopped. Preserve its revision/evidence for audit, but never # expose that stale gate as usable after restart. @@ -1318,7 +1352,8 @@ class VpcCoordinator: for host in value.get("hosts", []): if isinstance(host, dict): host.update({"state": "uncertain", - "transport_verified": False}) + "transport_verified": False, + "private_dataplane_verified": False}) value["updated_at"] = int(time.time()) self._save("vpc_desired", key, value) self._save_children(value) diff --git a/web/app/Services/Cloud/CloudFirewallService.php b/web/app/Services/Cloud/CloudFirewallService.php index ed8e0d8e..39e95b55 100644 --- a/web/app/Services/Cloud/CloudFirewallService.php +++ b/web/app/Services/Cloud/CloudFirewallService.php @@ -227,6 +227,9 @@ final class CloudFirewallService ], 'owner_id' => (int) $locked->user_id, 'can_manage' => true, + 'private_expectation' => CloudVpcService::instancePrivateExpectation( + (int) $locked->id + ), 'firewall_version_id' => (int) $version->id, 'firewall_revision' => $revision, 'firewall_policy_digest' => $policyDigest, diff --git a/web/app/Services/Cloud/CloudInstancePageService.php b/web/app/Services/Cloud/CloudInstancePageService.php index a380b9f1..c125461f 100644 --- a/web/app/Services/Cloud/CloudInstancePageService.php +++ b/web/app/Services/Cloud/CloudInstancePageService.php @@ -113,6 +113,8 @@ final class CloudInstancePageService $runtimeHint = in_array($runtimeSync, ['offline', 'unavailable'], true) ? '暂时无法同步' : ($runtimeState === 'unknown' ? '等待宿主机同步' : ''); } + $privateState = CloudVpcService::instancePrivateProjection((int) $instance->id); + $privateState['vm_uuid'] = (string) $instance->uuid; return [ 'id' => (string) $instance->id, 'api_id' => (string) $instance->id, @@ -130,6 +132,7 @@ final class CloudInstancePageService 'instance_status' => (string) $instance->status, 'service_status' => (string) $instance->service_status, 'network_status' => (string) $instance->network_status, + 'private_state' => $privateState, 'service_expires_at' => $instance->expires_at === null ? null : (int) $instance->expires_at, 'current_period_started_at' => $period === null ? null : (int) $period->starts_at, @@ -191,11 +194,14 @@ final class CloudInstancePageService 'state_sync_status' => 'pending', 'state_label_hint' => '等待宿主机同步', ]; + $privateState = CloudVpcService::instancePrivateProjection((int) $instance->id); + $privateState['vm_uuid'] = (string) $instance->uuid; $vms[] = [ 'id' => (string) $instance->id, 'api_id' => (string) $instance->id, 'uuid' => (string) $instance->uuid, 'name' => (string) $instance->name, + 'private_state' => $privateState, ] + array_intersect_key($cached, array_flip([ 'state', 'state_transitioning', 'state_transition', 'state_fresh', 'state_observed_at', @@ -248,6 +254,10 @@ final class CloudInstancePageService 'instance_status' => (string) $snapshot['instance_status'], 'service_status' => (string) $snapshot['service_status'], 'network_status' => (string) $snapshot['network_status'], + 'private_state' => is_array($snapshot['private_state'] ?? null) + ? $snapshot['private_state'] : CloudVpcService::instancePrivateProjection( + (int) $snapshot['id'] + ), 'service_expires_at' => $snapshot['service_expires_at'], 'current_period_started_at' => $snapshot['current_period_started_at'], 'current_period_ends_at' => $snapshot['current_period_ends_at'], @@ -293,6 +303,10 @@ final class CloudInstancePageService 'state_sync_status' => $liveAvailable ? 'fresh' : (string) ($snapshot['state_sync_status'] ?? 'pending'), 'state_label_hint' => (string) ($snapshot['state_label_hint'] ?? ''), + 'private_state' => is_array($snapshot['private_state'] ?? null) + ? $snapshot['private_state'] : CloudVpcService::instancePrivateProjection( + (int) $snapshot['id'] + ), ]; } @@ -304,6 +318,7 @@ final class CloudInstancePageService 'state_transitioning', 'state_transition', 'state_fresh', 'state_observed_at', 'state_sync_status', 'state_label_hint', 'instance_status', + 'private_state', 'os_source', 'os_name', 'install_iso', 'public_ip_cidr', 'public_ip', 'login_name', 'engine_metadata_version', 'vcpu', 'cpu_cores', 'memory_mib', 'ram_mib', 'disks', 'disk_gib', @@ -339,6 +354,8 @@ final class CloudInstancePageService 'state_sync_status' => 'pending', 'state_label_hint' => '等待宿主机同步', ]; + $privateState = CloudVpcService::instancePrivateProjection((int) $instance->id); + $privateState['vm_uuid'] = (string) $instance->uuid; $result[] = self::listItem(array_merge($spec, [ 'id' => (string) $instance->id, 'api_id' => (string) $instance->id, @@ -346,6 +363,7 @@ final class CloudInstancePageService 'name' => (string) $instance->name, 'instance_status' => (string) $instance->status, 'region' => $region, + 'private_state' => $privateState, ], $cached)); } return $result; @@ -504,6 +522,8 @@ final class CloudInstancePageService $runtimeHint = in_array($runtimeSync, ['offline', 'unavailable'], true) ? '暂时无法同步' : ($runtimeState === 'unknown' ? '等待宿主机同步' : ''); } + $privateState = CloudVpcService::instancePrivateProjection((int) $instance->id); + $privateState['vm_uuid'] = (string) $instance->uuid; $maximumFuturePeriods = max(1, min(120, (int) Config::get('cloud_max_renewal_periods_ahead', 24))); @@ -532,6 +552,7 @@ final class CloudInstancePageService 'instance_status' => (string) $instance->status, 'service_status' => (string) $instance->service_status, 'network_status' => (string) $instance->network_status, + 'private_state' => $privateState, 'service_expires_at' => $instance->expires_at === null ? null : (int) $instance->expires_at, 'current_period_started_at' => $period === null ? null : (int) $period->starts_at, diff --git a/web/app/Services/Cloud/CloudJobService.php b/web/app/Services/Cloud/CloudJobService.php index dc67abb5..6115f51e 100644 --- a/web/app/Services/Cloud/CloudJobService.php +++ b/web/app/Services/Cloud/CloudJobService.php @@ -244,7 +244,14 @@ final class CloudJobService $jobPayload = ['method' => $method, 'resource' => 'vms/' . rawurlencode($locked->uuid) . $suffix, 'body' => $body, 'credential' => $credential, 'owner_id' => (int) $locked->user_id, - 'can_manage' => (bool) ($actor['can_manage'] ?? false)]; + 'can_manage' => (bool) ($actor['can_manage'] ?? false), + // Bind the private target to the same immutable job payload + // as the public-IP ledger. A later VPC revision must make + // this operation uncertain rather than letting its response + // satisfy a newer target by accident. + 'private_expectation' => CloudVpcService::instancePrivateExpectation( + (int) $locked->id + )]; if ($networkLedger !== null) { $source = $networkLedger['source']; $target = $networkLedger['target']; @@ -322,6 +329,9 @@ final class CloudJobService 'body' => [], 'owner_id' => (int) $locked->user_id, 'system_cleanup' => true, + 'private_expectation' => CloudVpcService::instancePrivateExpectation( + (int) $locked->id + ), ]); $job->available_at = $job->created_at = $job->updated_at = time(); $job->save(); diff --git a/web/app/Services/Cloud/CloudLifecycleService.php b/web/app/Services/Cloud/CloudLifecycleService.php index b4d6a94f..a8ab21da 100644 --- a/web/app/Services/Cloud/CloudLifecycleService.php +++ b/web/app/Services/Cloud/CloudLifecycleService.php @@ -446,6 +446,9 @@ final class CloudLifecycleService 'resource' => 'vms/' . rawurlencode($instance->uuid) . '/' . $suffix, 'body' => $body, 'owner_id' => (int) $instance->user_id, 'system_policy' => true, 'suspend_reason' => $reason, + 'private_expectation' => CloudVpcService::instancePrivateExpectation( + (int) $instance->id + ), ]); $job->available_at = $job->created_at = $job->updated_at = time(); $job->save(); diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php index 5db73028..7c692b13 100644 --- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php +++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php @@ -121,6 +121,12 @@ final class CloudOperationErrorCatalog 'firewall_drift_detected' => '检测到防火墙策略漂移,请联系管理员核对', 'firewall_cleanup_pending' => '实例删除结果待核对,防火墙清理状态尚未确认', 'firewall_apply_superseded' => '防火墙请求已被更新的策略替代', + 'vpc_private_observation_missing' => '实例私网状态尚未完成回读,请稍后刷新,切勿重复提交', + 'vpc_private_observation_mismatch' => '实例私网状态与当前 VPC 目标不一致,正在核对', + 'vpc_private_dataplane_unverified' => 'VPC 私网数据面尚未完成核验,请稍后刷新', + 'vpc_private_teardown_uncertain' => '实例私网清理状态尚未确认,请勿重复提交', + 'vpc_private_expectation_invalid' => '实例私网目标记录无效,请联系管理员核对', + 'vpc_private_expectation_unavailable' => '实例私网目标暂时无法读取,请稍后核对', 'admin_cancelled' => '管理员已中止后续重试;已派发的宿主机操作仍需核查', ]; @@ -154,6 +160,7 @@ final class CloudOperationErrorCatalog 'created', 'restored', 'deleted', 'service_was_running', 'service_was_running_hint', 'vm', 'reinstall_committed', 'cleanup_pending', + 'private_state', 'vcpu', 'cpu_cores', 'memory_mib', 'ram_mib', 'disk_gib', 'os_source', 'os_name', 'install_iso', 'login_name', 'autostart', 'managed', @@ -317,6 +324,13 @@ final class CloudOperationErrorCatalog } return 'firewall_apply_failed'; } + if (in_array($engineCode, [ + 'vpc_private_observation_missing', 'vpc_private_observation_mismatch', + 'vpc_private_teardown_uncertain', 'vpc_private_expectation_invalid', + 'vpc_private_expectation_unavailable', 'vpc_private_dataplane_unverified', + ], true)) { + return $engineCode; + } // A worker may persist only a reviewed detail or a stable generic // message. Never put an arbitrary engine/admin summary into the @@ -542,6 +556,15 @@ final class CloudOperationErrorCatalog } continue; } + if ($name === 'private_state') { + if (is_array($item) && !array_is_list($item)) { + $safePrivate = self::sanitizePrivateState($item); + if ($safePrivate !== null) { + $result[$name] = $safePrivate; + } + } + continue; + } $safe = self::safeResultScalar($name, $item); if ($safe !== null) { $result[$name] = $safe; @@ -550,6 +573,81 @@ final class CloudOperationErrorCatalog return $result; } + /** Keep private-state evidence typed and free of host/XML diagnostics. */ + private static function sanitizePrivateState(array $value): ?array + { + $result = []; + $result['version'] = (int) ($value['version'] ?? 1); + if ($result['version'] !== 1) { + return null; + } + $result['required'] = ($value['required'] ?? false) === true; + foreach (['vm_uuid', 'vpc_id', 'member_id', 'interface', + 'private_ip_cidr', 'private_mac', 'desired_digest', 'observed_digest'] as $key) { + $item = $value[$key] ?? null; + $result[$key] = is_string($item) && $item !== '' + ? self::normalize($item) : null; + } + foreach (['state' => ['disabled', 'active', 'suspended', 'removed', 'degraded', 'uncertain'], + 'desired_state' => ['disabled', 'active', 'removed'], + 'readback' => ['confirmed', 'missing', 'mismatch'], + 'link_state' => ['up', 'down', 'absent']] as $key => $allowed) { + $default = match ($key) { + 'desired_state' => $result['required'] ? 'active' : 'disabled', + 'readback' => 'missing', + 'link_state' => 'absent', + default => 'uncertain', + }; + $item = is_string($value[$key] ?? null) + ? strtolower(trim((string) $value[$key])) : $default; + $result[$key] = in_array($item, $allowed, true) ? $item : $default; + } + $result['desired_revision'] = max(0, (int) ($value['desired_revision'] ?? 0)); + $result['observed_revision'] = max(0, (int) ($value['observed_revision'] ?? 0)); + $result['observed_at'] = max(0, (int) ($value['observed_at'] ?? 0)); + foreach (['desired_digest', 'observed_digest'] as $key) { + if ($result[$key] !== null + && preg_match('/^[a-f0-9]{64}$/Di', $result[$key]) !== 1) { + $result[$key] = null; + } + } + if ($result['vm_uuid'] !== null && preg_match( + '/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', + $result['vm_uuid']) !== 1) { + $result['vm_uuid'] = null; + } + foreach (['vpc_id', 'member_id'] as $key) { + if ($result[$key] !== null && preg_match('/^[0-9a-f]{32}$/Di', $result[$key]) !== 1) { + $result[$key] = null; + } + } + if ($result['interface'] !== null && $result['interface'] !== 'private0') { + $result['interface'] = null; + } + if ($result['private_ip_cidr'] !== null) { + $parts = explode('/', $result['private_ip_cidr'], 2); + if (count($parts) !== 2 || (int) $parts[1] !== 32 + || filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false) { + $result['private_ip_cidr'] = null; + } + } + if ($result['private_mac'] !== null + && preg_match('/^[0-9a-f]{2}(?::[0-9a-f]{2}){5}$/Di', $result['private_mac']) !== 1) { + $result['private_mac'] = null; + } + $reason = is_string($value['reason_code'] ?? null) + ? strtolower(trim((string) $value['reason_code'])) : null; + $result['reason_code'] = in_array($reason, [ + 'private_interface_role_invalid', 'private_source_invalid', + 'private_filter_present', 'private_mac_missing', 'private_ip_missing', + 'private_source_unverified', 'vpc_private_observation_missing', + 'vpc_private_observation_mismatch', 'vpc_private_teardown_uncertain', + 'vpc_private_dataplane_unverified', + 'vpc_private_expectation_invalid', 'vpc_private_expectation_unavailable', + ], true) ? $reason : null; + return $result; + } + private static function safeResultScalar(string $key, $value) { if (is_bool($value)) { diff --git a/web/app/Services/Cloud/CloudOrderService.php b/web/app/Services/Cloud/CloudOrderService.php index 009cfe24..47342915 100644 --- a/web/app/Services/Cloud/CloudOrderService.php +++ b/web/app/Services/Cloud/CloudOrderService.php @@ -240,7 +240,14 @@ final class CloudOrderService $job->host_id = $placement['host']->id; $job->kind = 'create'; $job->status = 'queued'; - $job->payload_cipher = CloudSecrets::encrypt($placement['payload']); + // A create has no instance row yet, so this is normally the explicit + // disabled target. Persist it so a future flow that pre-creates or + // attaches a VPC member can use the same immutable-target protocol + // instead of publishing a public-only success without a target. + $createPayload = $placement['payload']; + $createPayload['private_expectation'] = + CloudVpcService::instancePrivateExpectation(null); + $job->payload_cipher = CloudSecrets::encrypt($createPayload); $job->available_at = $job->created_at = $job->updated_at = time(); $job->save(); // The worker now owns the sole recoverable credential copy. diff --git a/web/app/Services/Cloud/CloudSnapshotService.php b/web/app/Services/Cloud/CloudSnapshotService.php index 5a95f4b8..55b6cb72 100644 --- a/web/app/Services/Cloud/CloudSnapshotService.php +++ b/web/app/Services/Cloud/CloudSnapshotService.php @@ -399,6 +399,13 @@ final class CloudSnapshotService 'can_manage' => (bool) ($actor['can_manage'] ?? false), 'snapshot_key' => (string) $snapshot->snapshot_key, 'snapshot_previous_state' => (string) $snapshot->state, + // Snapshot work changes storage, not the tenant network, but it + // still crosses the same instance commit boundary. Capture the + // private target so a VPC revision change while the snapshot is + // running cannot be mistaken for a readback of the newer target. + 'private_expectation' => CloudVpcService::instancePrivateExpectation( + (int) $instance->id + ), ]); $job->available_at = $job->created_at = $job->updated_at = $now; $job->save(); diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php index d368ee7a..eae56b2a 100644 --- a/web/app/Services/Cloud/CloudVpcService.php +++ b/web/app/Services/Cloud/CloudVpcService.php @@ -563,6 +563,14 @@ final class CloudVpcService ->exists()) { throw new CloudError('该成员仍被路由引用,请先替换或删除路由', 409); } + // Lifecycle commits lock VPC -> instance before validating the + // captured private target. Take the same order here so a detach + // cannot flip the member to deleting while a VM job is committing. + $instance = CloudInstance::where('id', (int) $member->instance_id) + ->lockForUpdate()->first(); + if ($instance === null || (string) $instance->status === 'deleted') { + throw new CloudError('虚拟机不存在', 404); + } $privateIp = self::lockedPrivateIpForMember($member); $member->state = 'deleting'; $member->updated_at = time(); @@ -940,8 +948,11 @@ final class CloudVpcService ?? $observed['observed_revision'] ?? 0); $observedDigest = (string) ($observed['digest'] ?? $observed['observed_digest'] ?? ''); + $privateDataplaneVerified = ($observed['private_dataplane_verified'] + ?? false) === true; $globalActive = ($observed['state'] ?? null) === 'active' && ($observed['next_stage_allowed'] ?? false) === true + && $privateDataplaneVerified && $observedRevision === $revision && hash_equals($digest, $observedDigest); $fabricGate = is_array($observed['fabric_gate'] ?? null) @@ -1029,6 +1040,7 @@ final class CloudVpcService $itemDigest = (string) ($item['observed_digest'] ?? $item['digest'] ?? ''); $active = $globalActive && ($item['state'] ?? null) === 'active' && ($item['transport_verified'] ?? false) === true + && ($item['private_dataplane_verified'] ?? false) === true && $itemRevision <= $revision && $itemRevision === $revision && hash_equals($digest, $itemDigest); $deleting = (string) $deployment->state === 'deleting'; @@ -1058,6 +1070,16 @@ final class CloudVpcService // remain untouched until the later data-plane readback phase. if ($allActive) { $observedAt = time(); + // Persist a proof marker on each member only after the full + // host-scoped private data-plane gate has passed. Older rows + // without this marker remain fail-closed in instance + // projections until a fresh observation is recorded. + CloudVpcMember::where('vpc_id', $vpc->id) + ->whereNotIn('state', ['deleted']) + ->update([ + 'last_operation_id' => $operationId, + 'updated_at' => $observedAt, + ]); foreach ([ CloudVpcSubnet::class, CloudVpcPrivateIp::class, @@ -1263,6 +1285,595 @@ final class CloudVpcService } } + /** + * Return the private-network expectation that belongs to one VM job. + * + * This deliberately reads the existing VPC ledger instead of copying a + * second target into cloud_instances/spec_json. A missing VPC schema is + * treated as the legacy "no private network" deployment; once the ledger + * exists, malformed rows are an explicit uncertain expectation and cannot + * be silently interpreted as an attached network. + */ + public static function instancePrivateExpectation(?int $instanceId): array + { + $disabled = [ + 'required' => false, 'state' => 'disabled', 'desired_state' => 'disabled', + 'vpc_id' => null, 'member_id' => null, 'private_ip_cidr' => null, + 'private_mac' => null, 'desired_revision' => 0, 'desired_digest' => null, + 'reason_code' => null, + ]; + if ($instanceId === null || $instanceId < 1) { + return $disabled; + } + try { + $memberConnection = (new CloudVpcMember())->getConnection(); + $schema = $memberConnection->getSchemaBuilder(); + if (!$schema->hasTable('cloud_vpc_members')) { + return $disabled; + } + if (!$schema->hasTable('cloud_vpc_private_ips') + || !$schema->hasTable('cloud_vpcs')) { + return [ + 'required' => true, 'state' => 'uncertain', + 'desired_state' => 'active', 'vpc_id' => null, + 'member_id' => null, 'private_ip_cidr' => null, + 'private_mac' => null, 'desired_revision' => 0, + 'desired_digest' => null, + 'reason_code' => 'vpc_private_expectation_invalid', + ]; + } + $member = CloudVpcMember::where('instance_id', $instanceId) + ->whereNotIn('state', ['deleted'])->orderByDesc('id')->first(); + if ($member === null) { + // A private-IP lease without a live member is still a + // control-plane residue. Treat it as an uncertain target so + // lifecycle workers cannot publish a successful/public-only + // result while the orphan may remain attached on a host. + $orphanLease = CloudVpcPrivateIp::where('instance_id', $instanceId) + ->whereNotIn('state', ['deleted'])->exists(); + if ($orphanLease) { + return [ + 'required' => true, 'state' => 'uncertain', + 'desired_state' => 'active', 'vpc_id' => null, + 'member_id' => null, 'private_ip_cidr' => null, + 'private_mac' => null, 'desired_revision' => 0, + 'desired_digest' => null, + 'reason_code' => 'vpc_private_expectation_invalid', + ]; + } + return $disabled; + } + $vpc = CloudVpc::find((int) $member->vpc_id); + $privateIp = $member->private_ip_id === null ? null + : CloudVpcPrivateIp::where('id', (int) $member->private_ip_id)->first(); + $state = (string) $member->state; + $desiredState = $state === 'deleting' ? 'removed' : 'active'; + // The member and its lease are one desired-state tuple. Falling + // back to the root revision/digest can hide a partially-written + // child row and would let a VM job capture a target that the VPC + // document never actually requested. + $revision = (int) ($member->desired_revision ?? 0); + $digest = self::safeDigest($member->desired_digest); + $rootRevision = (int) ($vpc?->desired_revision ?? 0); + $rootDigest = self::safeDigest($vpc?->desired_digest); + $leaseRevision = (int) ($privateIp?->desired_revision ?? 0); + $leaseDigest = self::safeDigest($privateIp?->desired_digest); + $validIp = is_string($member->private_ip) + && filter_var($member->private_ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false; + // `deleting` is an in-flight lease state, not proof that the + // dataplane has converged. Keep the expectation uncertain until + // the row is removed (and a fresh member is observed). + $leaseState = (string) ($privateIp?->state ?? ''); + $stateMatches = $privateIp !== null && $leaseState === $state; + $ledgerMatches = $privateIp !== null + && (int) $privateIp->vpc_id === (int) $member->vpc_id + && (int) $privateIp->subnet_id === (int) $member->subnet_id + && (int) ($privateIp->member_id ?? 0) === (int) $member->id + && (int) ($privateIp->instance_id ?? 0) === $instanceId + && (string) $privateIp->private_ip === (string) $member->private_ip + && $stateMatches + && !in_array($leaseState, ['deleted', 'deleting'], true); + $validRevision = $revision > 0; + $validDigest = $digest !== null; + $desiredLedgerMatches = $vpc !== null + && $rootRevision === $revision + && $rootDigest !== null && $digest !== null + && self::digestMatches($rootDigest, $digest) + && $leaseRevision === $revision + && $leaseDigest !== null + && self::digestMatches($digest, $leaseDigest); + if (!$validIp || !$ledgerMatches || !$validRevision || !$validDigest + || !$desiredLedgerMatches + || !is_string($member->member_key) + || preg_match(self::KEY_PATTERN, (string) $member->member_key) !== 1 + || !is_string($vpc?->vpc_key) + || preg_match(self::KEY_PATTERN, (string) $vpc->vpc_key) !== 1) { + return [ + 'required' => true, 'state' => 'uncertain', + 'desired_state' => $desiredState, + 'vpc_id' => is_string($vpc?->vpc_key) ? $vpc->vpc_key : null, + 'member_id' => is_string($member->member_key) ? $member->member_key : null, + 'private_ip_cidr' => $validIp ? $member->private_ip . '/32' : null, + 'private_mac' => null, 'desired_revision' => max(0, $revision), + 'desired_digest' => $validDigest ? $digest : null, + 'reason_code' => 'vpc_private_expectation_invalid', + ]; + } + return [ + 'required' => true, 'state' => $state, + 'desired_state' => $desiredState, + 'vpc_id' => (string) $vpc->vpc_key, + 'member_id' => (string) $member->member_key, + 'private_ip_cidr' => $member->private_ip . '/32', + 'private_mac' => null, + 'desired_revision' => $revision, + 'desired_digest' => $digest, + 'reason_code' => null, + ]; + } catch (\Throwable $error) { + // A live ledger that cannot be read is different from an + // installation that predates VPC tables. Fail closed for the + // former so a worker cannot publish a stale successful result. + return [ + 'required' => true, 'state' => 'uncertain', + 'desired_state' => 'active', 'vpc_id' => null, 'member_id' => null, + 'private_ip_cidr' => null, 'private_mac' => null, + 'desired_revision' => 0, 'desired_digest' => null, + 'reason_code' => 'vpc_private_expectation_unavailable', + ]; + } + } + + /** Compare a job's captured target with the current ledger under a lock. */ + public static function privateExpectationMatches(int $instanceId, array $captured): bool + { + if ($instanceId < 1 || $captured === []) { + return true; + } + $current = self::instancePrivateExpectation($instanceId); + // `state` is observed/transient (for example applying -> active) and + // may advance while a VM lifecycle job is running. The immutable + // target is bound by desired_state, identity, IP, revision and digest; + // removal remains strict because it changes desired_state to removed. + foreach (['required', 'desired_state', 'vpc_id', 'member_id', + 'private_ip_cidr', 'desired_revision', 'desired_digest'] as $key) { + if ((string) ($captured[$key] ?? null) !== (string) ($current[$key] ?? null)) { + return false; + } + } + return true; + } + + /** Build a bounded instance projection for page/runtime responses. */ + public static function instancePrivateProjection(int $instanceId): array + { + $expectation = self::instancePrivateExpectation($instanceId); + $base = [ + 'version' => 1, 'required' => (bool) $expectation['required'], + 'vm_uuid' => null, 'vpc_id' => $expectation['vpc_id'] ?? null, + 'member_id' => $expectation['member_id'] ?? null, + 'interface' => $expectation['required'] ? 'private0' : null, + 'state' => $expectation['required'] ? 'uncertain' : 'disabled', + 'desired_state' => $expectation['desired_state'] ?? 'disabled', + 'desired_revision' => (int) ($expectation['desired_revision'] ?? 0), + 'observed_revision' => 0, + 'desired_digest' => $expectation['desired_digest'] ?? null, + 'observed_digest' => null, 'readback' => $expectation['required'] ? 'missing' : 'confirmed', + 'link_state' => $expectation['required'] ? 'absent' : 'absent', + 'private_ip_cidr' => $expectation['private_ip_cidr'] ?? null, + 'private_mac' => null, + 'reason_code' => $expectation['reason_code'] + ?? ($expectation['required'] ? 'vpc_private_observation_missing' : null), + 'observed_at' => 0, + ]; + if (!$expectation['required']) { + return $base; + } + try { + $vpcKey = is_string($expectation['vpc_id'] ?? null) + ? (string) $expectation['vpc_id'] : ''; + $memberKey = is_string($expectation['member_id'] ?? null) + ? (string) $expectation['member_id'] : ''; + if (preg_match(self::KEY_PATTERN, $vpcKey) !== 1 + || preg_match(self::KEY_PATTERN, $memberKey) !== 1) { + return $base; + } + // member_key is only unique inside a VPC. Binding the instance + // and VPC roots here prevents a stale/cross-tenant row from + // becoming a usable private projection. + $vpc = CloudVpc::where('vpc_key', $vpcKey)->first([ + 'id', 'vpc_key', 'state', 'sync_state', 'desired_revision', + 'observed_revision', 'desired_digest', 'observed_digest', + 'last_operation_id', 'last_success_at', 'updated_at', + ]); + if ($vpc === null) { + return $base; + } + $member = CloudVpcMember::where('instance_id', $instanceId) + ->where('vpc_id', (int) $vpc->id) + ->where('member_key', $memberKey) + ->whereNotIn('state', ['deleted']) + ->first([ + 'id', 'vpc_id', 'subnet_id', 'private_ip_id', 'instance_id', + 'private_ip', 'state', 'desired_revision', 'observed_revision', + 'desired_digest', 'observed_digest', 'last_error_code', + 'last_operation_id', 'updated_at', + ]); + if ($member === null || $member->private_ip_id === null) { + return $base; + } + // A member row is not proof that its independent lease exists. + // Read back and bind every lease identity before considering the + // member's revision or digest. + $privateIp = CloudVpcPrivateIp::where('id', (int) $member->private_ip_id) + ->where('vpc_id', (int) $vpc->id) + ->where('member_id', (int) $member->id) + ->where('instance_id', $instanceId) + ->first([ + 'id', 'vpc_id', 'subnet_id', 'member_id', 'instance_id', + 'private_ip', 'state', 'desired_revision', 'observed_revision', + 'desired_digest', 'observed_digest', 'last_error_code', 'updated_at', + ]); + if ($privateIp === null + || (int) $privateIp->subnet_id !== (int) $member->subnet_id + || (string) $privateIp->private_ip !== (string) $member->private_ip) { + return $base; + } + + $observedRevision = max(0, (int) $member->observed_revision); + $observedDigest = self::safeDigest($member->observed_digest); + $privateObservedRevision = max(0, (int) $privateIp->observed_revision); + $privateObservedDigest = self::safeDigest($privateIp->observed_digest); + $privateDesiredRevision = max(0, (int) $privateIp->desired_revision); + $privateDesiredDigest = self::safeDigest($privateIp->desired_digest); + $rootDesiredRevision = max(0, (int) $vpc->desired_revision); + $rootObservedRevision = max(0, (int) $vpc->observed_revision); + $rootDesiredDigest = self::safeDigest($vpc->desired_digest); + $rootObservedDigest = self::safeDigest($vpc->observed_digest); + $rootOperationId = (string) ($vpc->last_operation_id ?? ''); + // A VM job operation ID is not a VPC activation proof. The + // marker must be the current root operation that advanced the + // all-host VPC gate, and that operation must have succeeded. + $proofMarker = preg_match(self::KEY_PATTERN, $rootOperationId) === 1 + && hash_equals($rootOperationId, + (string) ($member->last_operation_id ?? '')); + $operation = $proofMarker + ? CloudVpcOperation::where('vpc_id', (int) $vpc->id) + ->where('operation_id', $rootOperationId) + ->where('status', 'succeeded') + ->first(['operation_id', 'desired_revision', 'created_at', 'finished_at']) + : null; + $operationFresh = $operation !== null + && (int) $operation->desired_revision === (int) $base['desired_revision'] + && (int) ($operation->finished_at ?? 0) > 0 + && (int) ($operation->finished_at ?? 0) + >= (int) ($operation->created_at ?? 0) + && (int) ($vpc->last_success_at ?? 0) > 0 + && (int) ($vpc->last_success_at ?? 0) + >= (int) ($operation->created_at ?? 0) + && (int) ($member->updated_at ?? 0) > 0 + && (int) ($privateIp->updated_at ?? 0) > 0 + && (int) ($member->updated_at ?? 0) >= (int) ($operation->created_at ?? 0) + && (int) ($privateIp->updated_at ?? 0) >= (int) ($operation->created_at ?? 0); + $rootExact = (string) $vpc->state === 'active' + && (string) $vpc->sync_state === 'applied' + && $rootDesiredRevision === (int) $base['desired_revision'] + && $rootObservedRevision === $rootDesiredRevision + && $rootDesiredDigest !== null && $rootObservedDigest !== null + && self::digestMatches($rootDesiredDigest, $rootObservedDigest) + && self::digestMatches((string) $base['desired_digest'], $rootDesiredDigest); + $memberExact = (string) $member->state === 'active' + && $observedRevision === (int) $base['desired_revision'] + && $observedDigest !== null + && self::digestMatches((string) $base['desired_digest'], $observedDigest) + && (int) $member->desired_revision === (int) $base['desired_revision'] + && self::digestMatches((string) $base['desired_digest'], + (string) $member->desired_digest); + $privateExact = (string) $privateIp->state === 'active' + && $privateDesiredRevision === (int) $base['desired_revision'] + && $privateObservedRevision === $privateDesiredRevision + && $privateDesiredDigest !== null + && $privateObservedDigest !== null + && self::digestMatches((string) $base['desired_digest'], $privateDesiredDigest) + && self::digestMatches($privateDesiredDigest, $privateObservedDigest); + $confirmed = $proofMarker && $operationFresh && $rootExact + && $memberExact && $privateExact; + $base['observed_revision'] = $observedRevision; + $base['observed_digest'] = $observedDigest; + $base['state'] = $confirmed ? 'active' + : ((string) $member->state === 'degraded' + || (string) $privateIp->state === 'degraded' + ? 'degraded' : 'uncertain'); + $base['readback'] = $confirmed ? 'confirmed' : 'missing'; + $base['reason_code'] = $confirmed ? null + : (self::safeErrorCode((string) ($privateIp->last_error_code + ?? $member->last_error_code ?? '')) + ?? 'vpc_private_observation_missing'); + $freshness = array_values(array_filter([ + (int) ($member->updated_at ?? 0), + (int) ($privateIp->updated_at ?? 0), + (int) ($vpc->last_success_at ?? 0), + ], static fn (int $value): bool => $value > 0)); + $base['observed_at'] = $freshness === [] ? 0 : min($freshness); + if ($confirmed) { + // The member ledger proves the VPC revision, while the + // instance network policy records whether the last gated + // operation intentionally disconnected all NICs. Pending or + // failed policy transitions must remain uncertain until the + // next engine readback, even when the previous member digest + // was exact. + $networkStatus = (string) (CloudInstance::where('id', $instanceId) + ->value('network_status') ?? ''); + if (in_array($networkStatus, + ['blocking', 'blocking_expired', 'restore_pending', 'restoring'], true)) { + $base['state'] = 'uncertain'; + $base['readback'] = 'missing'; + $base['reason_code'] = 'vpc_private_observation_missing'; + } elseif (in_array($networkStatus, + ['blocked_traffic', 'blocked_expired'], true)) { + $base['state'] = 'suspended'; + $base['link_state'] = 'down'; + } else { + $base['link_state'] = 'up'; + } + } + } catch (\Throwable) { + // Keep the fail-closed default projection. + } + return $base; + } + + /** + * Validate one engine private_state at a mutation commit boundary. + * + * The engine's XML observation is necessary but not sufficient for an + * attached VPC: the member identity, desired revision and digest must + * also be echoed by a future data-plane reconciler. Until then this + * method intentionally returns an uncertain gate for VPC members. + */ + public static function validatePrivateObservation( + array $expectation, ?array $observation, string $kind + ): array { + $fallback = [ + 'version' => 1, 'required' => (bool) ($expectation['required'] ?? false), + 'vm_uuid' => null, 'vpc_id' => $expectation['vpc_id'] ?? null, + 'member_id' => $expectation['member_id'] ?? null, + 'interface' => null, 'state' => 'uncertain', + 'desired_state' => (string) ($expectation['desired_state'] ?? 'disabled'), + 'desired_revision' => (int) ($expectation['desired_revision'] ?? 0), + 'observed_revision' => 0, + 'desired_digest' => $expectation['desired_digest'] ?? null, + 'observed_digest' => null, 'readback' => 'missing', + 'link_state' => 'absent', 'private_ip_cidr' => null, + 'private_mac' => null, + 'reason_code' => 'vpc_private_observation_missing', + 'observed_at' => time(), + ]; + if (!is_array($observation)) { + return ['ok' => false, 'state' => $fallback, + 'error_code' => $fallback['reason_code']]; + } + if ((int) ($observation['version'] ?? 0) !== 1) { + $fallback['reason_code'] = 'vpc_private_observation_mismatch'; + return ['ok' => false, 'state' => $fallback, + 'error_code' => $fallback['reason_code']]; + } + $state = $fallback; + foreach (['version', 'required', 'vm_uuid', 'vpc_id', 'member_id', + 'interface', 'state', 'desired_revision', 'observed_revision', + 'desired_digest', 'observed_digest', 'readback', 'link_state', + 'private_ip_cidr', 'private_mac', 'reason_code', 'observed_at'] as $key) { + if (array_key_exists($key, $observation)) { + $state[$key] = $observation[$key]; + } + } + $state['required'] = (bool) $state['required']; + $state['desired_revision'] = max(0, (int) $state['desired_revision']); + $state['observed_revision'] = max(0, (int) $state['observed_revision']); + $state['desired_digest'] = self::safeDigest($state['desired_digest']); + $state['observed_digest'] = self::safeDigest($state['observed_digest']); + $expectedRequired = (bool) ($expectation['required'] ?? false); + $expectedState = (string) ($expectation['desired_state'] ?? 'disabled'); + $state['required'] = $expectedRequired; + $state['desired_state'] = $expectedState; + $observedVpcId = is_string($observation['vpc_id'] ?? null) + ? (string) $observation['vpc_id'] : null; + $observedMemberId = is_string($observation['member_id'] ?? null) + ? (string) $observation['member_id'] : null; + $state['vpc_id'] = $expectation['vpc_id'] ?? null; + $state['member_id'] = $expectation['member_id'] ?? null; + $state['desired_revision'] = (int) ($expectation['desired_revision'] ?? 0); + $state['desired_digest'] = self::safeDigest($expectation['desired_digest'] ?? null); + $state['private_ip_cidr'] = is_string($observation['private_ip_cidr'] ?? null) + ? trim($observation['private_ip_cidr']) : null; + $state['private_mac'] = is_string($observation['private_mac'] ?? null) + ? strtolower(trim($observation['private_mac'])) : null; + $state['state'] = is_string($observation['state'] ?? null) + ? strtolower(trim($observation['state'])) : 'uncertain'; + $state['readback'] = is_string($observation['readback'] ?? null) + ? strtolower(trim($observation['readback'])) : 'missing'; + + if (!$expectedRequired) { + // When a structured observation is available, a legacy VM is + // successful only when the host readback proves that no private + // NIC exists. An unexpected private interface, even if its XML + // looks healthy, is topology drift and must not be hidden behind + // a successful public-only operation. The worker compatibility + // gate handles older responses that omit private_state entirely. + $ok = $state['state'] === 'disabled' + && $state['readback'] === 'confirmed' + && ($observation['required'] ?? false) === false; + if (!$ok) { + $state['state'] = 'uncertain'; + $state['reason_code'] = 'vpc_private_observation_missing'; + } + return ['ok' => $ok, 'state' => $state, + 'error_code' => $ok ? null : $state['reason_code']]; + } + + $linkExpected = match ($kind) { + 'network_suspend' => 'down', + // service_resume restores the VM power/autostart target. A + // quota-blocked VM may still intentionally keep every NIC down; + // network_resume is the separate operation that proves link-up. + 'network_resume' => 'up', + default => null, + }; + $identity = is_string($expectation['vpc_id'] ?? null) + && is_string($expectation['member_id'] ?? null) + && $observedVpcId !== null && $observedMemberId !== null + && hash_equals((string) $expectation['vpc_id'], $observedVpcId) + && hash_equals((string) $expectation['member_id'], $observedMemberId); + $shape = ($observation['required'] ?? false) === true; + if ($expectedState === 'removed') { + // Teardown evidence is allowed to omit the interface/MAC only + // when it explicitly reports the removed state. Identity, + // revision and digest below still bind the tombstone to the + // original VPC member. + $shape = $shape && in_array($state['state'], ['disabled', 'removed'], true); + } else { + $shape = $shape && ($state['interface'] ?? null) === 'private0' + && is_string($state['private_mac'] ?? null) + && preg_match('/^[0-9a-f]{2}(?::[0-9a-f]{2}){5}$/Di', + (string) $state['private_mac']) === 1; + } + $revision = $state['observed_revision'] === $state['desired_revision'] + && $state['desired_revision'] > 0; + $digest = is_string($state['observed_digest']) + && is_string($state['desired_digest']) + && hash_equals($state['desired_digest'], $state['observed_digest']); + $ip = $expectedState === 'removed' + || (is_string($expectation['private_ip_cidr']) + && hash_equals((string) $expectation['private_ip_cidr'], (string) $state['private_ip_cidr'])); + $desired = $expectedState === 'removed' + ? in_array($state['state'], ['disabled', 'removed'], true) + : in_array($state['state'], ['active', 'suspended'], true); + $link = $linkExpected === null || $state['link_state'] === $linkExpected; + $ok = $shape && $identity && $revision && $digest && $ip && $desired && $link + && $state['readback'] === 'confirmed'; + if (!$ok) { + $state['state'] = $state['state'] === 'degraded' ? 'degraded' : 'uncertain'; + $state['readback'] = $state['readback'] === 'mismatch' ? 'mismatch' : 'missing'; + $state['reason_code'] = $expectedState === 'removed' + ? 'vpc_private_teardown_uncertain' : 'vpc_private_observation_mismatch'; + } + return ['ok' => $ok, 'state' => $state, + 'error_code' => $ok ? null : $state['reason_code']]; + } + + /** Persist the latest VM-side observation without advancing a VPC root. */ + public static function recordPrivateObservation( + int $instanceId, array $validation, string $operationId + ): bool { + if ($instanceId < 1 || preg_match(self::KEY_PATTERN, $operationId) !== 1) { + return false; + } + $connection = (new CloudVpcMember())->getConnection(); + $schema = $connection->getSchemaBuilder(); + if (!$schema->hasTable('cloud_vpc_members') + || !$schema->hasTable('cloud_vpc_private_ips') + || !$schema->hasTable('cloud_vpcs')) { + return false; + } + return (bool) $connection->transaction(function () use ( + $instanceId, $validation, $operationId + ): bool { + // Match the VPC -> member lock order used by control-plane + // mutations. A lifecycle readback from an older desired revision + // must never be written into a newer member row. + $probe = CloudVpcMember::where('instance_id', $instanceId) + ->whereNotIn('state', ['deleted'])->first(); + if ($probe === null) { + return false; + } + $vpc = CloudVpc::where('id', (int) $probe->vpc_id) + ->lockForUpdate()->first(); + if ($vpc === null) { + return false; + } + $member = CloudVpcMember::where('id', (int) $probe->id) + ->whereNotIn('state', ['deleted'])->lockForUpdate()->first(); + if ($member === null) { + return false; + } + // Lock and validate the independent lease before changing either + // side of the member/lease pair. A VM readback must never copy + // its revision into a lease belonging to another member. + if ($member->private_ip_id === null) { + return false; + } + $private = CloudVpcPrivateIp::where('id', (int) $member->private_ip_id) + ->where('vpc_id', (int) $vpc->id) + ->where('member_id', (int) $member->id) + ->where('instance_id', $instanceId) + ->lockForUpdate()->first(); + if ($private === null + || (int) $private->subnet_id !== (int) $member->subnet_id + || (string) $private->private_ip !== (string) $member->private_ip) { + return false; + } + $state = is_array($validation['state'] ?? null) + ? $validation['state'] : []; + $ok = ($validation['ok'] ?? false) === true; + $desired = (string) ($state['desired_state'] ?? 'active'); + $expectedRevision = max(0, (int) ($state['desired_revision'] ?? 0)); + $expectedDigest = self::safeDigest($state['desired_digest'] ?? null); + $memberDigest = self::safeDigest($member->desired_digest); + if ((int) $member->desired_revision !== $expectedRevision + || $memberDigest !== $expectedDigest) { + return false; + } + // `last_operation_id` on a member is the VPC proof marker. A + // VM lifecycle job may refresh the private lease, but must not + // replace a successful root operation marker with its unrelated + // job ID; doing so would make an already-proven VPC appear stale. + $rootOperationId = (string) ($vpc->last_operation_id ?? ''); + if ($rootOperationId === '' || $rootOperationId === $operationId) { + $member->last_operation_id = $operationId; + } + $member->observed_revision = max(0, (int) ($state['observed_revision'] ?? 0)); + $member->observed_digest = self::safeDigest($state['observed_digest'] ?? null); + // Mirror the member outcome into its lease row. A private-IP + // state left at `deleting` or `active` after readback would make + // the next expectation disagree with the member ledger. + $privateState = $ok + ? ($desired === 'removed' ? 'deleted' : 'active') + : ($desired === 'removed' ? 'deleting' : 'uncertain'); + if ($ok && $desired === 'removed') { + $member->state = 'deleted'; + $member->last_error_code = null; + $private->state = $privateState; + $private->observed_revision = $member->observed_revision; + $private->observed_digest = $member->observed_digest; + $private->last_error_code = null; + $private->updated_at = time(); + $private->save(); + } elseif ($ok) { + $member->state = 'active'; + $member->last_error_code = null; + } else { + $member->state = $desired === 'removed' ? 'deleting' : 'uncertain'; + $member->last_error_code = is_string($validation['error_code'] ?? null) + ? self::safeErrorCode((string) $validation['error_code']) : 'vpc_private_observation_missing'; + } + $member->updated_at = time(); + $member->save(); + $private->state = $privateState; + $private->observed_revision = $member->observed_revision; + $private->observed_digest = $member->observed_digest; + $private->last_error_code = $member->last_error_code; + $private->updated_at = time(); + $private->save(); + return true; + }, 3); + } + + private static function safeDigest($value): ?string + { + return is_string($value) && preg_match('/^[a-f0-9]{64}$/Di', $value) === 1 + ? strtolower($value) : null; + } + private static function touchDesiredLocked(CloudVpc $vpc, string $kind, string $requestKey, string $requestHash): array { @@ -1920,6 +2531,10 @@ final class CloudVpcService 'observed_revision' => max(0, (int) ($host['observed_revision'] ?? $host['revision'] ?? 0)), 'observed_digest' => is_string($host['observed_digest'] ?? $host['digest'] ?? null) ? substr((string) ($host['observed_digest'] ?? $host['digest']), 0, 64) : null, + 'transport_verified' => ($host['transport_verified'] ?? false) === true, + 'private_dataplane_verified' => ( + ($host['private_dataplane_verified'] ?? false) === true + ), 'next_stage_allowed' => ($host['next_stage_allowed'] ?? false) === true, ]; } diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 11130f75..6b7c1684 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -6,6 +6,7 @@ use App\Models\CloudInvoice; use App\Models\CloudJob; use App\Models\CloudOrder; use App\Models\CloudServicePeriod; +use App\Models\CloudVpc; use App\Models\User; final class CloudWorkerService @@ -179,7 +180,14 @@ final class CloudWorkerService $instance = CloudInstance::find($job->instance_id); $desired = self::policyDesired($instance, (string) $job->kind); if (!$desired) { - $this->finish($job, ['status' => 'succeeded', 'result' => ['skipped' => true]], $payload); + // Even a no-op policy result crosses the instance + // lifecycle boundary: a stale/missing private readback + // must not be hidden by the early skip path. + $skipped = $this->applyPrivateLifecycleGate( + $job, ['status' => 'succeeded', 'result' => ['skipped' => true]], + $payload, $actor + ); + $this->finish($job, $skipped, $payload); return true; } } @@ -280,8 +288,11 @@ final class CloudWorkerService && ($operation['service_was_running_hint'] ?? false) === true) { $result['service_was_running'] = true; } - $this->finish($job, ['status' => 'succeeded', - 'result' => $result], $payload); + $skipped = $this->applyPrivateLifecycleGate( + $job, ['status' => 'succeeded', 'result' => $result], + $payload, $actor + ); + $this->finish($job, $skipped, $payload); return true; } $operation = $this->engine->host((int) $job->host_id, @@ -347,6 +358,15 @@ final class CloudWorkerService $operation = ['status' => $operation['status'] ?? 'succeeded', 'result' => $operation]; } + // Every terminal VM mutation gets the same private-network + // readback gate. The engine's normal VM payload contains only a + // domain/XML observation; an attached VPC additionally requires + // member identity, revision and digest evidence from the future + // data-plane reconciler. A missing or stale observation is kept + // uncertain before finish() can commit public/billing state. + $operation = $this->applyPrivateLifecycleGate( + $job, $operation, $payload, $actor + ); $this->finish($job, $operation, $payload); } catch (\Throwable $error) { // Diagnostics deliberately contain no raw payloads, credentials or upstream tracebacks. @@ -504,7 +524,7 @@ final class CloudWorkerService $this->observedErrors[] = $status === 'failed' ? 'task_failed' : 'task_uncertain'; } $result = $operation['result'] ?? null; - (new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, $status, $result, $operation) { + (new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, &$status, &$result, &$operation) { // All financial transactions acquire the wallet first, avoiding wallet/order deadlocks. $user = User::where('id', $claimed->user_id)->lockForUpdate()->first(); $order = $claimed->order_id === null ? null : CloudOrder::where('id', $claimed->order_id)->lockForUpdate()->first(); @@ -513,6 +533,58 @@ final class CloudWorkerService || in_array($job->status, ['succeeded', 'failed', 'cancelled'], true)) { return; } + if ($status === 'succeeded' && $job->instance_id !== null) { + $capturedPrivateExpectation = $payload['private_expectation'] ?? null; + // VPC mutators lock VPC -> instance. Acquire the same order + // before checking the captured target so detach/attach cannot + // change a member between the readback and this commit. + $privateProbe = is_array($capturedPrivateExpectation) + ? $capturedPrivateExpectation + : CloudVpcService::instancePrivateExpectation((int) $job->instance_id); + $vpcKeyToLock = is_string($privateProbe['vpc_id'] ?? null) + ? (string) $privateProbe['vpc_id'] : ''; + if (preg_match('/^[0-9a-f]{32}$/D', $vpcKeyToLock) === 1) { + CloudVpc::where('vpc_key', $vpcKeyToLock) + ->lockForUpdate()->first(); + } + $lockedInstance = CloudInstance::where('id', $job->instance_id) + ->lockForUpdate()->first(); + $privateTargetChanged = $lockedInstance === null; + if (!$privateTargetChanged && is_array($capturedPrivateExpectation)) { + $privateTargetChanged = !CloudVpcService::privateExpectationMatches( + (int) $lockedInstance->id, $capturedPrivateExpectation + ); + } elseif (!$privateTargetChanged) { + // Jobs created before the immutable expectation field was + // introduced cannot prove that an attached VPC target + // stayed unchanged. Allow legacy public-only jobs to + // finish, but fail closed for any live private member. + $privateTargetChanged = CloudVpcService::instancePrivateExpectation( + (int) $lockedInstance->id + )['required'] === true; + } + if ($privateTargetChanged) { + // The VPC target changed after the worker's engine + // readback. Keep the operation uncertain while holding + // the same instance row lock used by attach/detach. + $status = 'uncertain'; + $operation['status'] = 'uncertain'; + $operation['error_code'] = 'vpc_private_observation_mismatch'; + $operation['error'] = '实例私网目标已变化,结果正在核对'; + if (is_array($operation['result'] ?? null)) { + $operation['result']['private_state'] = [ + 'version' => 1, 'required' => true, + 'state' => 'uncertain', 'desired_state' => 'active', + 'readback' => 'mismatch', + 'reason_code' => 'vpc_private_observation_mismatch', + 'desired_revision' => 0, 'observed_revision' => 0, + 'desired_digest' => null, 'observed_digest' => null, + 'link_state' => 'absent', 'observed_at' => time(), + ]; + } + $result = $operation['result'] ?? null; + } + } $safe = is_array($result) ? CloudOperationErrorCatalog::workerResult($result) : null; CloudSnapshotService::finishJobLocked($job, $status, $result, $payload); @@ -973,6 +1045,265 @@ final class CloudWorkerService }, 3); } + /** + * Failed/uncertain jobs normally must keep their original disposition, + * but a job that crossed the engine mutation boundary still needs the + * same private-state readback as a successful job. The durable engine + * marker avoids treating ordinary request validation failures as if they + * had changed a VM. Network changes retain their narrower checkpoint: + * the child must have claimed the target IP before this gate runs. + * + * The actual bounded private-state read happens outside finish()'s DB + * transaction below, so a slow or failed engine read never holds wallet or + * VPC rows. + */ + private static function privateMutationReadbackRequired( + CloudJob $job, array $operation, string $status + ): bool { + if (!in_array($status, ['failed', 'uncertain'], true)) { + return false; + } + $kind = (string) $job->kind; + $result = is_array($operation['result'] ?? null) + ? $operation['result'] : []; + $vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result; + + // Public-IP changes already have a commit checkpoint inside the + // child. A generic invocation marker alone would gate a preflight + // rejection before any address or XML mutation took place. + if ($kind === 'network_change') { + if (($operation['network_mutation_started'] ?? false) === true) { + return true; + } + foreach (['network_changed', 'network_mutation_started'] as $key) { + if (($result[$key] ?? false) === true || ($vm[$key] ?? false) === true) { + return true; + } + } + return false; + } + + if (($operation['mutation_started'] ?? false) === true + || ($operation['network_mutation_started'] ?? false) === true) { + return true; + } + if (in_array($kind, ['delete', 'expiry_delete'], true) + && (($operation['vm_delete_committed'] ?? false) === true + || array_key_exists('cleanup_pending', $operation))) { + return true; + } + if ($kind === 'reinstall' && self::reinstallNeedsReview($operation)) { + return true; + } + // Preserve existing post-commit markers from older engine versions. + foreach (['reinstall_committed', 'cleanup_pending', 'network_changed', + 'settings_changed', 'restored', 'deleted', 'created'] as $key) { + if (($result[$key] ?? false) === true || ($vm[$key] ?? false) === true) { + return true; + } + } + // An uncertain job dispatched by a legacy engine has no durable + // mutation marker. Its dispatched_at timestamp is the conservative + // compatibility signal; failed pre-dispatch jobs keep a false gate. + return $status === 'uncertain' && $job->dispatched_at !== null; + } + + private function applyPrivateLifecycleGate( + CloudJob $job, array $operation, array $payload, array $actor + ): array { + $status = is_string($operation['status'] ?? null) + ? strtolower((string) $operation['status']) : 'uncertain'; + $kind = (string) $job->kind; + $postCommitReinstall = $kind === 'reinstall' + && $status === 'failed' && self::reinstallNeedsReview($operation); + $terminalSuccess = in_array($status, ['succeeded', 'completed', 'done'], true); + if (!$terminalSuccess && !$postCommitReinstall + && !self::privateMutationReadbackRequired($job, $operation, $status)) { + return $operation; + } + $lifecycleKinds = [ + 'create', 'reinstall', 'network_change', 'instance_settings', 'rescue', + 'snapshot_create', 'snapshot_restore', 'snapshot_delete', 'delete', + 'expiry_delete', 'action', 'service_suspend', 'service_resume', + 'network_suspend', 'network_resume', 'firewall_apply', + ]; + if (!in_array($kind, $lifecycleKinds, true)) { + return $operation; + } + + $instance = $job->instance_id === null ? null : CloudInstance::find($job->instance_id); + $currentExpectation = CloudVpcService::instancePrivateExpectation( + $instance === null ? null : (int) $instance->id + ); + $capturedExpectation = is_array($payload['private_expectation'] ?? null) + ? $payload['private_expectation'] : null; + $expectationStale = false; + $expectation = $currentExpectation; + if ($capturedExpectation !== null) { + $expectation = $capturedExpectation; + // `state` is an observed/transient lifecycle value. A member can + // legitimately move applying -> active while this VM job is in + // flight; desired_state plus the revision/digest still bind the + // immutable target. Removal is represented by desired_state and + // therefore remains strict. + foreach (['required', 'desired_state', 'vpc_id', 'member_id', + 'private_ip_cidr', 'desired_revision', 'desired_digest'] as $key) { + $left = $capturedExpectation[$key] ?? null; + $right = $currentExpectation[$key] ?? null; + if ((string) $left !== (string) $right) { + $expectationStale = true; + break; + } + } + if ($expectationStale) { + // Keep the original target in the evidence, but make the + // gate fail closed instead of validating it against a newer + // VPC revision that arrived while this job was running. + $expectation['required'] = true; + $expectation['reason_code'] = 'vpc_private_observation_mismatch'; + } + } + $result = is_array($operation['result'] ?? null) ? $operation['result'] : []; + $vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result; + $uuid = is_string($instance?->uuid) ? (string) $instance->uuid + : (is_string($vm['uuid'] ?? null) ? $vm['uuid'] + : (is_string($result['uuid'] ?? null) ? $result['uuid'] : null)); + // Accept both the normal {vm:{...}} envelope and a rolling-upgrade + // response that places the bounded observation beside `vm`. A + // structured field at either level must stay strict; only a complete + // omission can use the legacy public-only compatibility path. + $hasPrivateState = array_key_exists('private_state', $vm) + || array_key_exists('private_state', $result); + $observation = is_array($vm['private_state'] ?? null) + ? $vm['private_state'] + : (is_array($result['private_state'] ?? null) + ? $result['private_state'] : null); + + // Pre-VPC engines and the existing public-only worker fixtures do not + // return a private_state object at all. There is no private target to + // prove for those jobs, so do not issue a second host request (which + // would also consume the next response in a legacy mock queue). Once + // a VPC member is expected, the gate remains fail-closed below; and a + // structured observation supplied by any engine is still validated, + // including the public-only case where it can reveal an unexpected + // private NIC. + if (($expectation['required'] ?? false) !== true && !$hasPrivateState) { + return $operation; + } + + // A delete may legitimately have no readable domain after the + // irreversible boundary. Only a VM without a VPC target can use the + // explicit disabled/confirmed result; a VPC tombstone still requires + // a future removed/readback proof. + if ($uuid !== null) { + try { + $observed = $this->engine->host( + (int) $job->host_id, 'GET', + 'vms/' . rawurlencode($uuid) . '/private-state', + $actor, null, (string) $job->operation_id + ); + if (is_array($observed)) { + $observation = $observed; + } + } catch (\Throwable $error) { + $absenceProven = in_array($kind, ['delete', 'expiry_delete'], true) + && $expectation['required'] !== true + && $error instanceof CloudError + && $error->httpStatus === 404; + if ($absenceProven) { + // A VM with no VPC member has no independent private + // teardown target. Once the engine has confirmed the + // domain deletion, a 404 is the expected absence proof. + $observation = [ + 'version' => 1, 'required' => false, + 'vm_uuid' => $uuid, 'vpc_id' => null, 'member_id' => null, + 'interface' => null, 'state' => 'disabled', + 'desired_revision' => 0, 'observed_revision' => 0, + 'desired_digest' => null, 'observed_digest' => null, + 'readback' => 'confirmed', 'link_state' => 'absent', + 'private_ip_cidr' => null, 'private_mac' => null, + 'reason_code' => null, 'observed_at' => time(), + ]; + } + if (!$absenceProven) { + // The operation itself may already be durable. Convert a + // missing observation into a stable uncertain result + // rather than leaking a transport/host diagnostic. + $observation = null; + } + } + } + $validation = $expectationStale + ? ['ok' => false, 'state' => [ + 'version' => 1, 'required' => true, + 'vm_uuid' => $uuid, 'vpc_id' => $expectation['vpc_id'] ?? null, + 'member_id' => $expectation['member_id'] ?? null, + 'interface' => null, 'state' => 'uncertain', + 'desired_state' => $expectation['desired_state'] ?? 'active', + 'desired_revision' => (int) ($expectation['desired_revision'] ?? 0), + 'observed_revision' => 0, + 'desired_digest' => $expectation['desired_digest'] ?? null, + 'observed_digest' => null, 'readback' => 'mismatch', + 'link_state' => 'absent', 'private_ip_cidr' => null, + 'private_mac' => null, + 'reason_code' => 'vpc_private_observation_mismatch', + 'observed_at' => time(), + ], 'error_code' => 'vpc_private_observation_mismatch'] + : CloudVpcService::validatePrivateObservation( + $expectation, $observation, $kind + ); + if ($instance !== null && $expectation['required'] === true && !$expectationStale) { + try { + $recorded = CloudVpcService::recordPrivateObservation( + (int) $instance->id, $validation, (string) $job->operation_id + ); + if ($recorded !== true && $validation['ok'] === true) { + $validation = [ + 'ok' => false, + 'state' => array_merge($validation['state'], [ + 'state' => 'uncertain', 'readback' => 'mismatch', + 'reason_code' => 'vpc_private_observation_mismatch', + 'observed_at' => time(), + ]), + 'error_code' => 'vpc_private_observation_mismatch', + ]; + } + } catch (\Throwable $error) { + // The operation gate remains uncertain even when the + // secondary VPC observation cache cannot be written. + error_log('cloud private observation could not be recorded: ' + . get_class($error)); + if ($validation['ok'] === true) { + $validation = [ + 'ok' => false, + 'state' => array_merge($validation['state'], [ + 'state' => 'uncertain', 'readback' => 'missing', + 'reason_code' => 'vpc_private_observation_missing', + 'observed_at' => time(), + ]), + 'error_code' => 'vpc_private_observation_missing', + ]; + } + } + } + $result['private_state'] = $validation['state']; + if ($kind === 'reinstall' && !$validation['ok']) { + // Preserve the replacement credential/spec on a post-commit + // uncertain reinstall; the existing finish() compatibility path + // will not mark the task successful or replay it blindly. + $result['reinstall_committed'] = true; + } + $operation['result'] = $result; + if (!$validation['ok']) { + $operation['status'] = 'uncertain'; + $operation['safe_to_refund'] = false; + $operation['error_code'] = $validation['error_code'] + ?? 'vpc_private_observation_missing'; + $operation['error'] = '实例私网状态尚未完成实际回读,请稍后核对'; + } + return $operation; + } + private static function validSystemCleanup(CloudJob $job, array $payload): bool { if ($job->instance_id === null || ($payload['body'] ?? null) !== []) { diff --git a/web/tests/CloudVpcControlPlaneContractTest.php b/web/tests/CloudVpcControlPlaneContractTest.php index 67fbfbdc..23fdd424 100644 --- a/web/tests/CloudVpcControlPlaneContractTest.php +++ b/web/tests/CloudVpcControlPlaneContractTest.php @@ -22,6 +22,7 @@ $middleware = $read('app/Middleware/CloudApi.php'); $engineClient = $read('app/Services/Cloud/CloudEngineClient.php'); $accountDeletion = $read('app/Services/UserDataDeletionService.php'); $jobService = $read('app/Services/Cloud/CloudJobService.php'); +$orderService = $read('app/Services/Cloud/CloudOrderService.php'); $workerService = $read('app/Services/Cloud/CloudWorkerService.php'); foreach ([ @@ -88,6 +89,19 @@ $assert(!str_contains($service, 'CloudJob') && !str_contains($service, 'network_change') && !str_contains($service, 'publicIpCidr'), 'VPC service must not reuse the public network_change/job payload.'); +$assert(str_contains($service, '$orphanLease') + && str_contains($service, '$stateMatches') + && str_contains($service, "where('instance_id', \$instanceId)") + && str_contains($service, "where('vpc_id', (int) \$vpc->id)") + && str_contains($service, 'hash_equals($rootOperationId') + && str_contains($service, "where('status', 'succeeded')") + && str_contains($service, '$operationFresh') + && str_contains($service, '$vpc->last_success_at') + && str_contains($service, '$privateIp->observed_revision') + && str_contains($service, '$privateIp->observed_digest') + && str_contains($service, "(string) \$privateIp->state === 'active'") + && str_contains($service, '$private->state = $privateState'), + 'VPC private projections must fail closed on orphan/mismatched leases and require current root readback proof.'); $assert(str_contains($service, '当前 VPC 控制面不支持子网网关') && str_contains($service, "'private_ip_cidr'") && str_contains($service, "'next_hop_member_id'") @@ -107,7 +121,20 @@ $assert(str_contains($engineClient, 'function vpc(') $assert(str_contains($accountDeletion, "cloud_vpcs") && str_contains($accountDeletion, "cloud_private_networks") && str_contains($jobService, 'assertInstanceCanBeDeleted') + && str_contains($orderService, "private_expectation") && str_contains($workerService, '旧删除任务缺少 VPC 删除门禁'), - 'Account/VM deletion can bypass the independent VPC control-plane ledger.'); + 'Account/VM creation/deletion can bypass the independent VPC control-plane ledger.'); +$legacyPrivateGate = strpos($workerService, + "if ((\$expectation['required'] ?? false) !== true && !\$hasPrivateState)"); +$privateReadback = strpos($workerService, + "'vms/' . rawurlencode(\$uuid) . '/private-state'"); +$assert(str_contains($workerService, '$hasPrivateState') + && str_contains($workerService, "&& !\$hasPrivateState") + && str_contains($workerService, "array_key_exists('private_state', \$result)") + && str_contains($workerService, 'Pre-VPC engines') + && $legacyPrivateGate !== false + && $privateReadback !== false + && $legacyPrivateGate < $privateReadback, + 'Legacy public-only jobs without private_state must remain compatible while structured observations stay gated.'); echo "VPC control-plane schema, API, typed DTO and independent operation contracts passed.\n";