diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py
index 5d13351d..1524a539 100644
--- a/engine/tests/test_engine_boundary.py
+++ b/engine/tests/test_engine_boundary.py
@@ -2234,6 +2234,8 @@ def test_power_action_has_persistent_idempotency(boundary):
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
saved = json.loads(store.get("operation", "reboot-operation"))
assert saved["target_action"] == "reboot"
+ assert saved["mutation_started"] is True
+ assert operation["mutation_started"] is True
def test_orphaned_power_action_reconciles_only_the_final_target_state(boundary):
diff --git a/engine/tests/test_libvirt_service.py b/engine/tests/test_libvirt_service.py
index 1cf429f4..647168f4 100644
--- a/engine/tests/test_libvirt_service.py
+++ b/engine/tests/test_libvirt_service.py
@@ -5396,6 +5396,41 @@ class VMServiceTests(unittest.TestCase):
self.assertEqual(result[0].vm_id, "00000000-0000-4000-8000-000000000002")
self.assertEqual(self.backend.opened, self.backend.closed)
+ def test_metric_counters_reject_ambiguous_extra_nic_role(self):
+ constants = {
+ "VIR_DOMAIN_STATS_CPU_TOTAL": 2,
+ "VIR_DOMAIN_STATS_BALLOON": 4,
+ "VIR_DOMAIN_STATS_BLOCK": 8,
+ "VIR_DOMAIN_STATS_INTERFACE": 16,
+ "VIR_CONNECT_GET_ALL_DOMAINS_STATS_RUNNING": 32,
+ }
+ for name, value in constants.items():
+ setattr(self.service_module.libvirt, name, value)
+ domain = types.SimpleNamespace(
+ UUIDString=lambda: "00000000-0000-4000-8000-000000000003",
+ info=lambda: (1, 2048 * 1024, 2048 * 1024, 2, 0),
+ ID=lambda: 9,
+ XMLDesc=lambda _flags: (""
+ ""
+ ""
+ ""
+ ""
+ ""),
+ )
+
+ def get_stats(_connection, _groups, _flags):
+ return [(domain, {"cpu.time": 123, "net.count": 2,
+ "net.0.name": "vnet9", "net.0.rx.bytes": 456,
+ "net.0.tx.bytes": 789,
+ "net.1.name": "vnet10", "net.1.rx.bytes": 12,
+ "net.1.tx.bytes": 34})]
+
+ with patch.object(FakeConnection, "getAllDomainStats", get_stats, create=True):
+ result = self.service.metric_counters()
+ self.assertEqual(len(result), 1)
+ self.assertEqual(result[0].values["net.0.role"], "public")
+ self.assertEqual(result[0].values["net.1.role"], "unknown")
+
if __name__ == "__main__":
unittest.main()
diff --git a/engine/tests/test_metrics.py b/engine/tests/test_metrics.py
index d75ecfe2..b6364371 100644
--- a/engine/tests/test_metrics.py
+++ b/engine/tests/test_metrics.py
@@ -496,6 +496,34 @@ def test_sampler_prefers_stable_network_identity_over_renamed_host_tap():
}
+def test_sampler_billing_frame_only_contains_explicit_public_role():
+ sampler = MetricSampler()
+ point = sampler.observe(host_scope(HOST), raw(extra={
+ "net.count": 2,
+ "net.0.identity": "mac:52:54:00:12:34:56",
+ "net.0.role": "public",
+ "net.1.name": "vnet2",
+ "net.1.identity": "mac:52:54:00:aa:bb:cc",
+ "net.1.role": "private",
+ "net.1.rx.bytes": 3000,
+ "net.1.tx.bytes": 4000,
+ }), timestamp=1000, monotonic=10)
+ assert point["network_billing_devices"] == {
+ "mac:52:54:00:12:34:56": [1000, 2000],
+ }
+
+
+def test_sampler_billing_rejects_ambiguous_multinic_frame():
+ sampler = MetricSampler()
+ point = sampler.observe(host_scope(HOST), raw(extra={
+ "net.count": 2,
+ "net.1.name": "vnet2",
+ "net.1.rx.bytes": 3000,
+ "net.1.tx.bytes": 4000,
+ }), timestamp=1000, monotonic=10)
+ assert point["network_billing_devices"] is None
+
+
def test_collector_associates_host_and_current_owner_without_hiding_unowned_vms():
saved = []
diff --git a/engine/tests/test_private_network.py b/engine/tests/test_private_network.py
new file mode 100644
index 00000000..9ee7e298
--- /dev/null
+++ b/engine/tests/test_private_network.py
@@ -0,0 +1,88 @@
+from virsh_panel.private_network import observe_private_state_from_xml
+
+
+def _xml(*, private=True, link="up", private_source="vc-private",
+ private_ip="10.24.3.7/32", verified="true", private_filter=False,
+ public_source="br0", private_type="bridge"):
+ private_block = ""
+ if private:
+ filt = '' if private_filter else ""
+ source = (f''
+ if private_type == "bridge"
+ else f'')
+ private_block = f'''
+
+ {source}
+ {filt}
+
+ '''
+ return f'''00000000-0000-4000-8000-000000000001
+
+
+
+
+ {private_block}
+ '''
+
+
+def test_private_state_without_private_nic_is_confirmed_disabled():
+ state = observe_private_state_from_xml(_xml(private=False), vm_uuid="00000000-0000-4000-8000-000000000001", observed_at=10)
+ assert state["state"] == "disabled"
+ assert state["readback"] == "confirmed"
+ assert state["interface"] is None
+
+
+def test_private_state_requires_verified_independent_bridge_and_ip():
+ state = observe_private_state_from_xml(_xml(), observed_at=10)
+ assert state["state"] == "degraded"
+ assert state["reason_code"] == "private_source_unverified"
+ state = observe_private_state_from_xml(_xml(), host_verified=True, observed_at=10)
+ assert state["state"] == "active"
+ assert state["readback"] == "confirmed"
+ assert state["private_ip_cidr"] == "10.24.3.7/32"
+
+ unverified = observe_private_state_from_xml(_xml(verified="false"))
+ assert unverified["state"] == "degraded"
+ assert unverified["reason_code"] == "private_source_unverified"
+
+ missing_ip = observe_private_state_from_xml(_xml(private_ip=None))
+ assert missing_ip["state"] == "degraded"
+ assert missing_ip["reason_code"] == "private_ip_missing"
+
+
+def test_private_state_marks_link_down_suspended():
+ state = observe_private_state_from_xml(_xml(link="down"), host_verified=True)
+ assert state["state"] == "suspended"
+ assert state["link_state"] == "down"
+
+
+def test_private_state_rejects_public_source_or_filter():
+ shared = observe_private_state_from_xml(_xml(private_source="br0"))
+ assert shared["state"] == "degraded"
+ assert shared["reason_code"] == "private_source_invalid"
+
+ filtered = observe_private_state_from_xml(_xml(private_filter=True), host_verified=True)
+ assert filtered["state"] == "degraded"
+ assert filtered["reason_code"] == "private_filter_present"
+
+
+def test_private_network_source_network_is_not_proven_isolated():
+ state = observe_private_state_from_xml(
+ _xml(private_source="default", private_type="network"), host_verified=True
+ )
+ # The helper only accepts an explicit host bridge; a libvirt network name
+ # does not prove which bridge it resolves to.
+ assert state["state"] == "degraded"
+ assert state["reason_code"] == "private_source_invalid"
+
+
+def test_private_state_rejects_ambiguous_extra_nic_without_private_role():
+ xml = _xml(private=False).replace(
+ '',
+ ''
+ '',
+ )
+ state = observe_private_state_from_xml(xml)
+ assert state["state"] == "degraded"
+ assert state["reason_code"] == "private_interface_role_invalid"
diff --git a/engine/tests/test_vpc.py b/engine/tests/test_vpc.py
index 00c5e050..f854c8bb 100644
--- a/engine/tests/test_vpc.py
+++ b/engine/tests/test_vpc.py
@@ -292,6 +292,7 @@ def test_coordinator_requires_exact_fabric_revision_and_all_host_readback():
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
+ "private_dataplane_verified": True,
"readback": {"safe": "ok", "oversized": "x" * 513}},
}))
assert observed["state"] == "active"
@@ -303,6 +304,25 @@ def test_coordinator_requires_exact_fabric_revision_and_all_host_readback():
assert blocked["next_stage_allowed"] is False
+def test_transport_only_observation_never_opens_private_dataplane_gate():
+ store = MemoryStore()
+ fabric = FakeFabric(hosts=("host-a",))
+ coordinator = VpcCoordinator(store, fabric)
+ result = asyncio.run(coordinator.apply(payload()))
+ digest = result["desired_digest"]
+ pending = asyncio.run(coordinator.observe("vpc-hk-1", {
+ "host-a": {"state": "active", "observed_revision": 1,
+ "observed_digest": digest, "transport_verified": True},
+ }))
+ assert pending["state"] == "applying"
+ assert pending["next_stage_allowed"] is False
+ assert pending["private_dataplane_verified"] is False
+ assert pending["hosts"][0]["state"] == "applying"
+ assert pending["hosts"][0]["last_error_code"] == (
+ "vpc_private_dataplane_unverified"
+ )
+
+
def test_partial_host_observation_is_degraded_and_never_active():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
@@ -322,7 +342,8 @@ def test_partial_host_observation_is_degraded_and_never_active():
digest = result["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
- "observed_digest": digest, "transport_verified": True},
+ "observed_digest": digest, "transport_verified": True,
+ "private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
@@ -331,7 +352,8 @@ def test_partial_host_observation_is_degraded_and_never_active():
with pytest.raises(VpcError) as inconsistent:
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
- "observed_digest": digest, "transport_verified": False},
+ "observed_digest": digest, "transport_verified": False,
+ "private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
@@ -454,7 +476,8 @@ def test_recover_invalidates_observed_gate_and_record_round_trips():
raw["observed_digest"] = raw["desired_digest"]
raw["hosts"][0].update(state="active", observed_revision=1,
observed_digest=raw["desired_digest"],
- transport_verified=True)
+ transport_verified=True,
+ private_dataplane_verified=True)
store.set("vpc_desired", "vpc-hk-1", json.dumps(raw))
coordinator.recover()
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
@@ -492,7 +515,8 @@ def test_new_desired_revision_keeps_last_observed_evidence_until_readback():
digest = first["desired_digest"]
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
- "observed_digest": digest, "transport_verified": True},
+ "observed_digest": digest, "transport_verified": True,
+ "private_dataplane_verified": True},
}))
second = asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
diff --git a/engine/virsh_panel/app.py b/engine/virsh_panel/app.py
index b5fd5bec..04e44e6e 100644
--- a/engine/virsh_panel/app.py
+++ b/engine/virsh_panel/app.py
@@ -3862,6 +3862,34 @@ def create_app(
require_vm_access(_session, profile, vm_id)
return new_service(profile).get_vm_state(vm_id)
+ @app.get("/api/v1/vms/{vm_id}/private-state")
+ def get_vm_private_state(vm_id: str, _session: Session = Depends(require_approved)):
+ """Return the bounded private-NIC readback for one VM.
+
+ This endpoint intentionally reports the domain/XML observation only;
+ VPC revision, WireGuard peer and nftables evidence are checked by the
+ PHP control plane before it publishes a mutation as successful.
+ """
+ profile = current_profile()
+ require_vm_access(_session, profile, vm_id)
+ service = new_service(profile)
+ getter = getattr(service, "get_private_state", None)
+ if callable(getter):
+ return getter(vm_id)
+ # Test/rolling-upgrade adapters may only expose get_vm; preserve the
+ # same bounded contract without making the route a second XML parser.
+ vm = service.get_vm(vm_id)
+ return vm.get("private_state", {
+ "version": 1, "required": False, "vm_uuid": vm.get("uuid"),
+ "vpc_id": None, "member_id": None, "interface": None,
+ "state": "uncertain", "desired_revision": 0,
+ "observed_revision": 0, "desired_digest": None,
+ "observed_digest": None, "readback": "missing",
+ "link_state": "absent", "private_ip_cidr": None,
+ "private_mac": None, "reason_code": "private_state_unavailable",
+ "observed_at": int(time.time()),
+ })
+
@app.get("/api/v1/vms/{vm_id}")
def get_vm(vm_id: str, _session: Session = Depends(require_approved)):
profile = current_profile()
diff --git a/engine/virsh_panel/database.py b/engine/virsh_panel/database.py
index e9866619..a54af544 100644
--- a/engine/virsh_panel/database.py
+++ b/engine/virsh_panel/database.py
@@ -337,6 +337,11 @@ class DatabaseStore:
or user_id > _MAX_UNSIGNED_BIGINT)):
raise ValueError("Metric user ID is invalid")
raw_devices = point.get("network_devices")
+ # New collectors provide a role-filtered public0 frame for quota
+ # accounting. Keep the legacy all-NIC fallback for old callers
+ # and old hosts that have not emitted explicit role metadata.
+ if "network_billing_devices" in point:
+ raw_devices = point.get("network_billing_devices")
# None means libvirt did not return a complete NIC counter frame.
# Never replace a durable baseline with incomplete data, otherwise
# the next healthy sample would look like a brand-new NIC and the
diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py
index e2d31a78..df7ecfbe 100644
--- a/engine/virsh_panel/internal.py
+++ b/engine/virsh_panel/internal.py
@@ -137,7 +137,7 @@ READ_PATHS = (
r"fabric", r"fabric/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}",
r"vpcs", r"vpcs/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}",
r"user-isos/[0-9a-f]{32}",
- rf"vms/{VM_ID}(?:/state|/metrics|/traffic/daily|/rescue|"
+ rf"vms/{VM_ID}(?:/state|/private-state|/metrics|/traffic/daily|/rescue|"
r"/firewall|/snapshots(?:/[0-9a-f]{32})?)?",
r"(?:reinstall-jobs|rescue-jobs|vm-templates/import-jobs|rescue-templates/import-jobs|user-isos/import-jobs)/[A-Za-z0-9_-]+",
)
@@ -868,6 +868,16 @@ class EngineApplication:
elif self._is_vm_delete_operation(operation):
await self._capture_delete_cleanup(
host_id, operation, delegated)
+ # Persist the mutation boundary before invoking the child.
+ # If this write fails, the child is not called and the
+ # operation remains a safe prevalidation/coordination failure.
+ # Network changes have a narrower checkpoint inside the child;
+ # PHP additionally requires network_mutation_started for that
+ # resource before gating failed results.
+ operation["mutation_started"] = True
+ await asyncio.to_thread(
+ self.store.set, "operation", operation["operation_id"],
+ json.dumps(operation))
await app(child_scope, receive, capture)
status = next(item["status"] for item in messages if item["type"] == "http.response.start")
payload = json.loads(b"".join(item.get("body", b"") for item in messages
@@ -914,8 +924,9 @@ class EngineApplication:
if operation["status"] in {"succeeded", "pending"}:
await json_response(send, status, {**payload, **self._public_operation(operation)})
else:
- await json_response(send, status, {**payload, "operation_id": operation["operation_id"],
- "status": operation["status"], "safe_to_refund": operation["safe_to_refund"]})
+ await json_response(send, status, {
+ **payload, **self._public_operation(operation),
+ })
except BaseException as exc:
network_preflight_failed = (
self._is_network_change_resource(
@@ -1922,7 +1933,11 @@ class EngineApplication:
"actor_can_manage": bool(delegated.get("can_manage")),
"actor_is_admin": bool(delegated.get("is_admin")),
"resource": resource, "method": method, "status": "running", "result": None,
- "safe_to_refund": False, "created_at": int(time.time())}
+ "safe_to_refund": False, "created_at": int(time.time()),
+ # Set immediately before invoking the child endpoint. This
+ # lets the PHP worker distinguish an actual mutation boundary
+ # from a durable prevalidation failure.
+ "mutation_started": False}
if create:
# New PHP-backed creates always receive the reserved UUID below.
# Keep this durable marker so cleanup reconciliation checks the
@@ -3166,6 +3181,11 @@ class EngineApplication:
"operation_id", "status", "result", "error", "safe_to_refund",
"service_was_running_hint", "created_at", "finished_at")
if key in record}
+ # Expose only bounded lifecycle markers. Do not leak persisted
+ # request bodies or host diagnostics through the operation API.
+ for key in ("mutation_started", "network_mutation_started"):
+ if isinstance(record.get(key), bool):
+ result[key] = record[key]
if ("/snapshots" in record.get("resource", "")
and record.get("error_code") in SNAPSHOT_PUBLIC_ERROR_CODES):
result["error_code"] = record["error_code"]
diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py
index eb5807bb..9675ab08 100644
--- a/engine/virsh_panel/libvirt_service.py
+++ b/engine/virsh_panel/libvirt_service.py
@@ -41,6 +41,7 @@ from virsh_panel.network_roles import (
select_private_interface,
select_public_interface,
)
+from virsh_panel.private_network import observe_private_state
from virsh_panel.platform_network_policy import (
MAX_PLATFORM_POLICY_SNAPSHOT_BYTES,
PLATFORM_DEFAULT_NETWORK_POLICY,
@@ -1219,6 +1220,13 @@ class VMService:
rescue.get("volume") if rescue is not None else
inactive_rescue.get("volume") if inactive_rescue is not None else None),
"console_port": self._vnc_port(root),
+ # This is a bounded XML observation only. It deliberately does
+ # not claim that the host WireGuard Fabric, guest route or nft
+ # ACL is active; the PHP VPC ledger supplies that expectation at
+ # each mutation commit boundary.
+ "private_state": observe_private_state(
+ root, vm_uuid=domain.UUIDString()
+ ),
}
@staticmethod
@@ -1465,6 +1473,18 @@ class VMService:
with self._connect() as conn:
return self._vm_dict(self._domain(conn, uuid))
+ def get_private_state(self, uuid: str) -> dict[str, Any]:
+ """Read one VM's bounded private-NIC observation.
+
+ Keep this as a separate endpoint so lifecycle reconciliation does not
+ need to load disk capacities or console metadata, and so a host-wide
+ inventory cannot accidentally execute a per-VM network probe.
+ """
+ with self._connect() as conn:
+ domain = self._domain(conn, uuid)
+ root = self._vm_xml(domain)
+ return observe_private_state(root, vm_uuid=domain.UUIDString())
+
def get_vm_state(self, uuid: str) -> dict[str, Any]:
"""Fetch one domain's power state without loading its XML metadata."""
with self._connect() as conn:
@@ -1500,8 +1520,39 @@ class VMService:
values = dict(values)
try:
root = ET.fromstring(domain.XMLDesc(0))
- by_target: dict[str, str] = {}
- for interface in root.findall("./devices/interface"):
+ interfaces = root.findall("./devices/interface")
+ # Resolve the complete role map before annotating any
+ # counters. Calling ``interface_role(...,
+ # legacy_public=True)`` independently would classify an
+ # extra unaliased NIC as public on a multi-NIC domain and
+ # could charge private traffic to the public quota lane.
+ try:
+ public_interface = select_public_interface(interfaces)
+ private_interface = select_private_interface(interfaces)
+ role_by_element = {
+ id(item): (
+ "public" if item is public_interface
+ else "private" if item is private_interface
+ else "unknown"
+ )
+ for item in interfaces
+ }
+ except (TypeError, ValueError):
+ # Preserve explicit roles for diagnostics while
+ # marking the complete frame unsafe. The billing
+ # sampler rejects any ``unknown`` role, but retaining
+ # the known public role keeps the observation useful
+ # for reconciliation and avoids silently relabelling
+ # a valid public0 interface.
+ role_by_element = {}
+ for item in interfaces:
+ try:
+ role_by_element[id(item)] = interface_role(
+ item, legacy_public=len(interfaces) == 1)
+ except ValueError:
+ role_by_element[id(item)] = "unknown"
+ by_target: dict[str, tuple[str, str]] = {}
+ for interface in interfaces:
target = interface.find("target")
mac = interface.find("mac")
target_name = target.get("dev") if target is not None else None
@@ -1510,14 +1561,18 @@ class VMService:
and isinstance(address, str)
and re.fullmatch(r"[0-9A-Fa-f]{2}(?::[0-9A-Fa-f]{2}){5}",
address)):
- by_target[target_name] = "mac:" + address.lower()
+ role = role_by_element.get(id(interface), "unknown")
+ by_target[target_name] = (
+ "mac:" + address.lower(), role
+ )
count = values.get("net.count")
if isinstance(count, int) and not isinstance(count, bool):
for number in range(min(max(0, count), 1024)):
name = values.get(f"net.{number}.name")
identity = by_target.get(name) if isinstance(name, str) else None
if identity is not None:
- values[f"net.{number}.identity"] = identity
+ values[f"net.{number}.identity"] = identity[0]
+ values[f"net.{number}.role"] = identity[1]
except (AttributeError, ET.ParseError, libvirt.libvirtError):
# Metrics remain useful on older/unusual domains whose
# live XML cannot be read; accounting falls back to the
diff --git a/engine/virsh_panel/metrics.py b/engine/virsh_panel/metrics.py
index 52ad5e46..29b6a51d 100644
--- a/engine/virsh_panel/metrics.py
+++ b/engine/virsh_panel/metrics.py
@@ -86,7 +86,9 @@ def _device_counters(stats: Mapping[str, Any], group: str,
return signature, counters_by_direction
-def _network_counter_devices(stats: Mapping[str, Any]) -> dict[str, list[int]] | None:
+def _network_counter_devices(
+ stats: Mapping[str, Any], *, role: str | None = None,
+) -> dict[str, list[int]] | None:
"""Return validated cumulative counters keyed by stable guest NIC identity.
libvirt reports rx/tx from the guest's point of view: rx is downloaded by
@@ -99,20 +101,52 @@ def _network_counter_devices(stats: Mapping[str, Any]) -> dict[str, list[int]] |
if count is None or count > 1024:
return None
candidates: dict[str, list[int] | None] = {}
+ seen_identities: set[str] = set()
+ # A role-filtered frame is safe to use for billing only when every NIC
+ # declares a role. The one-NIC legacy format has no role field and is
+ # still unambiguous; a multi-NIC frame without roles is not.
+ explicit_role_keys = [f"net.{number}.role" in stats for number in range(count)]
+ has_explicit_roles = any(explicit_role_keys)
+ if role is not None and not has_explicit_roles and count not in {0, 1}:
+ return None
for number in range(count):
name = stats.get(f"net.{number}.name")
identity = stats.get(f"net.{number}.identity", name)
+ observed_role = stats.get(f"net.{number}.role")
+ if (not isinstance(identity, str) or not 1 <= len(identity) <= 255
+ or any(ord(char) < 32 or ord(char) == 127 for char in identity)):
+ return None
+ if identity in seen_identities:
+ return None
+ seen_identities.add(identity)
+ if role is not None:
+ if has_explicit_roles:
+ if observed_role not in {"public", "private", "unknown"}:
+ # A partially labelled multi-NIC frame cannot establish
+ # which interface belongs to the public quota lane.
+ return None
+ if observed_role == "unknown":
+ return None
+ if observed_role != role:
+ continue
+ elif observed_role is not None and observed_role != role:
+ continue
rx = _nonnegative_int(stats.get(f"net.{number}.rx.bytes"))
tx = _nonnegative_int(stats.get(f"net.{number}.tx.bytes"))
- if (not isinstance(identity, str) or not 1 <= len(identity) <= 255
- or any(ord(char) < 32 or ord(char) == 127 for char in identity)
- or rx is None or tx is None
+ if (rx is None or tx is None
or rx > _MAX_UNSIGNED_BIGINT or tx > _MAX_UNSIGNED_BIGINT):
return None
- # Duplicate names cannot be safely matched to a previous observation.
+ # Duplicate identities were rejected above, including across
+ # selected and unselected roles, so a bad private NIC cannot alias a
+ # billable public NIC.
if identity in candidates:
return None
candidates[identity] = [rx, tx]
+ if role is None:
+ return {identity: counters for identity, counters in candidates.items()
+ if counters is not None}
+ # ``candidates`` contains only the requested role when explicit roles are
+ # present; for the legacy one-NIC path it contains the sole public NIC.
return {identity: counters for identity, counters in candidates.items()
if counters is not None}
@@ -210,6 +244,13 @@ class MetricSampler:
# totals here. The database store locks and advances its durable
# baseline in the same transaction as this sample.
"network_devices": _network_counter_devices(values),
+ # Once explicit public0/private0 roles are available, quota
+ # accounting must not charge east-west VPC traffic as public
+ # traffic. Legacy frames without role metadata continue through
+ # the historical fallback in DatabaseStore.
+ "network_billing_devices": _network_counter_devices(
+ values, role="public"
+ ),
}
self._previous[key] = _Baseline(monotonic, raw.generation, raw.vcpus, cpu_time,
disk_signature, disk, network_signature, network)
diff --git a/engine/virsh_panel/private_network.py b/engine/virsh_panel/private_network.py
new file mode 100644
index 00000000..13d09d0d
--- /dev/null
+++ b/engine/virsh_panel/private_network.py
@@ -0,0 +1,202 @@
+"""Bounded observation of a VM's private network role.
+
+This module deliberately only reads the domain definition. It does not run
+``ip``, ``wg`` or ``nft`` commands and it never treats the host Fabric status
+as proof that a tenant NIC is connected. The PHP control plane supplies the
+VPC/member revision when it evaluates this observation at an operation
+commit boundary.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+import time
+import xml.etree.ElementTree as ET
+from typing import Any
+
+from virsh_panel.network_roles import (
+ PRIVATE_INTERFACE_ALIAS,
+ interface_role,
+ private_interface_has_public_filter,
+ select_private_interface,
+ select_public_interface,
+)
+
+
+PRIVATE_STATE_VERSION = 1
+PANEL_NS = "urn:virsh-panel:1"
+_MAC_RE = re.compile(r"[0-9a-f]{2}(?::[0-9a-f]{2}){5}\Z")
+_DIGEST_RE = re.compile(r"[0-9a-f]{64}\Z")
+
+
+def _source(interface: ET.Element) -> tuple[str | None, str | None]:
+ source = interface.find("source")
+ if source is None:
+ return None, None
+ for kind in ("bridge", "network", "dev", "name", "path"):
+ value = source.get(kind)
+ if isinstance(value, str) and value.strip():
+ return kind, value.strip()
+ return None, None
+
+
+def _managed_metadata(root: ET.Element) -> ET.Element | None:
+ # Only the panel-owned namespace is trusted for the optional identity
+ # metadata. Guest/user XML must never become a host-state assertion.
+ for node in root.findall("./metadata"):
+ for child in list(node):
+ if child.tag == f"{{{PANEL_NS}}}managed":
+ return child
+ return None
+
+
+def _private_ip(root: ET.Element, managed: ET.Element | None) -> str | None:
+ values: list[str | None] = []
+ if managed is not None:
+ values.extend((managed.get("private-ip-cidr"), managed.get("private_ip_cidr")))
+ for node in root.findall("./metadata/*"):
+ if node.tag.rsplit("}", 1)[-1] in {"private-network", "private_state"}:
+ values.extend((node.get("ip-cidr"), node.get("private-ip-cidr"), node.get("cidr")))
+ for value in values:
+ if not isinstance(value, str) or not value:
+ continue
+ try:
+ address = ipaddress.ip_interface(value)
+ except ValueError:
+ continue
+ if isinstance(address, ipaddress.IPv4Interface) and address.network.prefixlen == 32:
+ return str(address)
+ return None
+
+
+def _revision(managed: ET.Element | None) -> int:
+ if managed is None:
+ return 0
+ raw = managed.get("private-network-revision") or managed.get("private_revision")
+ try:
+ value = int(raw or 0)
+ except (TypeError, ValueError):
+ return 0
+ return value if 0 <= value <= (1 << 63) - 1 else 0
+
+
+def _digest(managed: ET.Element | None) -> str | None:
+ if managed is None:
+ return None
+ value = managed.get("private-network-digest") or managed.get("private_digest")
+ return value.lower() if isinstance(value, str) and _DIGEST_RE.fullmatch(value.lower()) else None
+
+
+def observe_private_state(
+ root: ET.Element,
+ *,
+ vm_uuid: str | None = None,
+ host_verified: bool = False,
+ observed_at: int | None = None,
+) -> dict[str, Any]:
+ """Return a stable, credential-free private NIC observation.
+
+ ``disabled/confirmed`` means that the domain has no private role and no
+ VPC expectation was supplied to this read. A caller that has a VPC
+ expectation must independently change that result to ``degraded`` or
+ ``uncertain`` until the expected member/revision/digest is verified.
+ """
+ now = int(time.time()) if observed_at is None else int(observed_at)
+ base: dict[str, Any] = {
+ "version": PRIVATE_STATE_VERSION,
+ "required": False,
+ "vm_uuid": vm_uuid,
+ "vpc_id": None,
+ "member_id": None,
+ "interface": None,
+ "state": "disabled",
+ "desired_revision": 0,
+ "observed_revision": 0,
+ "desired_digest": None,
+ "observed_digest": None,
+ "readback": "confirmed",
+ "link_state": "absent",
+ "private_ip_cidr": None,
+ "private_mac": None,
+ "reason_code": None,
+ "observed_at": now,
+ }
+ interfaces = list(root.findall("./devices/interface"))
+ # Validate the complete role map even when no private NIC is present. A
+ # second unlabelled/ambiguous NIC must not disappear behind a successful
+ # public-only observation and later become an accidental data-plane path.
+ try:
+ public = select_public_interface(interfaces)
+ private = select_private_interface(interfaces)
+ except (TypeError, ValueError):
+ base.update(state="degraded", readback="mismatch",
+ reason_code="private_interface_role_invalid")
+ return base
+ if private is None:
+ return base
+
+ base["interface"] = PRIVATE_INTERFACE_ALIAS
+ managed = _managed_metadata(root)
+ base["private_ip_cidr"] = _private_ip(root, managed)
+ base["observed_revision"] = _revision(managed)
+ base["observed_digest"] = _digest(managed)
+ mac_node = private.find("mac")
+ mac = mac_node.get("address", "").lower() if mac_node is not None else ""
+ base["private_mac"] = mac or None
+ link = private.find("link")
+ link_state = link.get("state", "up").lower() if link is not None else "up"
+ base["link_state"] = link_state if link_state in {"up", "down"} else "absent"
+
+ private_kind, private_source = _source(private)
+ public_kind, public_source = _source(public) if public is not None else (None, None)
+ # The tenant data plane is an independent host bridge. A libvirt
+ # ``network``/``direct`` source does not let this layer prove which host
+ # bridge it resolves to, so accepting it would make a public bridge reuse
+ # indistinguishable from a private network. The future data-plane apply
+ # path may add an explicit verification marker to the managed metadata.
+ if (private_kind != "bridge" or private_source is None
+ or private_kind == public_kind and private_source == public_source):
+ base.update(state="degraded", readback="mismatch",
+ reason_code="private_source_invalid")
+ return base
+ if private_interface_has_public_filter(private):
+ base.update(state="degraded", readback="mismatch",
+ reason_code="private_filter_present")
+ return base
+ if not _MAC_RE.fullmatch(mac):
+ base.update(state="degraded", readback="mismatch",
+ reason_code="private_mac_missing")
+ return base
+ if base["private_ip_cidr"] is None:
+ base.update(state="degraded", readback="missing",
+ reason_code="private_ip_missing")
+ return base
+ # XML metadata is tenant/domain data and can be edited independently of
+ # the host. Only a future host adapter probe may set ``host_verified``;
+ # never use a managed-XML marker as proof of bridge/WireGuard/nft state.
+ if not host_verified:
+ base.update(state="degraded", readback="missing",
+ reason_code="private_source_unverified")
+ return base
+ if link_state not in {"up", "down"}:
+ base.update(state="degraded", readback="mismatch",
+ reason_code="private_link_state_invalid")
+ return base
+ base["state"] = "suspended" if link_state == "down" else "active"
+ return base
+
+
+def observe_private_state_from_xml(
+ xml: str | bytes,
+ *,
+ vm_uuid: str | None = None,
+ host_verified: bool = False,
+ observed_at: int | None = None,
+) -> dict[str, Any]:
+ """Parse a libvirt XML document and return :func:`observe_private_state`."""
+ root = ET.fromstring(xml)
+ return observe_private_state(
+ root, vm_uuid=vm_uuid, host_verified=host_verified,
+ observed_at=observed_at,
+ )
diff --git a/engine/virsh_panel/vpc.py b/engine/virsh_panel/vpc.py
index 074dd23d..7ba82d6b 100644
--- a/engine/virsh_panel/vpc.py
+++ b/engine/virsh_panel/vpc.py
@@ -851,6 +851,10 @@ class VpcCoordinator:
"observed_digest": old_digest,
"state": "applying" if gate.get("active") else "uncertain",
"transport_verified": bool(gate.get("active")),
+ # Fabric transport is necessary but never sufficient for a
+ # tenant VPC. Only the future host adapter may set this
+ # after private0/route/nft/member readback.
+ "private_dataplane_verified": False,
"readback": safe_readback,
})
return {
@@ -866,6 +870,7 @@ class VpcCoordinator:
"operation_id": plan.operation_id,
"state": state,
"next_stage_allowed": False,
+ "private_dataplane_verified": False,
"fabric_gate": dict(gate),
"hosts": host_records,
"subnets": [item.__dict__ for item in plan.subnets],
@@ -891,7 +896,7 @@ class VpcCoordinator:
"vpc_id", "region", "fabric_id", "fabric_revision", "cidr",
"desired_revision", "desired_digest", "observed_revision",
"observed_digest", "state", "next_stage_allowed",
- "operation_id", "updated_at"):
+ "private_dataplane_verified", "operation_id", "updated_at"):
if key in value:
result[key] = value[key]
gate = value.get("fabric_gate")
@@ -927,7 +932,8 @@ class VpcCoordinator:
item = {key: raw[key] for key in (
"host_id", "member_ids", "desired_revision",
"observed_revision", "observed_digest", "state",
- "transport_verified", "last_error_code") if key in raw}
+ "transport_verified", "private_dataplane_verified",
+ "last_error_code") if key in raw}
readback = raw.get("readback")
if isinstance(readback, Mapping):
item["readback"] = {
@@ -948,12 +954,14 @@ class VpcCoordinator:
host_id = raw["host_id"]
item: dict[str, Any] = {}
for key in ("state", "observed_revision", "observed_digest",
- "transport_verified", "last_error_code"):
+ "transport_verified", "private_dataplane_verified",
+ "last_error_code"):
if key in raw:
item[key] = raw[key]
item["next_stage_allowed"] = (
raw.get("state") == "active"
and raw.get("transport_verified") is True
+ and raw.get("private_dataplane_verified") is True
and raw.get("observed_revision") == value.get("desired_revision")
and raw.get("observed_digest") == value.get("desired_digest")
)
@@ -963,6 +971,9 @@ class VpcCoordinator:
"digest": value.get("observed_digest"),
"state": value.get("state"),
"next_stage_allowed": value.get("next_stage_allowed") is True,
+ "private_dataplane_verified": (
+ value.get("private_dataplane_verified") is True
+ ),
"hosts": observed_hosts,
# PHP consumes this bounded observed projection when deciding
# whether the control-plane result is deliberately waiting for
@@ -1084,6 +1095,7 @@ class VpcCoordinator:
if isinstance(item, dict):
item["state"] = "applying"
item["transport_verified"] = True
+ item["private_dataplane_verified"] = False
hosts = record.get("hosts")
desired_revision = record.get("desired_revision")
desired_digest = record.get("desired_digest")
@@ -1093,25 +1105,31 @@ class VpcCoordinator:
and item.get("observed_revision") == desired_revision
and item.get("observed_digest") == desired_digest
and item.get("transport_verified") is True
+ and item.get("private_dataplane_verified") is True
for item in hosts)
fabric_active = gate.get("active") is True
if fabric_active and all_observed:
record["state"] = "active"
record["next_stage_allowed"] = True
+ record["private_dataplane_verified"] = True
record["observed_revision"] = desired_revision
record["observed_digest"] = desired_digest
elif not fabric_active:
record["state"] = "degraded" if gate.get("available") else "uncertain"
record["next_stage_allowed"] = False
+ record["private_dataplane_verified"] = False
elif hosts and any(item.get("state") == "uncertain" for item in hosts if isinstance(item, Mapping)):
record["state"] = "uncertain"
record["next_stage_allowed"] = False
+ record["private_dataplane_verified"] = False
elif hosts and any(item.get("state") == "degraded" for item in hosts if isinstance(item, Mapping)):
record["state"] = "degraded"
record["next_stage_allowed"] = False
+ record["private_dataplane_verified"] = False
else:
record["state"] = "applying"
record["next_stage_allowed"] = False
+ record["private_dataplane_verified"] = False
record["updated_at"] = int(time.time())
async def apply(self, value: Mapping[str, Any]) -> dict[str, Any]:
@@ -1226,6 +1244,7 @@ class VpcCoordinator:
if raw is None:
host["state"] = "uncertain"
host["transport_verified"] = False
+ host["private_dataplane_verified"] = False
continue
if not isinstance(raw, Mapping):
raise VpcError("VPC observed 宿主记录无效")
@@ -1233,6 +1252,7 @@ class VpcCoordinator:
observed_digest = raw.get("observed_digest")
state = raw.get("state")
transport = raw.get("transport_verified")
+ private_dataplane = raw.get("private_dataplane_verified", False)
if (isinstance(observed_revision, bool)
or not isinstance(observed_revision, int)
or observed_revision < 0):
@@ -1246,10 +1266,22 @@ class VpcCoordinator:
raise VpcError("VPC observed state 无效")
if not isinstance(transport, bool):
raise VpcError("VPC observed transport_verified 无效")
+ if not isinstance(private_dataplane, bool):
+ raise VpcError("VPC observed private_dataplane_verified 无效")
if state == "active" and (transport is not True
or observed_digest is None):
- raise VpcError("VPC active 观测缺少传输或摘要证据", 422,
- "vpc_observation_invalid")
+ raise VpcError(
+ "VPC active 观测缺少传输或摘要证据", 422,
+ "vpc_observation_invalid")
+ # The current Fabric adapter reports transport/readback before
+ # the tenant private data-plane adapter exists. Preserve that
+ # observation as an applying gate instead of rejecting the
+ # whole reconciliation request: transport is useful progress,
+ # but it must never open the VPC or be reported as active.
+ observation_error = raw.get("error_code")
+ if state == "active" and private_dataplane is not True:
+ state = "applying"
+ observation_error = "vpc_private_dataplane_unverified"
readback = raw.get("readback", {})
if not isinstance(readback, Mapping) or len(readback) > 64:
raise VpcError("VPC readback 无效")
@@ -1262,10 +1294,11 @@ class VpcCoordinator:
"observed_revision": observed_revision,
"observed_digest": observed_digest,
"transport_verified": transport,
+ "private_dataplane_verified": private_dataplane,
"readback": safe_readback,
- "last_error_code": raw.get("error_code")
- if isinstance(raw.get("error_code"), str)
- and len(raw.get("error_code")) <= 64 else None})
+ "last_error_code": observation_error
+ if isinstance(observation_error, str)
+ and len(observation_error) <= 64 else None})
gate = self._fabric_gate(plan)
self._finish(record, gate)
self._save("vpc_desired", vpc_id, record)
@@ -1303,6 +1336,7 @@ class VpcCoordinator:
continue
value["state"] = "uncertain"
value["next_stage_allowed"] = False
+ value["private_dataplane_verified"] = False
# A previous Fabric gate may have been active before the process
# stopped. Preserve its revision/evidence for audit, but never
# expose that stale gate as usable after restart.
@@ -1318,7 +1352,8 @@ class VpcCoordinator:
for host in value.get("hosts", []):
if isinstance(host, dict):
host.update({"state": "uncertain",
- "transport_verified": False})
+ "transport_verified": False,
+ "private_dataplane_verified": False})
value["updated_at"] = int(time.time())
self._save("vpc_desired", key, value)
self._save_children(value)
diff --git a/web/app/Services/Cloud/CloudFirewallService.php b/web/app/Services/Cloud/CloudFirewallService.php
index ed8e0d8e..39e95b55 100644
--- a/web/app/Services/Cloud/CloudFirewallService.php
+++ b/web/app/Services/Cloud/CloudFirewallService.php
@@ -227,6 +227,9 @@ final class CloudFirewallService
],
'owner_id' => (int) $locked->user_id,
'can_manage' => true,
+ 'private_expectation' => CloudVpcService::instancePrivateExpectation(
+ (int) $locked->id
+ ),
'firewall_version_id' => (int) $version->id,
'firewall_revision' => $revision,
'firewall_policy_digest' => $policyDigest,
diff --git a/web/app/Services/Cloud/CloudInstancePageService.php b/web/app/Services/Cloud/CloudInstancePageService.php
index a380b9f1..c125461f 100644
--- a/web/app/Services/Cloud/CloudInstancePageService.php
+++ b/web/app/Services/Cloud/CloudInstancePageService.php
@@ -113,6 +113,8 @@ final class CloudInstancePageService
$runtimeHint = in_array($runtimeSync, ['offline', 'unavailable'], true)
? '暂时无法同步' : ($runtimeState === 'unknown' ? '等待宿主机同步' : '');
}
+ $privateState = CloudVpcService::instancePrivateProjection((int) $instance->id);
+ $privateState['vm_uuid'] = (string) $instance->uuid;
return [
'id' => (string) $instance->id,
'api_id' => (string) $instance->id,
@@ -130,6 +132,7 @@ final class CloudInstancePageService
'instance_status' => (string) $instance->status,
'service_status' => (string) $instance->service_status,
'network_status' => (string) $instance->network_status,
+ 'private_state' => $privateState,
'service_expires_at' => $instance->expires_at === null
? null : (int) $instance->expires_at,
'current_period_started_at' => $period === null ? null : (int) $period->starts_at,
@@ -191,11 +194,14 @@ final class CloudInstancePageService
'state_sync_status' => 'pending',
'state_label_hint' => '等待宿主机同步',
];
+ $privateState = CloudVpcService::instancePrivateProjection((int) $instance->id);
+ $privateState['vm_uuid'] = (string) $instance->uuid;
$vms[] = [
'id' => (string) $instance->id,
'api_id' => (string) $instance->id,
'uuid' => (string) $instance->uuid,
'name' => (string) $instance->name,
+ 'private_state' => $privateState,
] + array_intersect_key($cached, array_flip([
'state', 'state_transitioning', 'state_transition',
'state_fresh', 'state_observed_at',
@@ -248,6 +254,10 @@ final class CloudInstancePageService
'instance_status' => (string) $snapshot['instance_status'],
'service_status' => (string) $snapshot['service_status'],
'network_status' => (string) $snapshot['network_status'],
+ 'private_state' => is_array($snapshot['private_state'] ?? null)
+ ? $snapshot['private_state'] : CloudVpcService::instancePrivateProjection(
+ (int) $snapshot['id']
+ ),
'service_expires_at' => $snapshot['service_expires_at'],
'current_period_started_at' => $snapshot['current_period_started_at'],
'current_period_ends_at' => $snapshot['current_period_ends_at'],
@@ -293,6 +303,10 @@ final class CloudInstancePageService
'state_sync_status' => $liveAvailable ? 'fresh'
: (string) ($snapshot['state_sync_status'] ?? 'pending'),
'state_label_hint' => (string) ($snapshot['state_label_hint'] ?? ''),
+ 'private_state' => is_array($snapshot['private_state'] ?? null)
+ ? $snapshot['private_state'] : CloudVpcService::instancePrivateProjection(
+ (int) $snapshot['id']
+ ),
];
}
@@ -304,6 +318,7 @@ final class CloudInstancePageService
'state_transitioning', 'state_transition', 'state_fresh',
'state_observed_at', 'state_sync_status',
'state_label_hint', 'instance_status',
+ 'private_state',
'os_source', 'os_name', 'install_iso', 'public_ip_cidr', 'public_ip',
'login_name', 'engine_metadata_version', 'vcpu', 'cpu_cores',
'memory_mib', 'ram_mib', 'disks', 'disk_gib',
@@ -339,6 +354,8 @@ final class CloudInstancePageService
'state_sync_status' => 'pending',
'state_label_hint' => '等待宿主机同步',
];
+ $privateState = CloudVpcService::instancePrivateProjection((int) $instance->id);
+ $privateState['vm_uuid'] = (string) $instance->uuid;
$result[] = self::listItem(array_merge($spec, [
'id' => (string) $instance->id,
'api_id' => (string) $instance->id,
@@ -346,6 +363,7 @@ final class CloudInstancePageService
'name' => (string) $instance->name,
'instance_status' => (string) $instance->status,
'region' => $region,
+ 'private_state' => $privateState,
], $cached));
}
return $result;
@@ -504,6 +522,8 @@ final class CloudInstancePageService
$runtimeHint = in_array($runtimeSync, ['offline', 'unavailable'], true)
? '暂时无法同步' : ($runtimeState === 'unknown' ? '等待宿主机同步' : '');
}
+ $privateState = CloudVpcService::instancePrivateProjection((int) $instance->id);
+ $privateState['vm_uuid'] = (string) $instance->uuid;
$maximumFuturePeriods = max(1, min(120,
(int) Config::get('cloud_max_renewal_periods_ahead', 24)));
@@ -532,6 +552,7 @@ final class CloudInstancePageService
'instance_status' => (string) $instance->status,
'service_status' => (string) $instance->service_status,
'network_status' => (string) $instance->network_status,
+ 'private_state' => $privateState,
'service_expires_at' => $instance->expires_at === null
? null : (int) $instance->expires_at,
'current_period_started_at' => $period === null ? null : (int) $period->starts_at,
diff --git a/web/app/Services/Cloud/CloudJobService.php b/web/app/Services/Cloud/CloudJobService.php
index dc67abb5..6115f51e 100644
--- a/web/app/Services/Cloud/CloudJobService.php
+++ b/web/app/Services/Cloud/CloudJobService.php
@@ -244,7 +244,14 @@ final class CloudJobService
$jobPayload = ['method' => $method,
'resource' => 'vms/' . rawurlencode($locked->uuid) . $suffix, 'body' => $body,
'credential' => $credential, 'owner_id' => (int) $locked->user_id,
- 'can_manage' => (bool) ($actor['can_manage'] ?? false)];
+ 'can_manage' => (bool) ($actor['can_manage'] ?? false),
+ // Bind the private target to the same immutable job payload
+ // as the public-IP ledger. A later VPC revision must make
+ // this operation uncertain rather than letting its response
+ // satisfy a newer target by accident.
+ 'private_expectation' => CloudVpcService::instancePrivateExpectation(
+ (int) $locked->id
+ )];
if ($networkLedger !== null) {
$source = $networkLedger['source'];
$target = $networkLedger['target'];
@@ -322,6 +329,9 @@ final class CloudJobService
'body' => [],
'owner_id' => (int) $locked->user_id,
'system_cleanup' => true,
+ 'private_expectation' => CloudVpcService::instancePrivateExpectation(
+ (int) $locked->id
+ ),
]);
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
diff --git a/web/app/Services/Cloud/CloudLifecycleService.php b/web/app/Services/Cloud/CloudLifecycleService.php
index b4d6a94f..a8ab21da 100644
--- a/web/app/Services/Cloud/CloudLifecycleService.php
+++ b/web/app/Services/Cloud/CloudLifecycleService.php
@@ -446,6 +446,9 @@ final class CloudLifecycleService
'resource' => 'vms/' . rawurlencode($instance->uuid) . '/' . $suffix,
'body' => $body, 'owner_id' => (int) $instance->user_id,
'system_policy' => true, 'suspend_reason' => $reason,
+ 'private_expectation' => CloudVpcService::instancePrivateExpectation(
+ (int) $instance->id
+ ),
]);
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php
index 5db73028..7c692b13 100644
--- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php
+++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php
@@ -121,6 +121,12 @@ final class CloudOperationErrorCatalog
'firewall_drift_detected' => '检测到防火墙策略漂移,请联系管理员核对',
'firewall_cleanup_pending' => '实例删除结果待核对,防火墙清理状态尚未确认',
'firewall_apply_superseded' => '防火墙请求已被更新的策略替代',
+ 'vpc_private_observation_missing' => '实例私网状态尚未完成回读,请稍后刷新,切勿重复提交',
+ 'vpc_private_observation_mismatch' => '实例私网状态与当前 VPC 目标不一致,正在核对',
+ 'vpc_private_dataplane_unverified' => 'VPC 私网数据面尚未完成核验,请稍后刷新',
+ 'vpc_private_teardown_uncertain' => '实例私网清理状态尚未确认,请勿重复提交',
+ 'vpc_private_expectation_invalid' => '实例私网目标记录无效,请联系管理员核对',
+ 'vpc_private_expectation_unavailable' => '实例私网目标暂时无法读取,请稍后核对',
'admin_cancelled' => '管理员已中止后续重试;已派发的宿主机操作仍需核查',
];
@@ -154,6 +160,7 @@ final class CloudOperationErrorCatalog
'created', 'restored', 'deleted',
'service_was_running', 'service_was_running_hint', 'vm',
'reinstall_committed', 'cleanup_pending',
+ 'private_state',
'vcpu', 'cpu_cores', 'memory_mib', 'ram_mib', 'disk_gib',
'os_source', 'os_name', 'install_iso', 'login_name',
'autostart', 'managed',
@@ -317,6 +324,13 @@ final class CloudOperationErrorCatalog
}
return 'firewall_apply_failed';
}
+ if (in_array($engineCode, [
+ 'vpc_private_observation_missing', 'vpc_private_observation_mismatch',
+ 'vpc_private_teardown_uncertain', 'vpc_private_expectation_invalid',
+ 'vpc_private_expectation_unavailable', 'vpc_private_dataplane_unverified',
+ ], true)) {
+ return $engineCode;
+ }
// A worker may persist only a reviewed detail or a stable generic
// message. Never put an arbitrary engine/admin summary into the
@@ -542,6 +556,15 @@ final class CloudOperationErrorCatalog
}
continue;
}
+ if ($name === 'private_state') {
+ if (is_array($item) && !array_is_list($item)) {
+ $safePrivate = self::sanitizePrivateState($item);
+ if ($safePrivate !== null) {
+ $result[$name] = $safePrivate;
+ }
+ }
+ continue;
+ }
$safe = self::safeResultScalar($name, $item);
if ($safe !== null) {
$result[$name] = $safe;
@@ -550,6 +573,81 @@ final class CloudOperationErrorCatalog
return $result;
}
+ /** Keep private-state evidence typed and free of host/XML diagnostics. */
+ private static function sanitizePrivateState(array $value): ?array
+ {
+ $result = [];
+ $result['version'] = (int) ($value['version'] ?? 1);
+ if ($result['version'] !== 1) {
+ return null;
+ }
+ $result['required'] = ($value['required'] ?? false) === true;
+ foreach (['vm_uuid', 'vpc_id', 'member_id', 'interface',
+ 'private_ip_cidr', 'private_mac', 'desired_digest', 'observed_digest'] as $key) {
+ $item = $value[$key] ?? null;
+ $result[$key] = is_string($item) && $item !== ''
+ ? self::normalize($item) : null;
+ }
+ foreach (['state' => ['disabled', 'active', 'suspended', 'removed', 'degraded', 'uncertain'],
+ 'desired_state' => ['disabled', 'active', 'removed'],
+ 'readback' => ['confirmed', 'missing', 'mismatch'],
+ 'link_state' => ['up', 'down', 'absent']] as $key => $allowed) {
+ $default = match ($key) {
+ 'desired_state' => $result['required'] ? 'active' : 'disabled',
+ 'readback' => 'missing',
+ 'link_state' => 'absent',
+ default => 'uncertain',
+ };
+ $item = is_string($value[$key] ?? null)
+ ? strtolower(trim((string) $value[$key])) : $default;
+ $result[$key] = in_array($item, $allowed, true) ? $item : $default;
+ }
+ $result['desired_revision'] = max(0, (int) ($value['desired_revision'] ?? 0));
+ $result['observed_revision'] = max(0, (int) ($value['observed_revision'] ?? 0));
+ $result['observed_at'] = max(0, (int) ($value['observed_at'] ?? 0));
+ foreach (['desired_digest', 'observed_digest'] as $key) {
+ if ($result[$key] !== null
+ && preg_match('/^[a-f0-9]{64}$/Di', $result[$key]) !== 1) {
+ $result[$key] = null;
+ }
+ }
+ if ($result['vm_uuid'] !== null && preg_match(
+ '/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD',
+ $result['vm_uuid']) !== 1) {
+ $result['vm_uuid'] = null;
+ }
+ foreach (['vpc_id', 'member_id'] as $key) {
+ if ($result[$key] !== null && preg_match('/^[0-9a-f]{32}$/Di', $result[$key]) !== 1) {
+ $result[$key] = null;
+ }
+ }
+ if ($result['interface'] !== null && $result['interface'] !== 'private0') {
+ $result['interface'] = null;
+ }
+ if ($result['private_ip_cidr'] !== null) {
+ $parts = explode('/', $result['private_ip_cidr'], 2);
+ if (count($parts) !== 2 || (int) $parts[1] !== 32
+ || filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false) {
+ $result['private_ip_cidr'] = null;
+ }
+ }
+ if ($result['private_mac'] !== null
+ && preg_match('/^[0-9a-f]{2}(?::[0-9a-f]{2}){5}$/Di', $result['private_mac']) !== 1) {
+ $result['private_mac'] = null;
+ }
+ $reason = is_string($value['reason_code'] ?? null)
+ ? strtolower(trim((string) $value['reason_code'])) : null;
+ $result['reason_code'] = in_array($reason, [
+ 'private_interface_role_invalid', 'private_source_invalid',
+ 'private_filter_present', 'private_mac_missing', 'private_ip_missing',
+ 'private_source_unverified', 'vpc_private_observation_missing',
+ 'vpc_private_observation_mismatch', 'vpc_private_teardown_uncertain',
+ 'vpc_private_dataplane_unverified',
+ 'vpc_private_expectation_invalid', 'vpc_private_expectation_unavailable',
+ ], true) ? $reason : null;
+ return $result;
+ }
+
private static function safeResultScalar(string $key, $value)
{
if (is_bool($value)) {
diff --git a/web/app/Services/Cloud/CloudOrderService.php b/web/app/Services/Cloud/CloudOrderService.php
index 009cfe24..47342915 100644
--- a/web/app/Services/Cloud/CloudOrderService.php
+++ b/web/app/Services/Cloud/CloudOrderService.php
@@ -240,7 +240,14 @@ final class CloudOrderService
$job->host_id = $placement['host']->id;
$job->kind = 'create';
$job->status = 'queued';
- $job->payload_cipher = CloudSecrets::encrypt($placement['payload']);
+ // A create has no instance row yet, so this is normally the explicit
+ // disabled target. Persist it so a future flow that pre-creates or
+ // attaches a VPC member can use the same immutable-target protocol
+ // instead of publishing a public-only success without a target.
+ $createPayload = $placement['payload'];
+ $createPayload['private_expectation'] =
+ CloudVpcService::instancePrivateExpectation(null);
+ $job->payload_cipher = CloudSecrets::encrypt($createPayload);
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
// The worker now owns the sole recoverable credential copy.
diff --git a/web/app/Services/Cloud/CloudSnapshotService.php b/web/app/Services/Cloud/CloudSnapshotService.php
index 5a95f4b8..55b6cb72 100644
--- a/web/app/Services/Cloud/CloudSnapshotService.php
+++ b/web/app/Services/Cloud/CloudSnapshotService.php
@@ -399,6 +399,13 @@ final class CloudSnapshotService
'can_manage' => (bool) ($actor['can_manage'] ?? false),
'snapshot_key' => (string) $snapshot->snapshot_key,
'snapshot_previous_state' => (string) $snapshot->state,
+ // Snapshot work changes storage, not the tenant network, but it
+ // still crosses the same instance commit boundary. Capture the
+ // private target so a VPC revision change while the snapshot is
+ // running cannot be mistaken for a readback of the newer target.
+ 'private_expectation' => CloudVpcService::instancePrivateExpectation(
+ (int) $instance->id
+ ),
]);
$job->available_at = $job->created_at = $job->updated_at = $now;
$job->save();
diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php
index d368ee7a..eae56b2a 100644
--- a/web/app/Services/Cloud/CloudVpcService.php
+++ b/web/app/Services/Cloud/CloudVpcService.php
@@ -563,6 +563,14 @@ final class CloudVpcService
->exists()) {
throw new CloudError('该成员仍被路由引用,请先替换或删除路由', 409);
}
+ // Lifecycle commits lock VPC -> instance before validating the
+ // captured private target. Take the same order here so a detach
+ // cannot flip the member to deleting while a VM job is committing.
+ $instance = CloudInstance::where('id', (int) $member->instance_id)
+ ->lockForUpdate()->first();
+ if ($instance === null || (string) $instance->status === 'deleted') {
+ throw new CloudError('虚拟机不存在', 404);
+ }
$privateIp = self::lockedPrivateIpForMember($member);
$member->state = 'deleting';
$member->updated_at = time();
@@ -940,8 +948,11 @@ final class CloudVpcService
?? $observed['observed_revision'] ?? 0);
$observedDigest = (string) ($observed['digest']
?? $observed['observed_digest'] ?? '');
+ $privateDataplaneVerified = ($observed['private_dataplane_verified']
+ ?? false) === true;
$globalActive = ($observed['state'] ?? null) === 'active'
&& ($observed['next_stage_allowed'] ?? false) === true
+ && $privateDataplaneVerified
&& $observedRevision === $revision
&& hash_equals($digest, $observedDigest);
$fabricGate = is_array($observed['fabric_gate'] ?? null)
@@ -1029,6 +1040,7 @@ final class CloudVpcService
$itemDigest = (string) ($item['observed_digest'] ?? $item['digest'] ?? '');
$active = $globalActive && ($item['state'] ?? null) === 'active'
&& ($item['transport_verified'] ?? false) === true
+ && ($item['private_dataplane_verified'] ?? false) === true
&& $itemRevision <= $revision
&& $itemRevision === $revision && hash_equals($digest, $itemDigest);
$deleting = (string) $deployment->state === 'deleting';
@@ -1058,6 +1070,16 @@ final class CloudVpcService
// remain untouched until the later data-plane readback phase.
if ($allActive) {
$observedAt = time();
+ // Persist a proof marker on each member only after the full
+ // host-scoped private data-plane gate has passed. Older rows
+ // without this marker remain fail-closed in instance
+ // projections until a fresh observation is recorded.
+ CloudVpcMember::where('vpc_id', $vpc->id)
+ ->whereNotIn('state', ['deleted'])
+ ->update([
+ 'last_operation_id' => $operationId,
+ 'updated_at' => $observedAt,
+ ]);
foreach ([
CloudVpcSubnet::class,
CloudVpcPrivateIp::class,
@@ -1263,6 +1285,595 @@ final class CloudVpcService
}
}
+ /**
+ * Return the private-network expectation that belongs to one VM job.
+ *
+ * This deliberately reads the existing VPC ledger instead of copying a
+ * second target into cloud_instances/spec_json. A missing VPC schema is
+ * treated as the legacy "no private network" deployment; once the ledger
+ * exists, malformed rows are an explicit uncertain expectation and cannot
+ * be silently interpreted as an attached network.
+ */
+ public static function instancePrivateExpectation(?int $instanceId): array
+ {
+ $disabled = [
+ 'required' => false, 'state' => 'disabled', 'desired_state' => 'disabled',
+ 'vpc_id' => null, 'member_id' => null, 'private_ip_cidr' => null,
+ 'private_mac' => null, 'desired_revision' => 0, 'desired_digest' => null,
+ 'reason_code' => null,
+ ];
+ if ($instanceId === null || $instanceId < 1) {
+ return $disabled;
+ }
+ try {
+ $memberConnection = (new CloudVpcMember())->getConnection();
+ $schema = $memberConnection->getSchemaBuilder();
+ if (!$schema->hasTable('cloud_vpc_members')) {
+ return $disabled;
+ }
+ if (!$schema->hasTable('cloud_vpc_private_ips')
+ || !$schema->hasTable('cloud_vpcs')) {
+ return [
+ 'required' => true, 'state' => 'uncertain',
+ 'desired_state' => 'active', 'vpc_id' => null,
+ 'member_id' => null, 'private_ip_cidr' => null,
+ 'private_mac' => null, 'desired_revision' => 0,
+ 'desired_digest' => null,
+ 'reason_code' => 'vpc_private_expectation_invalid',
+ ];
+ }
+ $member = CloudVpcMember::where('instance_id', $instanceId)
+ ->whereNotIn('state', ['deleted'])->orderByDesc('id')->first();
+ if ($member === null) {
+ // A private-IP lease without a live member is still a
+ // control-plane residue. Treat it as an uncertain target so
+ // lifecycle workers cannot publish a successful/public-only
+ // result while the orphan may remain attached on a host.
+ $orphanLease = CloudVpcPrivateIp::where('instance_id', $instanceId)
+ ->whereNotIn('state', ['deleted'])->exists();
+ if ($orphanLease) {
+ return [
+ 'required' => true, 'state' => 'uncertain',
+ 'desired_state' => 'active', 'vpc_id' => null,
+ 'member_id' => null, 'private_ip_cidr' => null,
+ 'private_mac' => null, 'desired_revision' => 0,
+ 'desired_digest' => null,
+ 'reason_code' => 'vpc_private_expectation_invalid',
+ ];
+ }
+ return $disabled;
+ }
+ $vpc = CloudVpc::find((int) $member->vpc_id);
+ $privateIp = $member->private_ip_id === null ? null
+ : CloudVpcPrivateIp::where('id', (int) $member->private_ip_id)->first();
+ $state = (string) $member->state;
+ $desiredState = $state === 'deleting' ? 'removed' : 'active';
+ // The member and its lease are one desired-state tuple. Falling
+ // back to the root revision/digest can hide a partially-written
+ // child row and would let a VM job capture a target that the VPC
+ // document never actually requested.
+ $revision = (int) ($member->desired_revision ?? 0);
+ $digest = self::safeDigest($member->desired_digest);
+ $rootRevision = (int) ($vpc?->desired_revision ?? 0);
+ $rootDigest = self::safeDigest($vpc?->desired_digest);
+ $leaseRevision = (int) ($privateIp?->desired_revision ?? 0);
+ $leaseDigest = self::safeDigest($privateIp?->desired_digest);
+ $validIp = is_string($member->private_ip)
+ && filter_var($member->private_ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false;
+ // `deleting` is an in-flight lease state, not proof that the
+ // dataplane has converged. Keep the expectation uncertain until
+ // the row is removed (and a fresh member is observed).
+ $leaseState = (string) ($privateIp?->state ?? '');
+ $stateMatches = $privateIp !== null && $leaseState === $state;
+ $ledgerMatches = $privateIp !== null
+ && (int) $privateIp->vpc_id === (int) $member->vpc_id
+ && (int) $privateIp->subnet_id === (int) $member->subnet_id
+ && (int) ($privateIp->member_id ?? 0) === (int) $member->id
+ && (int) ($privateIp->instance_id ?? 0) === $instanceId
+ && (string) $privateIp->private_ip === (string) $member->private_ip
+ && $stateMatches
+ && !in_array($leaseState, ['deleted', 'deleting'], true);
+ $validRevision = $revision > 0;
+ $validDigest = $digest !== null;
+ $desiredLedgerMatches = $vpc !== null
+ && $rootRevision === $revision
+ && $rootDigest !== null && $digest !== null
+ && self::digestMatches($rootDigest, $digest)
+ && $leaseRevision === $revision
+ && $leaseDigest !== null
+ && self::digestMatches($digest, $leaseDigest);
+ if (!$validIp || !$ledgerMatches || !$validRevision || !$validDigest
+ || !$desiredLedgerMatches
+ || !is_string($member->member_key)
+ || preg_match(self::KEY_PATTERN, (string) $member->member_key) !== 1
+ || !is_string($vpc?->vpc_key)
+ || preg_match(self::KEY_PATTERN, (string) $vpc->vpc_key) !== 1) {
+ return [
+ 'required' => true, 'state' => 'uncertain',
+ 'desired_state' => $desiredState,
+ 'vpc_id' => is_string($vpc?->vpc_key) ? $vpc->vpc_key : null,
+ 'member_id' => is_string($member->member_key) ? $member->member_key : null,
+ 'private_ip_cidr' => $validIp ? $member->private_ip . '/32' : null,
+ 'private_mac' => null, 'desired_revision' => max(0, $revision),
+ 'desired_digest' => $validDigest ? $digest : null,
+ 'reason_code' => 'vpc_private_expectation_invalid',
+ ];
+ }
+ return [
+ 'required' => true, 'state' => $state,
+ 'desired_state' => $desiredState,
+ 'vpc_id' => (string) $vpc->vpc_key,
+ 'member_id' => (string) $member->member_key,
+ 'private_ip_cidr' => $member->private_ip . '/32',
+ 'private_mac' => null,
+ 'desired_revision' => $revision,
+ 'desired_digest' => $digest,
+ 'reason_code' => null,
+ ];
+ } catch (\Throwable $error) {
+ // A live ledger that cannot be read is different from an
+ // installation that predates VPC tables. Fail closed for the
+ // former so a worker cannot publish a stale successful result.
+ return [
+ 'required' => true, 'state' => 'uncertain',
+ 'desired_state' => 'active', 'vpc_id' => null, 'member_id' => null,
+ 'private_ip_cidr' => null, 'private_mac' => null,
+ 'desired_revision' => 0, 'desired_digest' => null,
+ 'reason_code' => 'vpc_private_expectation_unavailable',
+ ];
+ }
+ }
+
+ /** Compare a job's captured target with the current ledger under a lock. */
+ public static function privateExpectationMatches(int $instanceId, array $captured): bool
+ {
+ if ($instanceId < 1 || $captured === []) {
+ return true;
+ }
+ $current = self::instancePrivateExpectation($instanceId);
+ // `state` is observed/transient (for example applying -> active) and
+ // may advance while a VM lifecycle job is running. The immutable
+ // target is bound by desired_state, identity, IP, revision and digest;
+ // removal remains strict because it changes desired_state to removed.
+ foreach (['required', 'desired_state', 'vpc_id', 'member_id',
+ 'private_ip_cidr', 'desired_revision', 'desired_digest'] as $key) {
+ if ((string) ($captured[$key] ?? null) !== (string) ($current[$key] ?? null)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /** Build a bounded instance projection for page/runtime responses. */
+ public static function instancePrivateProjection(int $instanceId): array
+ {
+ $expectation = self::instancePrivateExpectation($instanceId);
+ $base = [
+ 'version' => 1, 'required' => (bool) $expectation['required'],
+ 'vm_uuid' => null, 'vpc_id' => $expectation['vpc_id'] ?? null,
+ 'member_id' => $expectation['member_id'] ?? null,
+ 'interface' => $expectation['required'] ? 'private0' : null,
+ 'state' => $expectation['required'] ? 'uncertain' : 'disabled',
+ 'desired_state' => $expectation['desired_state'] ?? 'disabled',
+ 'desired_revision' => (int) ($expectation['desired_revision'] ?? 0),
+ 'observed_revision' => 0,
+ 'desired_digest' => $expectation['desired_digest'] ?? null,
+ 'observed_digest' => null, 'readback' => $expectation['required'] ? 'missing' : 'confirmed',
+ 'link_state' => $expectation['required'] ? 'absent' : 'absent',
+ 'private_ip_cidr' => $expectation['private_ip_cidr'] ?? null,
+ 'private_mac' => null,
+ 'reason_code' => $expectation['reason_code']
+ ?? ($expectation['required'] ? 'vpc_private_observation_missing' : null),
+ 'observed_at' => 0,
+ ];
+ if (!$expectation['required']) {
+ return $base;
+ }
+ try {
+ $vpcKey = is_string($expectation['vpc_id'] ?? null)
+ ? (string) $expectation['vpc_id'] : '';
+ $memberKey = is_string($expectation['member_id'] ?? null)
+ ? (string) $expectation['member_id'] : '';
+ if (preg_match(self::KEY_PATTERN, $vpcKey) !== 1
+ || preg_match(self::KEY_PATTERN, $memberKey) !== 1) {
+ return $base;
+ }
+ // member_key is only unique inside a VPC. Binding the instance
+ // and VPC roots here prevents a stale/cross-tenant row from
+ // becoming a usable private projection.
+ $vpc = CloudVpc::where('vpc_key', $vpcKey)->first([
+ 'id', 'vpc_key', 'state', 'sync_state', 'desired_revision',
+ 'observed_revision', 'desired_digest', 'observed_digest',
+ 'last_operation_id', 'last_success_at', 'updated_at',
+ ]);
+ if ($vpc === null) {
+ return $base;
+ }
+ $member = CloudVpcMember::where('instance_id', $instanceId)
+ ->where('vpc_id', (int) $vpc->id)
+ ->where('member_key', $memberKey)
+ ->whereNotIn('state', ['deleted'])
+ ->first([
+ 'id', 'vpc_id', 'subnet_id', 'private_ip_id', 'instance_id',
+ 'private_ip', 'state', 'desired_revision', 'observed_revision',
+ 'desired_digest', 'observed_digest', 'last_error_code',
+ 'last_operation_id', 'updated_at',
+ ]);
+ if ($member === null || $member->private_ip_id === null) {
+ return $base;
+ }
+ // A member row is not proof that its independent lease exists.
+ // Read back and bind every lease identity before considering the
+ // member's revision or digest.
+ $privateIp = CloudVpcPrivateIp::where('id', (int) $member->private_ip_id)
+ ->where('vpc_id', (int) $vpc->id)
+ ->where('member_id', (int) $member->id)
+ ->where('instance_id', $instanceId)
+ ->first([
+ 'id', 'vpc_id', 'subnet_id', 'member_id', 'instance_id',
+ 'private_ip', 'state', 'desired_revision', 'observed_revision',
+ 'desired_digest', 'observed_digest', 'last_error_code', 'updated_at',
+ ]);
+ if ($privateIp === null
+ || (int) $privateIp->subnet_id !== (int) $member->subnet_id
+ || (string) $privateIp->private_ip !== (string) $member->private_ip) {
+ return $base;
+ }
+
+ $observedRevision = max(0, (int) $member->observed_revision);
+ $observedDigest = self::safeDigest($member->observed_digest);
+ $privateObservedRevision = max(0, (int) $privateIp->observed_revision);
+ $privateObservedDigest = self::safeDigest($privateIp->observed_digest);
+ $privateDesiredRevision = max(0, (int) $privateIp->desired_revision);
+ $privateDesiredDigest = self::safeDigest($privateIp->desired_digest);
+ $rootDesiredRevision = max(0, (int) $vpc->desired_revision);
+ $rootObservedRevision = max(0, (int) $vpc->observed_revision);
+ $rootDesiredDigest = self::safeDigest($vpc->desired_digest);
+ $rootObservedDigest = self::safeDigest($vpc->observed_digest);
+ $rootOperationId = (string) ($vpc->last_operation_id ?? '');
+ // A VM job operation ID is not a VPC activation proof. The
+ // marker must be the current root operation that advanced the
+ // all-host VPC gate, and that operation must have succeeded.
+ $proofMarker = preg_match(self::KEY_PATTERN, $rootOperationId) === 1
+ && hash_equals($rootOperationId,
+ (string) ($member->last_operation_id ?? ''));
+ $operation = $proofMarker
+ ? CloudVpcOperation::where('vpc_id', (int) $vpc->id)
+ ->where('operation_id', $rootOperationId)
+ ->where('status', 'succeeded')
+ ->first(['operation_id', 'desired_revision', 'created_at', 'finished_at'])
+ : null;
+ $operationFresh = $operation !== null
+ && (int) $operation->desired_revision === (int) $base['desired_revision']
+ && (int) ($operation->finished_at ?? 0) > 0
+ && (int) ($operation->finished_at ?? 0)
+ >= (int) ($operation->created_at ?? 0)
+ && (int) ($vpc->last_success_at ?? 0) > 0
+ && (int) ($vpc->last_success_at ?? 0)
+ >= (int) ($operation->created_at ?? 0)
+ && (int) ($member->updated_at ?? 0) > 0
+ && (int) ($privateIp->updated_at ?? 0) > 0
+ && (int) ($member->updated_at ?? 0) >= (int) ($operation->created_at ?? 0)
+ && (int) ($privateIp->updated_at ?? 0) >= (int) ($operation->created_at ?? 0);
+ $rootExact = (string) $vpc->state === 'active'
+ && (string) $vpc->sync_state === 'applied'
+ && $rootDesiredRevision === (int) $base['desired_revision']
+ && $rootObservedRevision === $rootDesiredRevision
+ && $rootDesiredDigest !== null && $rootObservedDigest !== null
+ && self::digestMatches($rootDesiredDigest, $rootObservedDigest)
+ && self::digestMatches((string) $base['desired_digest'], $rootDesiredDigest);
+ $memberExact = (string) $member->state === 'active'
+ && $observedRevision === (int) $base['desired_revision']
+ && $observedDigest !== null
+ && self::digestMatches((string) $base['desired_digest'], $observedDigest)
+ && (int) $member->desired_revision === (int) $base['desired_revision']
+ && self::digestMatches((string) $base['desired_digest'],
+ (string) $member->desired_digest);
+ $privateExact = (string) $privateIp->state === 'active'
+ && $privateDesiredRevision === (int) $base['desired_revision']
+ && $privateObservedRevision === $privateDesiredRevision
+ && $privateDesiredDigest !== null
+ && $privateObservedDigest !== null
+ && self::digestMatches((string) $base['desired_digest'], $privateDesiredDigest)
+ && self::digestMatches($privateDesiredDigest, $privateObservedDigest);
+ $confirmed = $proofMarker && $operationFresh && $rootExact
+ && $memberExact && $privateExact;
+ $base['observed_revision'] = $observedRevision;
+ $base['observed_digest'] = $observedDigest;
+ $base['state'] = $confirmed ? 'active'
+ : ((string) $member->state === 'degraded'
+ || (string) $privateIp->state === 'degraded'
+ ? 'degraded' : 'uncertain');
+ $base['readback'] = $confirmed ? 'confirmed' : 'missing';
+ $base['reason_code'] = $confirmed ? null
+ : (self::safeErrorCode((string) ($privateIp->last_error_code
+ ?? $member->last_error_code ?? ''))
+ ?? 'vpc_private_observation_missing');
+ $freshness = array_values(array_filter([
+ (int) ($member->updated_at ?? 0),
+ (int) ($privateIp->updated_at ?? 0),
+ (int) ($vpc->last_success_at ?? 0),
+ ], static fn (int $value): bool => $value > 0));
+ $base['observed_at'] = $freshness === [] ? 0 : min($freshness);
+ if ($confirmed) {
+ // The member ledger proves the VPC revision, while the
+ // instance network policy records whether the last gated
+ // operation intentionally disconnected all NICs. Pending or
+ // failed policy transitions must remain uncertain until the
+ // next engine readback, even when the previous member digest
+ // was exact.
+ $networkStatus = (string) (CloudInstance::where('id', $instanceId)
+ ->value('network_status') ?? '');
+ if (in_array($networkStatus,
+ ['blocking', 'blocking_expired', 'restore_pending', 'restoring'], true)) {
+ $base['state'] = 'uncertain';
+ $base['readback'] = 'missing';
+ $base['reason_code'] = 'vpc_private_observation_missing';
+ } elseif (in_array($networkStatus,
+ ['blocked_traffic', 'blocked_expired'], true)) {
+ $base['state'] = 'suspended';
+ $base['link_state'] = 'down';
+ } else {
+ $base['link_state'] = 'up';
+ }
+ }
+ } catch (\Throwable) {
+ // Keep the fail-closed default projection.
+ }
+ return $base;
+ }
+
+ /**
+ * Validate one engine private_state at a mutation commit boundary.
+ *
+ * The engine's XML observation is necessary but not sufficient for an
+ * attached VPC: the member identity, desired revision and digest must
+ * also be echoed by a future data-plane reconciler. Until then this
+ * method intentionally returns an uncertain gate for VPC members.
+ */
+ public static function validatePrivateObservation(
+ array $expectation, ?array $observation, string $kind
+ ): array {
+ $fallback = [
+ 'version' => 1, 'required' => (bool) ($expectation['required'] ?? false),
+ 'vm_uuid' => null, 'vpc_id' => $expectation['vpc_id'] ?? null,
+ 'member_id' => $expectation['member_id'] ?? null,
+ 'interface' => null, 'state' => 'uncertain',
+ 'desired_state' => (string) ($expectation['desired_state'] ?? 'disabled'),
+ 'desired_revision' => (int) ($expectation['desired_revision'] ?? 0),
+ 'observed_revision' => 0,
+ 'desired_digest' => $expectation['desired_digest'] ?? null,
+ 'observed_digest' => null, 'readback' => 'missing',
+ 'link_state' => 'absent', 'private_ip_cidr' => null,
+ 'private_mac' => null,
+ 'reason_code' => 'vpc_private_observation_missing',
+ 'observed_at' => time(),
+ ];
+ if (!is_array($observation)) {
+ return ['ok' => false, 'state' => $fallback,
+ 'error_code' => $fallback['reason_code']];
+ }
+ if ((int) ($observation['version'] ?? 0) !== 1) {
+ $fallback['reason_code'] = 'vpc_private_observation_mismatch';
+ return ['ok' => false, 'state' => $fallback,
+ 'error_code' => $fallback['reason_code']];
+ }
+ $state = $fallback;
+ foreach (['version', 'required', 'vm_uuid', 'vpc_id', 'member_id',
+ 'interface', 'state', 'desired_revision', 'observed_revision',
+ 'desired_digest', 'observed_digest', 'readback', 'link_state',
+ 'private_ip_cidr', 'private_mac', 'reason_code', 'observed_at'] as $key) {
+ if (array_key_exists($key, $observation)) {
+ $state[$key] = $observation[$key];
+ }
+ }
+ $state['required'] = (bool) $state['required'];
+ $state['desired_revision'] = max(0, (int) $state['desired_revision']);
+ $state['observed_revision'] = max(0, (int) $state['observed_revision']);
+ $state['desired_digest'] = self::safeDigest($state['desired_digest']);
+ $state['observed_digest'] = self::safeDigest($state['observed_digest']);
+ $expectedRequired = (bool) ($expectation['required'] ?? false);
+ $expectedState = (string) ($expectation['desired_state'] ?? 'disabled');
+ $state['required'] = $expectedRequired;
+ $state['desired_state'] = $expectedState;
+ $observedVpcId = is_string($observation['vpc_id'] ?? null)
+ ? (string) $observation['vpc_id'] : null;
+ $observedMemberId = is_string($observation['member_id'] ?? null)
+ ? (string) $observation['member_id'] : null;
+ $state['vpc_id'] = $expectation['vpc_id'] ?? null;
+ $state['member_id'] = $expectation['member_id'] ?? null;
+ $state['desired_revision'] = (int) ($expectation['desired_revision'] ?? 0);
+ $state['desired_digest'] = self::safeDigest($expectation['desired_digest'] ?? null);
+ $state['private_ip_cidr'] = is_string($observation['private_ip_cidr'] ?? null)
+ ? trim($observation['private_ip_cidr']) : null;
+ $state['private_mac'] = is_string($observation['private_mac'] ?? null)
+ ? strtolower(trim($observation['private_mac'])) : null;
+ $state['state'] = is_string($observation['state'] ?? null)
+ ? strtolower(trim($observation['state'])) : 'uncertain';
+ $state['readback'] = is_string($observation['readback'] ?? null)
+ ? strtolower(trim($observation['readback'])) : 'missing';
+
+ if (!$expectedRequired) {
+ // When a structured observation is available, a legacy VM is
+ // successful only when the host readback proves that no private
+ // NIC exists. An unexpected private interface, even if its XML
+ // looks healthy, is topology drift and must not be hidden behind
+ // a successful public-only operation. The worker compatibility
+ // gate handles older responses that omit private_state entirely.
+ $ok = $state['state'] === 'disabled'
+ && $state['readback'] === 'confirmed'
+ && ($observation['required'] ?? false) === false;
+ if (!$ok) {
+ $state['state'] = 'uncertain';
+ $state['reason_code'] = 'vpc_private_observation_missing';
+ }
+ return ['ok' => $ok, 'state' => $state,
+ 'error_code' => $ok ? null : $state['reason_code']];
+ }
+
+ $linkExpected = match ($kind) {
+ 'network_suspend' => 'down',
+ // service_resume restores the VM power/autostart target. A
+ // quota-blocked VM may still intentionally keep every NIC down;
+ // network_resume is the separate operation that proves link-up.
+ 'network_resume' => 'up',
+ default => null,
+ };
+ $identity = is_string($expectation['vpc_id'] ?? null)
+ && is_string($expectation['member_id'] ?? null)
+ && $observedVpcId !== null && $observedMemberId !== null
+ && hash_equals((string) $expectation['vpc_id'], $observedVpcId)
+ && hash_equals((string) $expectation['member_id'], $observedMemberId);
+ $shape = ($observation['required'] ?? false) === true;
+ if ($expectedState === 'removed') {
+ // Teardown evidence is allowed to omit the interface/MAC only
+ // when it explicitly reports the removed state. Identity,
+ // revision and digest below still bind the tombstone to the
+ // original VPC member.
+ $shape = $shape && in_array($state['state'], ['disabled', 'removed'], true);
+ } else {
+ $shape = $shape && ($state['interface'] ?? null) === 'private0'
+ && is_string($state['private_mac'] ?? null)
+ && preg_match('/^[0-9a-f]{2}(?::[0-9a-f]{2}){5}$/Di',
+ (string) $state['private_mac']) === 1;
+ }
+ $revision = $state['observed_revision'] === $state['desired_revision']
+ && $state['desired_revision'] > 0;
+ $digest = is_string($state['observed_digest'])
+ && is_string($state['desired_digest'])
+ && hash_equals($state['desired_digest'], $state['observed_digest']);
+ $ip = $expectedState === 'removed'
+ || (is_string($expectation['private_ip_cidr'])
+ && hash_equals((string) $expectation['private_ip_cidr'], (string) $state['private_ip_cidr']));
+ $desired = $expectedState === 'removed'
+ ? in_array($state['state'], ['disabled', 'removed'], true)
+ : in_array($state['state'], ['active', 'suspended'], true);
+ $link = $linkExpected === null || $state['link_state'] === $linkExpected;
+ $ok = $shape && $identity && $revision && $digest && $ip && $desired && $link
+ && $state['readback'] === 'confirmed';
+ if (!$ok) {
+ $state['state'] = $state['state'] === 'degraded' ? 'degraded' : 'uncertain';
+ $state['readback'] = $state['readback'] === 'mismatch' ? 'mismatch' : 'missing';
+ $state['reason_code'] = $expectedState === 'removed'
+ ? 'vpc_private_teardown_uncertain' : 'vpc_private_observation_mismatch';
+ }
+ return ['ok' => $ok, 'state' => $state,
+ 'error_code' => $ok ? null : $state['reason_code']];
+ }
+
+ /** Persist the latest VM-side observation without advancing a VPC root. */
+ public static function recordPrivateObservation(
+ int $instanceId, array $validation, string $operationId
+ ): bool {
+ if ($instanceId < 1 || preg_match(self::KEY_PATTERN, $operationId) !== 1) {
+ return false;
+ }
+ $connection = (new CloudVpcMember())->getConnection();
+ $schema = $connection->getSchemaBuilder();
+ if (!$schema->hasTable('cloud_vpc_members')
+ || !$schema->hasTable('cloud_vpc_private_ips')
+ || !$schema->hasTable('cloud_vpcs')) {
+ return false;
+ }
+ return (bool) $connection->transaction(function () use (
+ $instanceId, $validation, $operationId
+ ): bool {
+ // Match the VPC -> member lock order used by control-plane
+ // mutations. A lifecycle readback from an older desired revision
+ // must never be written into a newer member row.
+ $probe = CloudVpcMember::where('instance_id', $instanceId)
+ ->whereNotIn('state', ['deleted'])->first();
+ if ($probe === null) {
+ return false;
+ }
+ $vpc = CloudVpc::where('id', (int) $probe->vpc_id)
+ ->lockForUpdate()->first();
+ if ($vpc === null) {
+ return false;
+ }
+ $member = CloudVpcMember::where('id', (int) $probe->id)
+ ->whereNotIn('state', ['deleted'])->lockForUpdate()->first();
+ if ($member === null) {
+ return false;
+ }
+ // Lock and validate the independent lease before changing either
+ // side of the member/lease pair. A VM readback must never copy
+ // its revision into a lease belonging to another member.
+ if ($member->private_ip_id === null) {
+ return false;
+ }
+ $private = CloudVpcPrivateIp::where('id', (int) $member->private_ip_id)
+ ->where('vpc_id', (int) $vpc->id)
+ ->where('member_id', (int) $member->id)
+ ->where('instance_id', $instanceId)
+ ->lockForUpdate()->first();
+ if ($private === null
+ || (int) $private->subnet_id !== (int) $member->subnet_id
+ || (string) $private->private_ip !== (string) $member->private_ip) {
+ return false;
+ }
+ $state = is_array($validation['state'] ?? null)
+ ? $validation['state'] : [];
+ $ok = ($validation['ok'] ?? false) === true;
+ $desired = (string) ($state['desired_state'] ?? 'active');
+ $expectedRevision = max(0, (int) ($state['desired_revision'] ?? 0));
+ $expectedDigest = self::safeDigest($state['desired_digest'] ?? null);
+ $memberDigest = self::safeDigest($member->desired_digest);
+ if ((int) $member->desired_revision !== $expectedRevision
+ || $memberDigest !== $expectedDigest) {
+ return false;
+ }
+ // `last_operation_id` on a member is the VPC proof marker. A
+ // VM lifecycle job may refresh the private lease, but must not
+ // replace a successful root operation marker with its unrelated
+ // job ID; doing so would make an already-proven VPC appear stale.
+ $rootOperationId = (string) ($vpc->last_operation_id ?? '');
+ if ($rootOperationId === '' || $rootOperationId === $operationId) {
+ $member->last_operation_id = $operationId;
+ }
+ $member->observed_revision = max(0, (int) ($state['observed_revision'] ?? 0));
+ $member->observed_digest = self::safeDigest($state['observed_digest'] ?? null);
+ // Mirror the member outcome into its lease row. A private-IP
+ // state left at `deleting` or `active` after readback would make
+ // the next expectation disagree with the member ledger.
+ $privateState = $ok
+ ? ($desired === 'removed' ? 'deleted' : 'active')
+ : ($desired === 'removed' ? 'deleting' : 'uncertain');
+ if ($ok && $desired === 'removed') {
+ $member->state = 'deleted';
+ $member->last_error_code = null;
+ $private->state = $privateState;
+ $private->observed_revision = $member->observed_revision;
+ $private->observed_digest = $member->observed_digest;
+ $private->last_error_code = null;
+ $private->updated_at = time();
+ $private->save();
+ } elseif ($ok) {
+ $member->state = 'active';
+ $member->last_error_code = null;
+ } else {
+ $member->state = $desired === 'removed' ? 'deleting' : 'uncertain';
+ $member->last_error_code = is_string($validation['error_code'] ?? null)
+ ? self::safeErrorCode((string) $validation['error_code']) : 'vpc_private_observation_missing';
+ }
+ $member->updated_at = time();
+ $member->save();
+ $private->state = $privateState;
+ $private->observed_revision = $member->observed_revision;
+ $private->observed_digest = $member->observed_digest;
+ $private->last_error_code = $member->last_error_code;
+ $private->updated_at = time();
+ $private->save();
+ return true;
+ }, 3);
+ }
+
+ private static function safeDigest($value): ?string
+ {
+ return is_string($value) && preg_match('/^[a-f0-9]{64}$/Di', $value) === 1
+ ? strtolower($value) : null;
+ }
+
private static function touchDesiredLocked(CloudVpc $vpc, string $kind,
string $requestKey, string $requestHash): array
{
@@ -1920,6 +2531,10 @@ final class CloudVpcService
'observed_revision' => max(0, (int) ($host['observed_revision'] ?? $host['revision'] ?? 0)),
'observed_digest' => is_string($host['observed_digest'] ?? $host['digest'] ?? null)
? substr((string) ($host['observed_digest'] ?? $host['digest']), 0, 64) : null,
+ 'transport_verified' => ($host['transport_verified'] ?? false) === true,
+ 'private_dataplane_verified' => (
+ ($host['private_dataplane_verified'] ?? false) === true
+ ),
'next_stage_allowed' => ($host['next_stage_allowed'] ?? false) === true,
];
}
diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php
index 11130f75..6b7c1684 100644
--- a/web/app/Services/Cloud/CloudWorkerService.php
+++ b/web/app/Services/Cloud/CloudWorkerService.php
@@ -6,6 +6,7 @@ use App\Models\CloudInvoice;
use App\Models\CloudJob;
use App\Models\CloudOrder;
use App\Models\CloudServicePeriod;
+use App\Models\CloudVpc;
use App\Models\User;
final class CloudWorkerService
@@ -179,7 +180,14 @@ final class CloudWorkerService
$instance = CloudInstance::find($job->instance_id);
$desired = self::policyDesired($instance, (string) $job->kind);
if (!$desired) {
- $this->finish($job, ['status' => 'succeeded', 'result' => ['skipped' => true]], $payload);
+ // Even a no-op policy result crosses the instance
+ // lifecycle boundary: a stale/missing private readback
+ // must not be hidden by the early skip path.
+ $skipped = $this->applyPrivateLifecycleGate(
+ $job, ['status' => 'succeeded', 'result' => ['skipped' => true]],
+ $payload, $actor
+ );
+ $this->finish($job, $skipped, $payload);
return true;
}
}
@@ -280,8 +288,11 @@ final class CloudWorkerService
&& ($operation['service_was_running_hint'] ?? false) === true) {
$result['service_was_running'] = true;
}
- $this->finish($job, ['status' => 'succeeded',
- 'result' => $result], $payload);
+ $skipped = $this->applyPrivateLifecycleGate(
+ $job, ['status' => 'succeeded', 'result' => $result],
+ $payload, $actor
+ );
+ $this->finish($job, $skipped, $payload);
return true;
}
$operation = $this->engine->host((int) $job->host_id,
@@ -347,6 +358,15 @@ final class CloudWorkerService
$operation = ['status' => $operation['status'] ?? 'succeeded',
'result' => $operation];
}
+ // Every terminal VM mutation gets the same private-network
+ // readback gate. The engine's normal VM payload contains only a
+ // domain/XML observation; an attached VPC additionally requires
+ // member identity, revision and digest evidence from the future
+ // data-plane reconciler. A missing or stale observation is kept
+ // uncertain before finish() can commit public/billing state.
+ $operation = $this->applyPrivateLifecycleGate(
+ $job, $operation, $payload, $actor
+ );
$this->finish($job, $operation, $payload);
} catch (\Throwable $error) {
// Diagnostics deliberately contain no raw payloads, credentials or upstream tracebacks.
@@ -504,7 +524,7 @@ final class CloudWorkerService
$this->observedErrors[] = $status === 'failed' ? 'task_failed' : 'task_uncertain';
}
$result = $operation['result'] ?? null;
- (new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, $status, $result, $operation) {
+ (new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, &$status, &$result, &$operation) {
// All financial transactions acquire the wallet first, avoiding wallet/order deadlocks.
$user = User::where('id', $claimed->user_id)->lockForUpdate()->first();
$order = $claimed->order_id === null ? null : CloudOrder::where('id', $claimed->order_id)->lockForUpdate()->first();
@@ -513,6 +533,58 @@ final class CloudWorkerService
|| in_array($job->status, ['succeeded', 'failed', 'cancelled'], true)) {
return;
}
+ if ($status === 'succeeded' && $job->instance_id !== null) {
+ $capturedPrivateExpectation = $payload['private_expectation'] ?? null;
+ // VPC mutators lock VPC -> instance. Acquire the same order
+ // before checking the captured target so detach/attach cannot
+ // change a member between the readback and this commit.
+ $privateProbe = is_array($capturedPrivateExpectation)
+ ? $capturedPrivateExpectation
+ : CloudVpcService::instancePrivateExpectation((int) $job->instance_id);
+ $vpcKeyToLock = is_string($privateProbe['vpc_id'] ?? null)
+ ? (string) $privateProbe['vpc_id'] : '';
+ if (preg_match('/^[0-9a-f]{32}$/D', $vpcKeyToLock) === 1) {
+ CloudVpc::where('vpc_key', $vpcKeyToLock)
+ ->lockForUpdate()->first();
+ }
+ $lockedInstance = CloudInstance::where('id', $job->instance_id)
+ ->lockForUpdate()->first();
+ $privateTargetChanged = $lockedInstance === null;
+ if (!$privateTargetChanged && is_array($capturedPrivateExpectation)) {
+ $privateTargetChanged = !CloudVpcService::privateExpectationMatches(
+ (int) $lockedInstance->id, $capturedPrivateExpectation
+ );
+ } elseif (!$privateTargetChanged) {
+ // Jobs created before the immutable expectation field was
+ // introduced cannot prove that an attached VPC target
+ // stayed unchanged. Allow legacy public-only jobs to
+ // finish, but fail closed for any live private member.
+ $privateTargetChanged = CloudVpcService::instancePrivateExpectation(
+ (int) $lockedInstance->id
+ )['required'] === true;
+ }
+ if ($privateTargetChanged) {
+ // The VPC target changed after the worker's engine
+ // readback. Keep the operation uncertain while holding
+ // the same instance row lock used by attach/detach.
+ $status = 'uncertain';
+ $operation['status'] = 'uncertain';
+ $operation['error_code'] = 'vpc_private_observation_mismatch';
+ $operation['error'] = '实例私网目标已变化,结果正在核对';
+ if (is_array($operation['result'] ?? null)) {
+ $operation['result']['private_state'] = [
+ 'version' => 1, 'required' => true,
+ 'state' => 'uncertain', 'desired_state' => 'active',
+ 'readback' => 'mismatch',
+ 'reason_code' => 'vpc_private_observation_mismatch',
+ 'desired_revision' => 0, 'observed_revision' => 0,
+ 'desired_digest' => null, 'observed_digest' => null,
+ 'link_state' => 'absent', 'observed_at' => time(),
+ ];
+ }
+ $result = $operation['result'] ?? null;
+ }
+ }
$safe = is_array($result)
? CloudOperationErrorCatalog::workerResult($result) : null;
CloudSnapshotService::finishJobLocked($job, $status, $result, $payload);
@@ -973,6 +1045,265 @@ final class CloudWorkerService
}, 3);
}
+ /**
+ * Failed/uncertain jobs normally must keep their original disposition,
+ * but a job that crossed the engine mutation boundary still needs the
+ * same private-state readback as a successful job. The durable engine
+ * marker avoids treating ordinary request validation failures as if they
+ * had changed a VM. Network changes retain their narrower checkpoint:
+ * the child must have claimed the target IP before this gate runs.
+ *
+ * The actual bounded private-state read happens outside finish()'s DB
+ * transaction below, so a slow or failed engine read never holds wallet or
+ * VPC rows.
+ */
+ private static function privateMutationReadbackRequired(
+ CloudJob $job, array $operation, string $status
+ ): bool {
+ if (!in_array($status, ['failed', 'uncertain'], true)) {
+ return false;
+ }
+ $kind = (string) $job->kind;
+ $result = is_array($operation['result'] ?? null)
+ ? $operation['result'] : [];
+ $vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
+
+ // Public-IP changes already have a commit checkpoint inside the
+ // child. A generic invocation marker alone would gate a preflight
+ // rejection before any address or XML mutation took place.
+ if ($kind === 'network_change') {
+ if (($operation['network_mutation_started'] ?? false) === true) {
+ return true;
+ }
+ foreach (['network_changed', 'network_mutation_started'] as $key) {
+ if (($result[$key] ?? false) === true || ($vm[$key] ?? false) === true) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ if (($operation['mutation_started'] ?? false) === true
+ || ($operation['network_mutation_started'] ?? false) === true) {
+ return true;
+ }
+ if (in_array($kind, ['delete', 'expiry_delete'], true)
+ && (($operation['vm_delete_committed'] ?? false) === true
+ || array_key_exists('cleanup_pending', $operation))) {
+ return true;
+ }
+ if ($kind === 'reinstall' && self::reinstallNeedsReview($operation)) {
+ return true;
+ }
+ // Preserve existing post-commit markers from older engine versions.
+ foreach (['reinstall_committed', 'cleanup_pending', 'network_changed',
+ 'settings_changed', 'restored', 'deleted', 'created'] as $key) {
+ if (($result[$key] ?? false) === true || ($vm[$key] ?? false) === true) {
+ return true;
+ }
+ }
+ // An uncertain job dispatched by a legacy engine has no durable
+ // mutation marker. Its dispatched_at timestamp is the conservative
+ // compatibility signal; failed pre-dispatch jobs keep a false gate.
+ return $status === 'uncertain' && $job->dispatched_at !== null;
+ }
+
+ private function applyPrivateLifecycleGate(
+ CloudJob $job, array $operation, array $payload, array $actor
+ ): array {
+ $status = is_string($operation['status'] ?? null)
+ ? strtolower((string) $operation['status']) : 'uncertain';
+ $kind = (string) $job->kind;
+ $postCommitReinstall = $kind === 'reinstall'
+ && $status === 'failed' && self::reinstallNeedsReview($operation);
+ $terminalSuccess = in_array($status, ['succeeded', 'completed', 'done'], true);
+ if (!$terminalSuccess && !$postCommitReinstall
+ && !self::privateMutationReadbackRequired($job, $operation, $status)) {
+ return $operation;
+ }
+ $lifecycleKinds = [
+ 'create', 'reinstall', 'network_change', 'instance_settings', 'rescue',
+ 'snapshot_create', 'snapshot_restore', 'snapshot_delete', 'delete',
+ 'expiry_delete', 'action', 'service_suspend', 'service_resume',
+ 'network_suspend', 'network_resume', 'firewall_apply',
+ ];
+ if (!in_array($kind, $lifecycleKinds, true)) {
+ return $operation;
+ }
+
+ $instance = $job->instance_id === null ? null : CloudInstance::find($job->instance_id);
+ $currentExpectation = CloudVpcService::instancePrivateExpectation(
+ $instance === null ? null : (int) $instance->id
+ );
+ $capturedExpectation = is_array($payload['private_expectation'] ?? null)
+ ? $payload['private_expectation'] : null;
+ $expectationStale = false;
+ $expectation = $currentExpectation;
+ if ($capturedExpectation !== null) {
+ $expectation = $capturedExpectation;
+ // `state` is an observed/transient lifecycle value. A member can
+ // legitimately move applying -> active while this VM job is in
+ // flight; desired_state plus the revision/digest still bind the
+ // immutable target. Removal is represented by desired_state and
+ // therefore remains strict.
+ foreach (['required', 'desired_state', 'vpc_id', 'member_id',
+ 'private_ip_cidr', 'desired_revision', 'desired_digest'] as $key) {
+ $left = $capturedExpectation[$key] ?? null;
+ $right = $currentExpectation[$key] ?? null;
+ if ((string) $left !== (string) $right) {
+ $expectationStale = true;
+ break;
+ }
+ }
+ if ($expectationStale) {
+ // Keep the original target in the evidence, but make the
+ // gate fail closed instead of validating it against a newer
+ // VPC revision that arrived while this job was running.
+ $expectation['required'] = true;
+ $expectation['reason_code'] = 'vpc_private_observation_mismatch';
+ }
+ }
+ $result = is_array($operation['result'] ?? null) ? $operation['result'] : [];
+ $vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
+ $uuid = is_string($instance?->uuid) ? (string) $instance->uuid
+ : (is_string($vm['uuid'] ?? null) ? $vm['uuid']
+ : (is_string($result['uuid'] ?? null) ? $result['uuid'] : null));
+ // Accept both the normal {vm:{...}} envelope and a rolling-upgrade
+ // response that places the bounded observation beside `vm`. A
+ // structured field at either level must stay strict; only a complete
+ // omission can use the legacy public-only compatibility path.
+ $hasPrivateState = array_key_exists('private_state', $vm)
+ || array_key_exists('private_state', $result);
+ $observation = is_array($vm['private_state'] ?? null)
+ ? $vm['private_state']
+ : (is_array($result['private_state'] ?? null)
+ ? $result['private_state'] : null);
+
+ // Pre-VPC engines and the existing public-only worker fixtures do not
+ // return a private_state object at all. There is no private target to
+ // prove for those jobs, so do not issue a second host request (which
+ // would also consume the next response in a legacy mock queue). Once
+ // a VPC member is expected, the gate remains fail-closed below; and a
+ // structured observation supplied by any engine is still validated,
+ // including the public-only case where it can reveal an unexpected
+ // private NIC.
+ if (($expectation['required'] ?? false) !== true && !$hasPrivateState) {
+ return $operation;
+ }
+
+ // A delete may legitimately have no readable domain after the
+ // irreversible boundary. Only a VM without a VPC target can use the
+ // explicit disabled/confirmed result; a VPC tombstone still requires
+ // a future removed/readback proof.
+ if ($uuid !== null) {
+ try {
+ $observed = $this->engine->host(
+ (int) $job->host_id, 'GET',
+ 'vms/' . rawurlencode($uuid) . '/private-state',
+ $actor, null, (string) $job->operation_id
+ );
+ if (is_array($observed)) {
+ $observation = $observed;
+ }
+ } catch (\Throwable $error) {
+ $absenceProven = in_array($kind, ['delete', 'expiry_delete'], true)
+ && $expectation['required'] !== true
+ && $error instanceof CloudError
+ && $error->httpStatus === 404;
+ if ($absenceProven) {
+ // A VM with no VPC member has no independent private
+ // teardown target. Once the engine has confirmed the
+ // domain deletion, a 404 is the expected absence proof.
+ $observation = [
+ 'version' => 1, 'required' => false,
+ 'vm_uuid' => $uuid, 'vpc_id' => null, 'member_id' => null,
+ 'interface' => null, 'state' => 'disabled',
+ 'desired_revision' => 0, 'observed_revision' => 0,
+ 'desired_digest' => null, 'observed_digest' => null,
+ 'readback' => 'confirmed', 'link_state' => 'absent',
+ 'private_ip_cidr' => null, 'private_mac' => null,
+ 'reason_code' => null, 'observed_at' => time(),
+ ];
+ }
+ if (!$absenceProven) {
+ // The operation itself may already be durable. Convert a
+ // missing observation into a stable uncertain result
+ // rather than leaking a transport/host diagnostic.
+ $observation = null;
+ }
+ }
+ }
+ $validation = $expectationStale
+ ? ['ok' => false, 'state' => [
+ 'version' => 1, 'required' => true,
+ 'vm_uuid' => $uuid, 'vpc_id' => $expectation['vpc_id'] ?? null,
+ 'member_id' => $expectation['member_id'] ?? null,
+ 'interface' => null, 'state' => 'uncertain',
+ 'desired_state' => $expectation['desired_state'] ?? 'active',
+ 'desired_revision' => (int) ($expectation['desired_revision'] ?? 0),
+ 'observed_revision' => 0,
+ 'desired_digest' => $expectation['desired_digest'] ?? null,
+ 'observed_digest' => null, 'readback' => 'mismatch',
+ 'link_state' => 'absent', 'private_ip_cidr' => null,
+ 'private_mac' => null,
+ 'reason_code' => 'vpc_private_observation_mismatch',
+ 'observed_at' => time(),
+ ], 'error_code' => 'vpc_private_observation_mismatch']
+ : CloudVpcService::validatePrivateObservation(
+ $expectation, $observation, $kind
+ );
+ if ($instance !== null && $expectation['required'] === true && !$expectationStale) {
+ try {
+ $recorded = CloudVpcService::recordPrivateObservation(
+ (int) $instance->id, $validation, (string) $job->operation_id
+ );
+ if ($recorded !== true && $validation['ok'] === true) {
+ $validation = [
+ 'ok' => false,
+ 'state' => array_merge($validation['state'], [
+ 'state' => 'uncertain', 'readback' => 'mismatch',
+ 'reason_code' => 'vpc_private_observation_mismatch',
+ 'observed_at' => time(),
+ ]),
+ 'error_code' => 'vpc_private_observation_mismatch',
+ ];
+ }
+ } catch (\Throwable $error) {
+ // The operation gate remains uncertain even when the
+ // secondary VPC observation cache cannot be written.
+ error_log('cloud private observation could not be recorded: '
+ . get_class($error));
+ if ($validation['ok'] === true) {
+ $validation = [
+ 'ok' => false,
+ 'state' => array_merge($validation['state'], [
+ 'state' => 'uncertain', 'readback' => 'missing',
+ 'reason_code' => 'vpc_private_observation_missing',
+ 'observed_at' => time(),
+ ]),
+ 'error_code' => 'vpc_private_observation_missing',
+ ];
+ }
+ }
+ }
+ $result['private_state'] = $validation['state'];
+ if ($kind === 'reinstall' && !$validation['ok']) {
+ // Preserve the replacement credential/spec on a post-commit
+ // uncertain reinstall; the existing finish() compatibility path
+ // will not mark the task successful or replay it blindly.
+ $result['reinstall_committed'] = true;
+ }
+ $operation['result'] = $result;
+ if (!$validation['ok']) {
+ $operation['status'] = 'uncertain';
+ $operation['safe_to_refund'] = false;
+ $operation['error_code'] = $validation['error_code']
+ ?? 'vpc_private_observation_missing';
+ $operation['error'] = '实例私网状态尚未完成实际回读,请稍后核对';
+ }
+ return $operation;
+ }
+
private static function validSystemCleanup(CloudJob $job, array $payload): bool
{
if ($job->instance_id === null || ($payload['body'] ?? null) !== []) {
diff --git a/web/tests/CloudVpcControlPlaneContractTest.php b/web/tests/CloudVpcControlPlaneContractTest.php
index 67fbfbdc..23fdd424 100644
--- a/web/tests/CloudVpcControlPlaneContractTest.php
+++ b/web/tests/CloudVpcControlPlaneContractTest.php
@@ -22,6 +22,7 @@ $middleware = $read('app/Middleware/CloudApi.php');
$engineClient = $read('app/Services/Cloud/CloudEngineClient.php');
$accountDeletion = $read('app/Services/UserDataDeletionService.php');
$jobService = $read('app/Services/Cloud/CloudJobService.php');
+$orderService = $read('app/Services/Cloud/CloudOrderService.php');
$workerService = $read('app/Services/Cloud/CloudWorkerService.php');
foreach ([
@@ -88,6 +89,19 @@ $assert(!str_contains($service, 'CloudJob')
&& !str_contains($service, 'network_change')
&& !str_contains($service, 'publicIpCidr'),
'VPC service must not reuse the public network_change/job payload.');
+$assert(str_contains($service, '$orphanLease')
+ && str_contains($service, '$stateMatches')
+ && str_contains($service, "where('instance_id', \$instanceId)")
+ && str_contains($service, "where('vpc_id', (int) \$vpc->id)")
+ && str_contains($service, 'hash_equals($rootOperationId')
+ && str_contains($service, "where('status', 'succeeded')")
+ && str_contains($service, '$operationFresh')
+ && str_contains($service, '$vpc->last_success_at')
+ && str_contains($service, '$privateIp->observed_revision')
+ && str_contains($service, '$privateIp->observed_digest')
+ && str_contains($service, "(string) \$privateIp->state === 'active'")
+ && str_contains($service, '$private->state = $privateState'),
+ 'VPC private projections must fail closed on orphan/mismatched leases and require current root readback proof.');
$assert(str_contains($service, '当前 VPC 控制面不支持子网网关')
&& str_contains($service, "'private_ip_cidr'")
&& str_contains($service, "'next_hop_member_id'")
@@ -107,7 +121,20 @@ $assert(str_contains($engineClient, 'function vpc(')
$assert(str_contains($accountDeletion, "cloud_vpcs")
&& str_contains($accountDeletion, "cloud_private_networks")
&& str_contains($jobService, 'assertInstanceCanBeDeleted')
+ && str_contains($orderService, "private_expectation")
&& str_contains($workerService, '旧删除任务缺少 VPC 删除门禁'),
- 'Account/VM deletion can bypass the independent VPC control-plane ledger.');
+ 'Account/VM creation/deletion can bypass the independent VPC control-plane ledger.');
+$legacyPrivateGate = strpos($workerService,
+ "if ((\$expectation['required'] ?? false) !== true && !\$hasPrivateState)");
+$privateReadback = strpos($workerService,
+ "'vms/' . rawurlencode(\$uuid) . '/private-state'");
+$assert(str_contains($workerService, '$hasPrivateState')
+ && str_contains($workerService, "&& !\$hasPrivateState")
+ && str_contains($workerService, "array_key_exists('private_state', \$result)")
+ && str_contains($workerService, 'Pre-VPC engines')
+ && $legacyPrivateGate !== false
+ && $privateReadback !== false
+ && $legacyPrivateGate < $privateReadback,
+ 'Legacy public-only jobs without private_state must remain compatible while structured observations stay gated.');
echo "VPC control-plane schema, API, typed DTO and independent operation contracts passed.\n";