fix: gate instance lifecycle on private readback

venix
admin 1 month ago
parent 3190012361
commit e003c2abc2

@ -2234,6 +2234,8 @@ def test_power_action_has_persistent_idempotency(boundary):
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
saved = json.loads(store.get("operation", "reboot-operation"))
assert saved["target_action"] == "reboot"
assert saved["mutation_started"] is True
assert operation["mutation_started"] is True
def test_orphaned_power_action_reconciles_only_the_final_target_state(boundary):

@ -5396,6 +5396,41 @@ class VMServiceTests(unittest.TestCase):
self.assertEqual(result[0].vm_id, "00000000-0000-4000-8000-000000000002")
self.assertEqual(self.backend.opened, self.backend.closed)
def test_metric_counters_reject_ambiguous_extra_nic_role(self):
constants = {
"VIR_DOMAIN_STATS_CPU_TOTAL": 2,
"VIR_DOMAIN_STATS_BALLOON": 4,
"VIR_DOMAIN_STATS_BLOCK": 8,
"VIR_DOMAIN_STATS_INTERFACE": 16,
"VIR_CONNECT_GET_ALL_DOMAINS_STATS_RUNNING": 32,
}
for name, value in constants.items():
setattr(self.service_module.libvirt, name, value)
domain = types.SimpleNamespace(
UUIDString=lambda: "00000000-0000-4000-8000-000000000003",
info=lambda: (1, 2048 * 1024, 2048 * 1024, 2, 0),
ID=lambda: 9,
XMLDesc=lambda _flags: ("<domain><devices>"
"<interface type='bridge'><mac address='52:54:00:12:34:56'/>"
"<target dev='vnet9'/><alias name='public0'/></interface>"
"<interface type='bridge'><mac address='52:54:00:aa:bb:cc'/>"
"<target dev='vnet10'/></interface>"
"</devices></domain>"),
)
def get_stats(_connection, _groups, _flags):
return [(domain, {"cpu.time": 123, "net.count": 2,
"net.0.name": "vnet9", "net.0.rx.bytes": 456,
"net.0.tx.bytes": 789,
"net.1.name": "vnet10", "net.1.rx.bytes": 12,
"net.1.tx.bytes": 34})]
with patch.object(FakeConnection, "getAllDomainStats", get_stats, create=True):
result = self.service.metric_counters()
self.assertEqual(len(result), 1)
self.assertEqual(result[0].values["net.0.role"], "public")
self.assertEqual(result[0].values["net.1.role"], "unknown")
if __name__ == "__main__":
unittest.main()

@ -496,6 +496,34 @@ def test_sampler_prefers_stable_network_identity_over_renamed_host_tap():
}
def test_sampler_billing_frame_only_contains_explicit_public_role():
sampler = MetricSampler()
point = sampler.observe(host_scope(HOST), raw(extra={
"net.count": 2,
"net.0.identity": "mac:52:54:00:12:34:56",
"net.0.role": "public",
"net.1.name": "vnet2",
"net.1.identity": "mac:52:54:00:aa:bb:cc",
"net.1.role": "private",
"net.1.rx.bytes": 3000,
"net.1.tx.bytes": 4000,
}), timestamp=1000, monotonic=10)
assert point["network_billing_devices"] == {
"mac:52:54:00:12:34:56": [1000, 2000],
}
def test_sampler_billing_rejects_ambiguous_multinic_frame():
sampler = MetricSampler()
point = sampler.observe(host_scope(HOST), raw(extra={
"net.count": 2,
"net.1.name": "vnet2",
"net.1.rx.bytes": 3000,
"net.1.tx.bytes": 4000,
}), timestamp=1000, monotonic=10)
assert point["network_billing_devices"] is None
def test_collector_associates_host_and_current_owner_without_hiding_unowned_vms():
saved = []

@ -0,0 +1,88 @@
from virsh_panel.private_network import observe_private_state_from_xml
def _xml(*, private=True, link="up", private_source="vc-private",
private_ip="10.24.3.7/32", verified="true", private_filter=False,
public_source="br0", private_type="bridge"):
private_block = ""
if private:
filt = '<filterref filter="clean-traffic" />' if private_filter else ""
source = (f'<source bridge="{private_source}" />'
if private_type == "bridge"
else f'<source network="{private_source}" />')
private_block = f'''<interface type="{private_type}">
<mac address="52:54:00:aa:bb:cc" />
{source}
<link state="{link}" />{filt}
<alias name="private0" />
</interface>'''
return f'''<domain><uuid>00000000-0000-4000-8000-000000000001</uuid>
<metadata><panel:managed xmlns:panel="urn:virsh-panel:1" private-ip-cidr="{private_ip or ''}"
private-network-verified="{verified}" /></metadata>
<devices>
<interface type="bridge"><mac address="52:54:00:11:22:33" />
<source bridge="{public_source}" /><alias name="public0" /></interface>
{private_block}
</devices></domain>'''
def test_private_state_without_private_nic_is_confirmed_disabled():
state = observe_private_state_from_xml(_xml(private=False), vm_uuid="00000000-0000-4000-8000-000000000001", observed_at=10)
assert state["state"] == "disabled"
assert state["readback"] == "confirmed"
assert state["interface"] is None
def test_private_state_requires_verified_independent_bridge_and_ip():
state = observe_private_state_from_xml(_xml(), observed_at=10)
assert state["state"] == "degraded"
assert state["reason_code"] == "private_source_unverified"
state = observe_private_state_from_xml(_xml(), host_verified=True, observed_at=10)
assert state["state"] == "active"
assert state["readback"] == "confirmed"
assert state["private_ip_cidr"] == "10.24.3.7/32"
unverified = observe_private_state_from_xml(_xml(verified="false"))
assert unverified["state"] == "degraded"
assert unverified["reason_code"] == "private_source_unverified"
missing_ip = observe_private_state_from_xml(_xml(private_ip=None))
assert missing_ip["state"] == "degraded"
assert missing_ip["reason_code"] == "private_ip_missing"
def test_private_state_marks_link_down_suspended():
state = observe_private_state_from_xml(_xml(link="down"), host_verified=True)
assert state["state"] == "suspended"
assert state["link_state"] == "down"
def test_private_state_rejects_public_source_or_filter():
shared = observe_private_state_from_xml(_xml(private_source="br0"))
assert shared["state"] == "degraded"
assert shared["reason_code"] == "private_source_invalid"
filtered = observe_private_state_from_xml(_xml(private_filter=True), host_verified=True)
assert filtered["state"] == "degraded"
assert filtered["reason_code"] == "private_filter_present"
def test_private_network_source_network_is_not_proven_isolated():
state = observe_private_state_from_xml(
_xml(private_source="default", private_type="network"), host_verified=True
)
# The helper only accepts an explicit host bridge; a libvirt network name
# does not prove which bridge it resolves to.
assert state["state"] == "degraded"
assert state["reason_code"] == "private_source_invalid"
def test_private_state_rejects_ambiguous_extra_nic_without_private_role():
xml = _xml(private=False).replace(
'</devices>',
'<interface type="bridge"><mac address="52:54:00:dd:ee:ff" />'
'<source bridge="vpc0" /></interface></devices>',
)
state = observe_private_state_from_xml(xml)
assert state["state"] == "degraded"
assert state["reason_code"] == "private_interface_role_invalid"

@ -292,6 +292,7 @@ def test_coordinator_requires_exact_fabric_revision_and_all_host_readback():
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True,
"readback": {"safe": "ok", "oversized": "x" * 513}},
}))
assert observed["state"] == "active"
@ -303,6 +304,25 @@ def test_coordinator_requires_exact_fabric_revision_and_all_host_readback():
assert blocked["next_stage_allowed"] is False
def test_transport_only_observation_never_opens_private_dataplane_gate():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload()))
digest = result["desired_digest"]
pending = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True},
}))
assert pending["state"] == "applying"
assert pending["next_stage_allowed"] is False
assert pending["private_dataplane_verified"] is False
assert pending["hosts"][0]["state"] == "applying"
assert pending["hosts"][0]["last_error_code"] == (
"vpc_private_dataplane_unverified"
)
def test_partial_host_observation_is_degraded_and_never_active():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
@ -322,7 +342,8 @@ def test_partial_host_observation_is_degraded_and_never_active():
digest = result["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True},
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
@ -331,7 +352,8 @@ def test_partial_host_observation_is_degraded_and_never_active():
with pytest.raises(VpcError) as inconsistent:
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": False},
"observed_digest": digest, "transport_verified": False,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
@ -454,7 +476,8 @@ def test_recover_invalidates_observed_gate_and_record_round_trips():
raw["observed_digest"] = raw["desired_digest"]
raw["hosts"][0].update(state="active", observed_revision=1,
observed_digest=raw["desired_digest"],
transport_verified=True)
transport_verified=True,
private_dataplane_verified=True)
store.set("vpc_desired", "vpc-hk-1", json.dumps(raw))
coordinator.recover()
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
@ -492,7 +515,8 @@ def test_new_desired_revision_keeps_last_observed_evidence_until_readback():
digest = first["desired_digest"]
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True},
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
}))
second = asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",

@ -3862,6 +3862,34 @@ def create_app(
require_vm_access(_session, profile, vm_id)
return new_service(profile).get_vm_state(vm_id)
@app.get("/api/v1/vms/{vm_id}/private-state")
def get_vm_private_state(vm_id: str, _session: Session = Depends(require_approved)):
"""Return the bounded private-NIC readback for one VM.
This endpoint intentionally reports the domain/XML observation only;
VPC revision, WireGuard peer and nftables evidence are checked by the
PHP control plane before it publishes a mutation as successful.
"""
profile = current_profile()
require_vm_access(_session, profile, vm_id)
service = new_service(profile)
getter = getattr(service, "get_private_state", None)
if callable(getter):
return getter(vm_id)
# Test/rolling-upgrade adapters may only expose get_vm; preserve the
# same bounded contract without making the route a second XML parser.
vm = service.get_vm(vm_id)
return vm.get("private_state", {
"version": 1, "required": False, "vm_uuid": vm.get("uuid"),
"vpc_id": None, "member_id": None, "interface": None,
"state": "uncertain", "desired_revision": 0,
"observed_revision": 0, "desired_digest": None,
"observed_digest": None, "readback": "missing",
"link_state": "absent", "private_ip_cidr": None,
"private_mac": None, "reason_code": "private_state_unavailable",
"observed_at": int(time.time()),
})
@app.get("/api/v1/vms/{vm_id}")
def get_vm(vm_id: str, _session: Session = Depends(require_approved)):
profile = current_profile()

@ -337,6 +337,11 @@ class DatabaseStore:
or user_id > _MAX_UNSIGNED_BIGINT)):
raise ValueError("Metric user ID is invalid")
raw_devices = point.get("network_devices")
# New collectors provide a role-filtered public0 frame for quota
# accounting. Keep the legacy all-NIC fallback for old callers
# and old hosts that have not emitted explicit role metadata.
if "network_billing_devices" in point:
raw_devices = point.get("network_billing_devices")
# None means libvirt did not return a complete NIC counter frame.
# Never replace a durable baseline with incomplete data, otherwise
# the next healthy sample would look like a brand-new NIC and the

@ -137,7 +137,7 @@ READ_PATHS = (
r"fabric", r"fabric/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}",
r"vpcs", r"vpcs/[A-Za-z0-9][A-Za-z0-9_.-]{0,63}",
r"user-isos/[0-9a-f]{32}",
rf"vms/{VM_ID}(?:/state|/metrics|/traffic/daily|/rescue|"
rf"vms/{VM_ID}(?:/state|/private-state|/metrics|/traffic/daily|/rescue|"
r"/firewall|/snapshots(?:/[0-9a-f]{32})?)?",
r"(?:reinstall-jobs|rescue-jobs|vm-templates/import-jobs|rescue-templates/import-jobs|user-isos/import-jobs)/[A-Za-z0-9_-]+",
)
@ -868,6 +868,16 @@ class EngineApplication:
elif self._is_vm_delete_operation(operation):
await self._capture_delete_cleanup(
host_id, operation, delegated)
# Persist the mutation boundary before invoking the child.
# If this write fails, the child is not called and the
# operation remains a safe prevalidation/coordination failure.
# Network changes have a narrower checkpoint inside the child;
# PHP additionally requires network_mutation_started for that
# resource before gating failed results.
operation["mutation_started"] = True
await asyncio.to_thread(
self.store.set, "operation", operation["operation_id"],
json.dumps(operation))
await app(child_scope, receive, capture)
status = next(item["status"] for item in messages if item["type"] == "http.response.start")
payload = json.loads(b"".join(item.get("body", b"") for item in messages
@ -914,8 +924,9 @@ class EngineApplication:
if operation["status"] in {"succeeded", "pending"}:
await json_response(send, status, {**payload, **self._public_operation(operation)})
else:
await json_response(send, status, {**payload, "operation_id": operation["operation_id"],
"status": operation["status"], "safe_to_refund": operation["safe_to_refund"]})
await json_response(send, status, {
**payload, **self._public_operation(operation),
})
except BaseException as exc:
network_preflight_failed = (
self._is_network_change_resource(
@ -1922,7 +1933,11 @@ class EngineApplication:
"actor_can_manage": bool(delegated.get("can_manage")),
"actor_is_admin": bool(delegated.get("is_admin")),
"resource": resource, "method": method, "status": "running", "result": None,
"safe_to_refund": False, "created_at": int(time.time())}
"safe_to_refund": False, "created_at": int(time.time()),
# Set immediately before invoking the child endpoint. This
# lets the PHP worker distinguish an actual mutation boundary
# from a durable prevalidation failure.
"mutation_started": False}
if create:
# New PHP-backed creates always receive the reserved UUID below.
# Keep this durable marker so cleanup reconciliation checks the
@ -3166,6 +3181,11 @@ class EngineApplication:
"operation_id", "status", "result", "error", "safe_to_refund",
"service_was_running_hint", "created_at", "finished_at")
if key in record}
# Expose only bounded lifecycle markers. Do not leak persisted
# request bodies or host diagnostics through the operation API.
for key in ("mutation_started", "network_mutation_started"):
if isinstance(record.get(key), bool):
result[key] = record[key]
if ("/snapshots" in record.get("resource", "")
and record.get("error_code") in SNAPSHOT_PUBLIC_ERROR_CODES):
result["error_code"] = record["error_code"]

@ -41,6 +41,7 @@ from virsh_panel.network_roles import (
select_private_interface,
select_public_interface,
)
from virsh_panel.private_network import observe_private_state
from virsh_panel.platform_network_policy import (
MAX_PLATFORM_POLICY_SNAPSHOT_BYTES,
PLATFORM_DEFAULT_NETWORK_POLICY,
@ -1219,6 +1220,13 @@ class VMService:
rescue.get("volume") if rescue is not None else
inactive_rescue.get("volume") if inactive_rescue is not None else None),
"console_port": self._vnc_port(root),
# This is a bounded XML observation only. It deliberately does
# not claim that the host WireGuard Fabric, guest route or nft
# ACL is active; the PHP VPC ledger supplies that expectation at
# each mutation commit boundary.
"private_state": observe_private_state(
root, vm_uuid=domain.UUIDString()
),
}
@staticmethod
@ -1465,6 +1473,18 @@ class VMService:
with self._connect() as conn:
return self._vm_dict(self._domain(conn, uuid))
def get_private_state(self, uuid: str) -> dict[str, Any]:
"""Read one VM's bounded private-NIC observation.
Keep this as a separate endpoint so lifecycle reconciliation does not
need to load disk capacities or console metadata, and so a host-wide
inventory cannot accidentally execute a per-VM network probe.
"""
with self._connect() as conn:
domain = self._domain(conn, uuid)
root = self._vm_xml(domain)
return observe_private_state(root, vm_uuid=domain.UUIDString())
def get_vm_state(self, uuid: str) -> dict[str, Any]:
"""Fetch one domain's power state without loading its XML metadata."""
with self._connect() as conn:
@ -1500,8 +1520,39 @@ class VMService:
values = dict(values)
try:
root = ET.fromstring(domain.XMLDesc(0))
by_target: dict[str, str] = {}
for interface in root.findall("./devices/interface"):
interfaces = root.findall("./devices/interface")
# Resolve the complete role map before annotating any
# counters. Calling ``interface_role(...,
# legacy_public=True)`` independently would classify an
# extra unaliased NIC as public on a multi-NIC domain and
# could charge private traffic to the public quota lane.
try:
public_interface = select_public_interface(interfaces)
private_interface = select_private_interface(interfaces)
role_by_element = {
id(item): (
"public" if item is public_interface
else "private" if item is private_interface
else "unknown"
)
for item in interfaces
}
except (TypeError, ValueError):
# Preserve explicit roles for diagnostics while
# marking the complete frame unsafe. The billing
# sampler rejects any ``unknown`` role, but retaining
# the known public role keeps the observation useful
# for reconciliation and avoids silently relabelling
# a valid public0 interface.
role_by_element = {}
for item in interfaces:
try:
role_by_element[id(item)] = interface_role(
item, legacy_public=len(interfaces) == 1)
except ValueError:
role_by_element[id(item)] = "unknown"
by_target: dict[str, tuple[str, str]] = {}
for interface in interfaces:
target = interface.find("target")
mac = interface.find("mac")
target_name = target.get("dev") if target is not None else None
@ -1510,14 +1561,18 @@ class VMService:
and isinstance(address, str)
and re.fullmatch(r"[0-9A-Fa-f]{2}(?::[0-9A-Fa-f]{2}){5}",
address)):
by_target[target_name] = "mac:" + address.lower()
role = role_by_element.get(id(interface), "unknown")
by_target[target_name] = (
"mac:" + address.lower(), role
)
count = values.get("net.count")
if isinstance(count, int) and not isinstance(count, bool):
for number in range(min(max(0, count), 1024)):
name = values.get(f"net.{number}.name")
identity = by_target.get(name) if isinstance(name, str) else None
if identity is not None:
values[f"net.{number}.identity"] = identity
values[f"net.{number}.identity"] = identity[0]
values[f"net.{number}.role"] = identity[1]
except (AttributeError, ET.ParseError, libvirt.libvirtError):
# Metrics remain useful on older/unusual domains whose
# live XML cannot be read; accounting falls back to the

@ -86,7 +86,9 @@ def _device_counters(stats: Mapping[str, Any], group: str,
return signature, counters_by_direction
def _network_counter_devices(stats: Mapping[str, Any]) -> dict[str, list[int]] | None:
def _network_counter_devices(
stats: Mapping[str, Any], *, role: str | None = None,
) -> dict[str, list[int]] | None:
"""Return validated cumulative counters keyed by stable guest NIC identity.
libvirt reports rx/tx from the guest's point of view: rx is downloaded by
@ -99,20 +101,52 @@ def _network_counter_devices(stats: Mapping[str, Any]) -> dict[str, list[int]] |
if count is None or count > 1024:
return None
candidates: dict[str, list[int] | None] = {}
seen_identities: set[str] = set()
# A role-filtered frame is safe to use for billing only when every NIC
# declares a role. The one-NIC legacy format has no role field and is
# still unambiguous; a multi-NIC frame without roles is not.
explicit_role_keys = [f"net.{number}.role" in stats for number in range(count)]
has_explicit_roles = any(explicit_role_keys)
if role is not None and not has_explicit_roles and count not in {0, 1}:
return None
for number in range(count):
name = stats.get(f"net.{number}.name")
identity = stats.get(f"net.{number}.identity", name)
observed_role = stats.get(f"net.{number}.role")
if (not isinstance(identity, str) or not 1 <= len(identity) <= 255
or any(ord(char) < 32 or ord(char) == 127 for char in identity)):
return None
if identity in seen_identities:
return None
seen_identities.add(identity)
if role is not None:
if has_explicit_roles:
if observed_role not in {"public", "private", "unknown"}:
# A partially labelled multi-NIC frame cannot establish
# which interface belongs to the public quota lane.
return None
if observed_role == "unknown":
return None
if observed_role != role:
continue
elif observed_role is not None and observed_role != role:
continue
rx = _nonnegative_int(stats.get(f"net.{number}.rx.bytes"))
tx = _nonnegative_int(stats.get(f"net.{number}.tx.bytes"))
if (not isinstance(identity, str) or not 1 <= len(identity) <= 255
or any(ord(char) < 32 or ord(char) == 127 for char in identity)
or rx is None or tx is None
if (rx is None or tx is None
or rx > _MAX_UNSIGNED_BIGINT or tx > _MAX_UNSIGNED_BIGINT):
return None
# Duplicate names cannot be safely matched to a previous observation.
# Duplicate identities were rejected above, including across
# selected and unselected roles, so a bad private NIC cannot alias a
# billable public NIC.
if identity in candidates:
return None
candidates[identity] = [rx, tx]
if role is None:
return {identity: counters for identity, counters in candidates.items()
if counters is not None}
# ``candidates`` contains only the requested role when explicit roles are
# present; for the legacy one-NIC path it contains the sole public NIC.
return {identity: counters for identity, counters in candidates.items()
if counters is not None}
@ -210,6 +244,13 @@ class MetricSampler:
# totals here. The database store locks and advances its durable
# baseline in the same transaction as this sample.
"network_devices": _network_counter_devices(values),
# Once explicit public0/private0 roles are available, quota
# accounting must not charge east-west VPC traffic as public
# traffic. Legacy frames without role metadata continue through
# the historical fallback in DatabaseStore.
"network_billing_devices": _network_counter_devices(
values, role="public"
),
}
self._previous[key] = _Baseline(monotonic, raw.generation, raw.vcpus, cpu_time,
disk_signature, disk, network_signature, network)

@ -0,0 +1,202 @@
"""Bounded observation of a VM's private network role.
This module deliberately only reads the domain definition. It does not run
``ip``, ``wg`` or ``nft`` commands and it never treats the host Fabric status
as proof that a tenant NIC is connected. The PHP control plane supplies the
VPC/member revision when it evaluates this observation at an operation
commit boundary.
"""
from __future__ import annotations
import ipaddress
import re
import time
import xml.etree.ElementTree as ET
from typing import Any
from virsh_panel.network_roles import (
PRIVATE_INTERFACE_ALIAS,
interface_role,
private_interface_has_public_filter,
select_private_interface,
select_public_interface,
)
PRIVATE_STATE_VERSION = 1
PANEL_NS = "urn:virsh-panel:1"
_MAC_RE = re.compile(r"[0-9a-f]{2}(?::[0-9a-f]{2}){5}\Z")
_DIGEST_RE = re.compile(r"[0-9a-f]{64}\Z")
def _source(interface: ET.Element) -> tuple[str | None, str | None]:
source = interface.find("source")
if source is None:
return None, None
for kind in ("bridge", "network", "dev", "name", "path"):
value = source.get(kind)
if isinstance(value, str) and value.strip():
return kind, value.strip()
return None, None
def _managed_metadata(root: ET.Element) -> ET.Element | None:
# Only the panel-owned namespace is trusted for the optional identity
# metadata. Guest/user XML must never become a host-state assertion.
for node in root.findall("./metadata"):
for child in list(node):
if child.tag == f"{{{PANEL_NS}}}managed":
return child
return None
def _private_ip(root: ET.Element, managed: ET.Element | None) -> str | None:
values: list[str | None] = []
if managed is not None:
values.extend((managed.get("private-ip-cidr"), managed.get("private_ip_cidr")))
for node in root.findall("./metadata/*"):
if node.tag.rsplit("}", 1)[-1] in {"private-network", "private_state"}:
values.extend((node.get("ip-cidr"), node.get("private-ip-cidr"), node.get("cidr")))
for value in values:
if not isinstance(value, str) or not value:
continue
try:
address = ipaddress.ip_interface(value)
except ValueError:
continue
if isinstance(address, ipaddress.IPv4Interface) and address.network.prefixlen == 32:
return str(address)
return None
def _revision(managed: ET.Element | None) -> int:
if managed is None:
return 0
raw = managed.get("private-network-revision") or managed.get("private_revision")
try:
value = int(raw or 0)
except (TypeError, ValueError):
return 0
return value if 0 <= value <= (1 << 63) - 1 else 0
def _digest(managed: ET.Element | None) -> str | None:
if managed is None:
return None
value = managed.get("private-network-digest") or managed.get("private_digest")
return value.lower() if isinstance(value, str) and _DIGEST_RE.fullmatch(value.lower()) else None
def observe_private_state(
root: ET.Element,
*,
vm_uuid: str | None = None,
host_verified: bool = False,
observed_at: int | None = None,
) -> dict[str, Any]:
"""Return a stable, credential-free private NIC observation.
``disabled/confirmed`` means that the domain has no private role and no
VPC expectation was supplied to this read. A caller that has a VPC
expectation must independently change that result to ``degraded`` or
``uncertain`` until the expected member/revision/digest is verified.
"""
now = int(time.time()) if observed_at is None else int(observed_at)
base: dict[str, Any] = {
"version": PRIVATE_STATE_VERSION,
"required": False,
"vm_uuid": vm_uuid,
"vpc_id": None,
"member_id": None,
"interface": None,
"state": "disabled",
"desired_revision": 0,
"observed_revision": 0,
"desired_digest": None,
"observed_digest": None,
"readback": "confirmed",
"link_state": "absent",
"private_ip_cidr": None,
"private_mac": None,
"reason_code": None,
"observed_at": now,
}
interfaces = list(root.findall("./devices/interface"))
# Validate the complete role map even when no private NIC is present. A
# second unlabelled/ambiguous NIC must not disappear behind a successful
# public-only observation and later become an accidental data-plane path.
try:
public = select_public_interface(interfaces)
private = select_private_interface(interfaces)
except (TypeError, ValueError):
base.update(state="degraded", readback="mismatch",
reason_code="private_interface_role_invalid")
return base
if private is None:
return base
base["interface"] = PRIVATE_INTERFACE_ALIAS
managed = _managed_metadata(root)
base["private_ip_cidr"] = _private_ip(root, managed)
base["observed_revision"] = _revision(managed)
base["observed_digest"] = _digest(managed)
mac_node = private.find("mac")
mac = mac_node.get("address", "").lower() if mac_node is not None else ""
base["private_mac"] = mac or None
link = private.find("link")
link_state = link.get("state", "up").lower() if link is not None else "up"
base["link_state"] = link_state if link_state in {"up", "down"} else "absent"
private_kind, private_source = _source(private)
public_kind, public_source = _source(public) if public is not None else (None, None)
# The tenant data plane is an independent host bridge. A libvirt
# ``network``/``direct`` source does not let this layer prove which host
# bridge it resolves to, so accepting it would make a public bridge reuse
# indistinguishable from a private network. The future data-plane apply
# path may add an explicit verification marker to the managed metadata.
if (private_kind != "bridge" or private_source is None
or private_kind == public_kind and private_source == public_source):
base.update(state="degraded", readback="mismatch",
reason_code="private_source_invalid")
return base
if private_interface_has_public_filter(private):
base.update(state="degraded", readback="mismatch",
reason_code="private_filter_present")
return base
if not _MAC_RE.fullmatch(mac):
base.update(state="degraded", readback="mismatch",
reason_code="private_mac_missing")
return base
if base["private_ip_cidr"] is None:
base.update(state="degraded", readback="missing",
reason_code="private_ip_missing")
return base
# XML metadata is tenant/domain data and can be edited independently of
# the host. Only a future host adapter probe may set ``host_verified``;
# never use a managed-XML marker as proof of bridge/WireGuard/nft state.
if not host_verified:
base.update(state="degraded", readback="missing",
reason_code="private_source_unverified")
return base
if link_state not in {"up", "down"}:
base.update(state="degraded", readback="mismatch",
reason_code="private_link_state_invalid")
return base
base["state"] = "suspended" if link_state == "down" else "active"
return base
def observe_private_state_from_xml(
xml: str | bytes,
*,
vm_uuid: str | None = None,
host_verified: bool = False,
observed_at: int | None = None,
) -> dict[str, Any]:
"""Parse a libvirt XML document and return :func:`observe_private_state`."""
root = ET.fromstring(xml)
return observe_private_state(
root, vm_uuid=vm_uuid, host_verified=host_verified,
observed_at=observed_at,
)

@ -851,6 +851,10 @@ class VpcCoordinator:
"observed_digest": old_digest,
"state": "applying" if gate.get("active") else "uncertain",
"transport_verified": bool(gate.get("active")),
# Fabric transport is necessary but never sufficient for a
# tenant VPC. Only the future host adapter may set this
# after private0/route/nft/member readback.
"private_dataplane_verified": False,
"readback": safe_readback,
})
return {
@ -866,6 +870,7 @@ class VpcCoordinator:
"operation_id": plan.operation_id,
"state": state,
"next_stage_allowed": False,
"private_dataplane_verified": False,
"fabric_gate": dict(gate),
"hosts": host_records,
"subnets": [item.__dict__ for item in plan.subnets],
@ -891,7 +896,7 @@ class VpcCoordinator:
"vpc_id", "region", "fabric_id", "fabric_revision", "cidr",
"desired_revision", "desired_digest", "observed_revision",
"observed_digest", "state", "next_stage_allowed",
"operation_id", "updated_at"):
"private_dataplane_verified", "operation_id", "updated_at"):
if key in value:
result[key] = value[key]
gate = value.get("fabric_gate")
@ -927,7 +932,8 @@ class VpcCoordinator:
item = {key: raw[key] for key in (
"host_id", "member_ids", "desired_revision",
"observed_revision", "observed_digest", "state",
"transport_verified", "last_error_code") if key in raw}
"transport_verified", "private_dataplane_verified",
"last_error_code") if key in raw}
readback = raw.get("readback")
if isinstance(readback, Mapping):
item["readback"] = {
@ -948,12 +954,14 @@ class VpcCoordinator:
host_id = raw["host_id"]
item: dict[str, Any] = {}
for key in ("state", "observed_revision", "observed_digest",
"transport_verified", "last_error_code"):
"transport_verified", "private_dataplane_verified",
"last_error_code"):
if key in raw:
item[key] = raw[key]
item["next_stage_allowed"] = (
raw.get("state") == "active"
and raw.get("transport_verified") is True
and raw.get("private_dataplane_verified") is True
and raw.get("observed_revision") == value.get("desired_revision")
and raw.get("observed_digest") == value.get("desired_digest")
)
@ -963,6 +971,9 @@ class VpcCoordinator:
"digest": value.get("observed_digest"),
"state": value.get("state"),
"next_stage_allowed": value.get("next_stage_allowed") is True,
"private_dataplane_verified": (
value.get("private_dataplane_verified") is True
),
"hosts": observed_hosts,
# PHP consumes this bounded observed projection when deciding
# whether the control-plane result is deliberately waiting for
@ -1084,6 +1095,7 @@ class VpcCoordinator:
if isinstance(item, dict):
item["state"] = "applying"
item["transport_verified"] = True
item["private_dataplane_verified"] = False
hosts = record.get("hosts")
desired_revision = record.get("desired_revision")
desired_digest = record.get("desired_digest")
@ -1093,25 +1105,31 @@ class VpcCoordinator:
and item.get("observed_revision") == desired_revision
and item.get("observed_digest") == desired_digest
and item.get("transport_verified") is True
and item.get("private_dataplane_verified") is True
for item in hosts)
fabric_active = gate.get("active") is True
if fabric_active and all_observed:
record["state"] = "active"
record["next_stage_allowed"] = True
record["private_dataplane_verified"] = True
record["observed_revision"] = desired_revision
record["observed_digest"] = desired_digest
elif not fabric_active:
record["state"] = "degraded" if gate.get("available") else "uncertain"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
elif hosts and any(item.get("state") == "uncertain" for item in hosts if isinstance(item, Mapping)):
record["state"] = "uncertain"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
elif hosts and any(item.get("state") == "degraded" for item in hosts if isinstance(item, Mapping)):
record["state"] = "degraded"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
else:
record["state"] = "applying"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
record["updated_at"] = int(time.time())
async def apply(self, value: Mapping[str, Any]) -> dict[str, Any]:
@ -1226,6 +1244,7 @@ class VpcCoordinator:
if raw is None:
host["state"] = "uncertain"
host["transport_verified"] = False
host["private_dataplane_verified"] = False
continue
if not isinstance(raw, Mapping):
raise VpcError("VPC observed 宿主记录无效")
@ -1233,6 +1252,7 @@ class VpcCoordinator:
observed_digest = raw.get("observed_digest")
state = raw.get("state")
transport = raw.get("transport_verified")
private_dataplane = raw.get("private_dataplane_verified", False)
if (isinstance(observed_revision, bool)
or not isinstance(observed_revision, int)
or observed_revision < 0):
@ -1246,10 +1266,22 @@ class VpcCoordinator:
raise VpcError("VPC observed state 无效")
if not isinstance(transport, bool):
raise VpcError("VPC observed transport_verified 无效")
if not isinstance(private_dataplane, bool):
raise VpcError("VPC observed private_dataplane_verified 无效")
if state == "active" and (transport is not True
or observed_digest is None):
raise VpcError("VPC active 观测缺少传输或摘要证据", 422,
"vpc_observation_invalid")
raise VpcError(
"VPC active 观测缺少传输或摘要证据", 422,
"vpc_observation_invalid")
# The current Fabric adapter reports transport/readback before
# the tenant private data-plane adapter exists. Preserve that
# observation as an applying gate instead of rejecting the
# whole reconciliation request: transport is useful progress,
# but it must never open the VPC or be reported as active.
observation_error = raw.get("error_code")
if state == "active" and private_dataplane is not True:
state = "applying"
observation_error = "vpc_private_dataplane_unverified"
readback = raw.get("readback", {})
if not isinstance(readback, Mapping) or len(readback) > 64:
raise VpcError("VPC readback 无效")
@ -1262,10 +1294,11 @@ class VpcCoordinator:
"observed_revision": observed_revision,
"observed_digest": observed_digest,
"transport_verified": transport,
"private_dataplane_verified": private_dataplane,
"readback": safe_readback,
"last_error_code": raw.get("error_code")
if isinstance(raw.get("error_code"), str)
and len(raw.get("error_code")) <= 64 else None})
"last_error_code": observation_error
if isinstance(observation_error, str)
and len(observation_error) <= 64 else None})
gate = self._fabric_gate(plan)
self._finish(record, gate)
self._save("vpc_desired", vpc_id, record)
@ -1303,6 +1336,7 @@ class VpcCoordinator:
continue
value["state"] = "uncertain"
value["next_stage_allowed"] = False
value["private_dataplane_verified"] = False
# A previous Fabric gate may have been active before the process
# stopped. Preserve its revision/evidence for audit, but never
# expose that stale gate as usable after restart.
@ -1318,7 +1352,8 @@ class VpcCoordinator:
for host in value.get("hosts", []):
if isinstance(host, dict):
host.update({"state": "uncertain",
"transport_verified": False})
"transport_verified": False,
"private_dataplane_verified": False})
value["updated_at"] = int(time.time())
self._save("vpc_desired", key, value)
self._save_children(value)

@ -227,6 +227,9 @@ final class CloudFirewallService
],
'owner_id' => (int) $locked->user_id,
'can_manage' => true,
'private_expectation' => CloudVpcService::instancePrivateExpectation(
(int) $locked->id
),
'firewall_version_id' => (int) $version->id,
'firewall_revision' => $revision,
'firewall_policy_digest' => $policyDigest,

@ -113,6 +113,8 @@ final class CloudInstancePageService
$runtimeHint = in_array($runtimeSync, ['offline', 'unavailable'], true)
? '暂时无法同步' : ($runtimeState === 'unknown' ? '等待宿主机同步' : '');
}
$privateState = CloudVpcService::instancePrivateProjection((int) $instance->id);
$privateState['vm_uuid'] = (string) $instance->uuid;
return [
'id' => (string) $instance->id,
'api_id' => (string) $instance->id,
@ -130,6 +132,7 @@ final class CloudInstancePageService
'instance_status' => (string) $instance->status,
'service_status' => (string) $instance->service_status,
'network_status' => (string) $instance->network_status,
'private_state' => $privateState,
'service_expires_at' => $instance->expires_at === null
? null : (int) $instance->expires_at,
'current_period_started_at' => $period === null ? null : (int) $period->starts_at,
@ -191,11 +194,14 @@ final class CloudInstancePageService
'state_sync_status' => 'pending',
'state_label_hint' => '等待宿主机同步',
];
$privateState = CloudVpcService::instancePrivateProjection((int) $instance->id);
$privateState['vm_uuid'] = (string) $instance->uuid;
$vms[] = [
'id' => (string) $instance->id,
'api_id' => (string) $instance->id,
'uuid' => (string) $instance->uuid,
'name' => (string) $instance->name,
'private_state' => $privateState,
] + array_intersect_key($cached, array_flip([
'state', 'state_transitioning', 'state_transition',
'state_fresh', 'state_observed_at',
@ -248,6 +254,10 @@ final class CloudInstancePageService
'instance_status' => (string) $snapshot['instance_status'],
'service_status' => (string) $snapshot['service_status'],
'network_status' => (string) $snapshot['network_status'],
'private_state' => is_array($snapshot['private_state'] ?? null)
? $snapshot['private_state'] : CloudVpcService::instancePrivateProjection(
(int) $snapshot['id']
),
'service_expires_at' => $snapshot['service_expires_at'],
'current_period_started_at' => $snapshot['current_period_started_at'],
'current_period_ends_at' => $snapshot['current_period_ends_at'],
@ -293,6 +303,10 @@ final class CloudInstancePageService
'state_sync_status' => $liveAvailable ? 'fresh'
: (string) ($snapshot['state_sync_status'] ?? 'pending'),
'state_label_hint' => (string) ($snapshot['state_label_hint'] ?? ''),
'private_state' => is_array($snapshot['private_state'] ?? null)
? $snapshot['private_state'] : CloudVpcService::instancePrivateProjection(
(int) $snapshot['id']
),
];
}
@ -304,6 +318,7 @@ final class CloudInstancePageService
'state_transitioning', 'state_transition', 'state_fresh',
'state_observed_at', 'state_sync_status',
'state_label_hint', 'instance_status',
'private_state',
'os_source', 'os_name', 'install_iso', 'public_ip_cidr', 'public_ip',
'login_name', 'engine_metadata_version', 'vcpu', 'cpu_cores',
'memory_mib', 'ram_mib', 'disks', 'disk_gib',
@ -339,6 +354,8 @@ final class CloudInstancePageService
'state_sync_status' => 'pending',
'state_label_hint' => '等待宿主机同步',
];
$privateState = CloudVpcService::instancePrivateProjection((int) $instance->id);
$privateState['vm_uuid'] = (string) $instance->uuid;
$result[] = self::listItem(array_merge($spec, [
'id' => (string) $instance->id,
'api_id' => (string) $instance->id,
@ -346,6 +363,7 @@ final class CloudInstancePageService
'name' => (string) $instance->name,
'instance_status' => (string) $instance->status,
'region' => $region,
'private_state' => $privateState,
], $cached));
}
return $result;
@ -504,6 +522,8 @@ final class CloudInstancePageService
$runtimeHint = in_array($runtimeSync, ['offline', 'unavailable'], true)
? '暂时无法同步' : ($runtimeState === 'unknown' ? '等待宿主机同步' : '');
}
$privateState = CloudVpcService::instancePrivateProjection((int) $instance->id);
$privateState['vm_uuid'] = (string) $instance->uuid;
$maximumFuturePeriods = max(1, min(120,
(int) Config::get('cloud_max_renewal_periods_ahead', 24)));
@ -532,6 +552,7 @@ final class CloudInstancePageService
'instance_status' => (string) $instance->status,
'service_status' => (string) $instance->service_status,
'network_status' => (string) $instance->network_status,
'private_state' => $privateState,
'service_expires_at' => $instance->expires_at === null
? null : (int) $instance->expires_at,
'current_period_started_at' => $period === null ? null : (int) $period->starts_at,

@ -244,7 +244,14 @@ final class CloudJobService
$jobPayload = ['method' => $method,
'resource' => 'vms/' . rawurlencode($locked->uuid) . $suffix, 'body' => $body,
'credential' => $credential, 'owner_id' => (int) $locked->user_id,
'can_manage' => (bool) ($actor['can_manage'] ?? false)];
'can_manage' => (bool) ($actor['can_manage'] ?? false),
// Bind the private target to the same immutable job payload
// as the public-IP ledger. A later VPC revision must make
// this operation uncertain rather than letting its response
// satisfy a newer target by accident.
'private_expectation' => CloudVpcService::instancePrivateExpectation(
(int) $locked->id
)];
if ($networkLedger !== null) {
$source = $networkLedger['source'];
$target = $networkLedger['target'];
@ -322,6 +329,9 @@ final class CloudJobService
'body' => [],
'owner_id' => (int) $locked->user_id,
'system_cleanup' => true,
'private_expectation' => CloudVpcService::instancePrivateExpectation(
(int) $locked->id
),
]);
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();

@ -446,6 +446,9 @@ final class CloudLifecycleService
'resource' => 'vms/' . rawurlencode($instance->uuid) . '/' . $suffix,
'body' => $body, 'owner_id' => (int) $instance->user_id,
'system_policy' => true, 'suspend_reason' => $reason,
'private_expectation' => CloudVpcService::instancePrivateExpectation(
(int) $instance->id
),
]);
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();

@ -121,6 +121,12 @@ final class CloudOperationErrorCatalog
'firewall_drift_detected' => '检测到防火墙策略漂移,请联系管理员核对',
'firewall_cleanup_pending' => '实例删除结果待核对,防火墙清理状态尚未确认',
'firewall_apply_superseded' => '防火墙请求已被更新的策略替代',
'vpc_private_observation_missing' => '实例私网状态尚未完成回读,请稍后刷新,切勿重复提交',
'vpc_private_observation_mismatch' => '实例私网状态与当前 VPC 目标不一致,正在核对',
'vpc_private_dataplane_unverified' => 'VPC 私网数据面尚未完成核验,请稍后刷新',
'vpc_private_teardown_uncertain' => '实例私网清理状态尚未确认,请勿重复提交',
'vpc_private_expectation_invalid' => '实例私网目标记录无效,请联系管理员核对',
'vpc_private_expectation_unavailable' => '实例私网目标暂时无法读取,请稍后核对',
'admin_cancelled' => '管理员已中止后续重试;已派发的宿主机操作仍需核查',
];
@ -154,6 +160,7 @@ final class CloudOperationErrorCatalog
'created', 'restored', 'deleted',
'service_was_running', 'service_was_running_hint', 'vm',
'reinstall_committed', 'cleanup_pending',
'private_state',
'vcpu', 'cpu_cores', 'memory_mib', 'ram_mib', 'disk_gib',
'os_source', 'os_name', 'install_iso', 'login_name',
'autostart', 'managed',
@ -317,6 +324,13 @@ final class CloudOperationErrorCatalog
}
return 'firewall_apply_failed';
}
if (in_array($engineCode, [
'vpc_private_observation_missing', 'vpc_private_observation_mismatch',
'vpc_private_teardown_uncertain', 'vpc_private_expectation_invalid',
'vpc_private_expectation_unavailable', 'vpc_private_dataplane_unverified',
], true)) {
return $engineCode;
}
// A worker may persist only a reviewed detail or a stable generic
// message. Never put an arbitrary engine/admin summary into the
@ -542,6 +556,15 @@ final class CloudOperationErrorCatalog
}
continue;
}
if ($name === 'private_state') {
if (is_array($item) && !array_is_list($item)) {
$safePrivate = self::sanitizePrivateState($item);
if ($safePrivate !== null) {
$result[$name] = $safePrivate;
}
}
continue;
}
$safe = self::safeResultScalar($name, $item);
if ($safe !== null) {
$result[$name] = $safe;
@ -550,6 +573,81 @@ final class CloudOperationErrorCatalog
return $result;
}
/** Keep private-state evidence typed and free of host/XML diagnostics. */
private static function sanitizePrivateState(array $value): ?array
{
$result = [];
$result['version'] = (int) ($value['version'] ?? 1);
if ($result['version'] !== 1) {
return null;
}
$result['required'] = ($value['required'] ?? false) === true;
foreach (['vm_uuid', 'vpc_id', 'member_id', 'interface',
'private_ip_cidr', 'private_mac', 'desired_digest', 'observed_digest'] as $key) {
$item = $value[$key] ?? null;
$result[$key] = is_string($item) && $item !== ''
? self::normalize($item) : null;
}
foreach (['state' => ['disabled', 'active', 'suspended', 'removed', 'degraded', 'uncertain'],
'desired_state' => ['disabled', 'active', 'removed'],
'readback' => ['confirmed', 'missing', 'mismatch'],
'link_state' => ['up', 'down', 'absent']] as $key => $allowed) {
$default = match ($key) {
'desired_state' => $result['required'] ? 'active' : 'disabled',
'readback' => 'missing',
'link_state' => 'absent',
default => 'uncertain',
};
$item = is_string($value[$key] ?? null)
? strtolower(trim((string) $value[$key])) : $default;
$result[$key] = in_array($item, $allowed, true) ? $item : $default;
}
$result['desired_revision'] = max(0, (int) ($value['desired_revision'] ?? 0));
$result['observed_revision'] = max(0, (int) ($value['observed_revision'] ?? 0));
$result['observed_at'] = max(0, (int) ($value['observed_at'] ?? 0));
foreach (['desired_digest', 'observed_digest'] as $key) {
if ($result[$key] !== null
&& preg_match('/^[a-f0-9]{64}$/Di', $result[$key]) !== 1) {
$result[$key] = null;
}
}
if ($result['vm_uuid'] !== null && preg_match(
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD',
$result['vm_uuid']) !== 1) {
$result['vm_uuid'] = null;
}
foreach (['vpc_id', 'member_id'] as $key) {
if ($result[$key] !== null && preg_match('/^[0-9a-f]{32}$/Di', $result[$key]) !== 1) {
$result[$key] = null;
}
}
if ($result['interface'] !== null && $result['interface'] !== 'private0') {
$result['interface'] = null;
}
if ($result['private_ip_cidr'] !== null) {
$parts = explode('/', $result['private_ip_cidr'], 2);
if (count($parts) !== 2 || (int) $parts[1] !== 32
|| filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false) {
$result['private_ip_cidr'] = null;
}
}
if ($result['private_mac'] !== null
&& preg_match('/^[0-9a-f]{2}(?::[0-9a-f]{2}){5}$/Di', $result['private_mac']) !== 1) {
$result['private_mac'] = null;
}
$reason = is_string($value['reason_code'] ?? null)
? strtolower(trim((string) $value['reason_code'])) : null;
$result['reason_code'] = in_array($reason, [
'private_interface_role_invalid', 'private_source_invalid',
'private_filter_present', 'private_mac_missing', 'private_ip_missing',
'private_source_unverified', 'vpc_private_observation_missing',
'vpc_private_observation_mismatch', 'vpc_private_teardown_uncertain',
'vpc_private_dataplane_unverified',
'vpc_private_expectation_invalid', 'vpc_private_expectation_unavailable',
], true) ? $reason : null;
return $result;
}
private static function safeResultScalar(string $key, $value)
{
if (is_bool($value)) {

@ -240,7 +240,14 @@ final class CloudOrderService
$job->host_id = $placement['host']->id;
$job->kind = 'create';
$job->status = 'queued';
$job->payload_cipher = CloudSecrets::encrypt($placement['payload']);
// A create has no instance row yet, so this is normally the explicit
// disabled target. Persist it so a future flow that pre-creates or
// attaches a VPC member can use the same immutable-target protocol
// instead of publishing a public-only success without a target.
$createPayload = $placement['payload'];
$createPayload['private_expectation'] =
CloudVpcService::instancePrivateExpectation(null);
$job->payload_cipher = CloudSecrets::encrypt($createPayload);
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
// The worker now owns the sole recoverable credential copy.

@ -399,6 +399,13 @@ final class CloudSnapshotService
'can_manage' => (bool) ($actor['can_manage'] ?? false),
'snapshot_key' => (string) $snapshot->snapshot_key,
'snapshot_previous_state' => (string) $snapshot->state,
// Snapshot work changes storage, not the tenant network, but it
// still crosses the same instance commit boundary. Capture the
// private target so a VPC revision change while the snapshot is
// running cannot be mistaken for a readback of the newer target.
'private_expectation' => CloudVpcService::instancePrivateExpectation(
(int) $instance->id
),
]);
$job->available_at = $job->created_at = $job->updated_at = $now;
$job->save();

@ -563,6 +563,14 @@ final class CloudVpcService
->exists()) {
throw new CloudError('该成员仍被路由引用,请先替换或删除路由', 409);
}
// Lifecycle commits lock VPC -> instance before validating the
// captured private target. Take the same order here so a detach
// cannot flip the member to deleting while a VM job is committing.
$instance = CloudInstance::where('id', (int) $member->instance_id)
->lockForUpdate()->first();
if ($instance === null || (string) $instance->status === 'deleted') {
throw new CloudError('虚拟机不存在', 404);
}
$privateIp = self::lockedPrivateIpForMember($member);
$member->state = 'deleting';
$member->updated_at = time();
@ -940,8 +948,11 @@ final class CloudVpcService
?? $observed['observed_revision'] ?? 0);
$observedDigest = (string) ($observed['digest']
?? $observed['observed_digest'] ?? '');
$privateDataplaneVerified = ($observed['private_dataplane_verified']
?? false) === true;
$globalActive = ($observed['state'] ?? null) === 'active'
&& ($observed['next_stage_allowed'] ?? false) === true
&& $privateDataplaneVerified
&& $observedRevision === $revision
&& hash_equals($digest, $observedDigest);
$fabricGate = is_array($observed['fabric_gate'] ?? null)
@ -1029,6 +1040,7 @@ final class CloudVpcService
$itemDigest = (string) ($item['observed_digest'] ?? $item['digest'] ?? '');
$active = $globalActive && ($item['state'] ?? null) === 'active'
&& ($item['transport_verified'] ?? false) === true
&& ($item['private_dataplane_verified'] ?? false) === true
&& $itemRevision <= $revision
&& $itemRevision === $revision && hash_equals($digest, $itemDigest);
$deleting = (string) $deployment->state === 'deleting';
@ -1058,6 +1070,16 @@ final class CloudVpcService
// remain untouched until the later data-plane readback phase.
if ($allActive) {
$observedAt = time();
// Persist a proof marker on each member only after the full
// host-scoped private data-plane gate has passed. Older rows
// without this marker remain fail-closed in instance
// projections until a fresh observation is recorded.
CloudVpcMember::where('vpc_id', $vpc->id)
->whereNotIn('state', ['deleted'])
->update([
'last_operation_id' => $operationId,
'updated_at' => $observedAt,
]);
foreach ([
CloudVpcSubnet::class,
CloudVpcPrivateIp::class,
@ -1263,6 +1285,595 @@ final class CloudVpcService
}
}
/**
* Return the private-network expectation that belongs to one VM job.
*
* This deliberately reads the existing VPC ledger instead of copying a
* second target into cloud_instances/spec_json. A missing VPC schema is
* treated as the legacy "no private network" deployment; once the ledger
* exists, malformed rows are an explicit uncertain expectation and cannot
* be silently interpreted as an attached network.
*/
public static function instancePrivateExpectation(?int $instanceId): array
{
$disabled = [
'required' => false, 'state' => 'disabled', 'desired_state' => 'disabled',
'vpc_id' => null, 'member_id' => null, 'private_ip_cidr' => null,
'private_mac' => null, 'desired_revision' => 0, 'desired_digest' => null,
'reason_code' => null,
];
if ($instanceId === null || $instanceId < 1) {
return $disabled;
}
try {
$memberConnection = (new CloudVpcMember())->getConnection();
$schema = $memberConnection->getSchemaBuilder();
if (!$schema->hasTable('cloud_vpc_members')) {
return $disabled;
}
if (!$schema->hasTable('cloud_vpc_private_ips')
|| !$schema->hasTable('cloud_vpcs')) {
return [
'required' => true, 'state' => 'uncertain',
'desired_state' => 'active', 'vpc_id' => null,
'member_id' => null, 'private_ip_cidr' => null,
'private_mac' => null, 'desired_revision' => 0,
'desired_digest' => null,
'reason_code' => 'vpc_private_expectation_invalid',
];
}
$member = CloudVpcMember::where('instance_id', $instanceId)
->whereNotIn('state', ['deleted'])->orderByDesc('id')->first();
if ($member === null) {
// A private-IP lease without a live member is still a
// control-plane residue. Treat it as an uncertain target so
// lifecycle workers cannot publish a successful/public-only
// result while the orphan may remain attached on a host.
$orphanLease = CloudVpcPrivateIp::where('instance_id', $instanceId)
->whereNotIn('state', ['deleted'])->exists();
if ($orphanLease) {
return [
'required' => true, 'state' => 'uncertain',
'desired_state' => 'active', 'vpc_id' => null,
'member_id' => null, 'private_ip_cidr' => null,
'private_mac' => null, 'desired_revision' => 0,
'desired_digest' => null,
'reason_code' => 'vpc_private_expectation_invalid',
];
}
return $disabled;
}
$vpc = CloudVpc::find((int) $member->vpc_id);
$privateIp = $member->private_ip_id === null ? null
: CloudVpcPrivateIp::where('id', (int) $member->private_ip_id)->first();
$state = (string) $member->state;
$desiredState = $state === 'deleting' ? 'removed' : 'active';
// The member and its lease are one desired-state tuple. Falling
// back to the root revision/digest can hide a partially-written
// child row and would let a VM job capture a target that the VPC
// document never actually requested.
$revision = (int) ($member->desired_revision ?? 0);
$digest = self::safeDigest($member->desired_digest);
$rootRevision = (int) ($vpc?->desired_revision ?? 0);
$rootDigest = self::safeDigest($vpc?->desired_digest);
$leaseRevision = (int) ($privateIp?->desired_revision ?? 0);
$leaseDigest = self::safeDigest($privateIp?->desired_digest);
$validIp = is_string($member->private_ip)
&& filter_var($member->private_ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false;
// `deleting` is an in-flight lease state, not proof that the
// dataplane has converged. Keep the expectation uncertain until
// the row is removed (and a fresh member is observed).
$leaseState = (string) ($privateIp?->state ?? '');
$stateMatches = $privateIp !== null && $leaseState === $state;
$ledgerMatches = $privateIp !== null
&& (int) $privateIp->vpc_id === (int) $member->vpc_id
&& (int) $privateIp->subnet_id === (int) $member->subnet_id
&& (int) ($privateIp->member_id ?? 0) === (int) $member->id
&& (int) ($privateIp->instance_id ?? 0) === $instanceId
&& (string) $privateIp->private_ip === (string) $member->private_ip
&& $stateMatches
&& !in_array($leaseState, ['deleted', 'deleting'], true);
$validRevision = $revision > 0;
$validDigest = $digest !== null;
$desiredLedgerMatches = $vpc !== null
&& $rootRevision === $revision
&& $rootDigest !== null && $digest !== null
&& self::digestMatches($rootDigest, $digest)
&& $leaseRevision === $revision
&& $leaseDigest !== null
&& self::digestMatches($digest, $leaseDigest);
if (!$validIp || !$ledgerMatches || !$validRevision || !$validDigest
|| !$desiredLedgerMatches
|| !is_string($member->member_key)
|| preg_match(self::KEY_PATTERN, (string) $member->member_key) !== 1
|| !is_string($vpc?->vpc_key)
|| preg_match(self::KEY_PATTERN, (string) $vpc->vpc_key) !== 1) {
return [
'required' => true, 'state' => 'uncertain',
'desired_state' => $desiredState,
'vpc_id' => is_string($vpc?->vpc_key) ? $vpc->vpc_key : null,
'member_id' => is_string($member->member_key) ? $member->member_key : null,
'private_ip_cidr' => $validIp ? $member->private_ip . '/32' : null,
'private_mac' => null, 'desired_revision' => max(0, $revision),
'desired_digest' => $validDigest ? $digest : null,
'reason_code' => 'vpc_private_expectation_invalid',
];
}
return [
'required' => true, 'state' => $state,
'desired_state' => $desiredState,
'vpc_id' => (string) $vpc->vpc_key,
'member_id' => (string) $member->member_key,
'private_ip_cidr' => $member->private_ip . '/32',
'private_mac' => null,
'desired_revision' => $revision,
'desired_digest' => $digest,
'reason_code' => null,
];
} catch (\Throwable $error) {
// A live ledger that cannot be read is different from an
// installation that predates VPC tables. Fail closed for the
// former so a worker cannot publish a stale successful result.
return [
'required' => true, 'state' => 'uncertain',
'desired_state' => 'active', 'vpc_id' => null, 'member_id' => null,
'private_ip_cidr' => null, 'private_mac' => null,
'desired_revision' => 0, 'desired_digest' => null,
'reason_code' => 'vpc_private_expectation_unavailable',
];
}
}
/** Compare a job's captured target with the current ledger under a lock. */
public static function privateExpectationMatches(int $instanceId, array $captured): bool
{
if ($instanceId < 1 || $captured === []) {
return true;
}
$current = self::instancePrivateExpectation($instanceId);
// `state` is observed/transient (for example applying -> active) and
// may advance while a VM lifecycle job is running. The immutable
// target is bound by desired_state, identity, IP, revision and digest;
// removal remains strict because it changes desired_state to removed.
foreach (['required', 'desired_state', 'vpc_id', 'member_id',
'private_ip_cidr', 'desired_revision', 'desired_digest'] as $key) {
if ((string) ($captured[$key] ?? null) !== (string) ($current[$key] ?? null)) {
return false;
}
}
return true;
}
/** Build a bounded instance projection for page/runtime responses. */
public static function instancePrivateProjection(int $instanceId): array
{
$expectation = self::instancePrivateExpectation($instanceId);
$base = [
'version' => 1, 'required' => (bool) $expectation['required'],
'vm_uuid' => null, 'vpc_id' => $expectation['vpc_id'] ?? null,
'member_id' => $expectation['member_id'] ?? null,
'interface' => $expectation['required'] ? 'private0' : null,
'state' => $expectation['required'] ? 'uncertain' : 'disabled',
'desired_state' => $expectation['desired_state'] ?? 'disabled',
'desired_revision' => (int) ($expectation['desired_revision'] ?? 0),
'observed_revision' => 0,
'desired_digest' => $expectation['desired_digest'] ?? null,
'observed_digest' => null, 'readback' => $expectation['required'] ? 'missing' : 'confirmed',
'link_state' => $expectation['required'] ? 'absent' : 'absent',
'private_ip_cidr' => $expectation['private_ip_cidr'] ?? null,
'private_mac' => null,
'reason_code' => $expectation['reason_code']
?? ($expectation['required'] ? 'vpc_private_observation_missing' : null),
'observed_at' => 0,
];
if (!$expectation['required']) {
return $base;
}
try {
$vpcKey = is_string($expectation['vpc_id'] ?? null)
? (string) $expectation['vpc_id'] : '';
$memberKey = is_string($expectation['member_id'] ?? null)
? (string) $expectation['member_id'] : '';
if (preg_match(self::KEY_PATTERN, $vpcKey) !== 1
|| preg_match(self::KEY_PATTERN, $memberKey) !== 1) {
return $base;
}
// member_key is only unique inside a VPC. Binding the instance
// and VPC roots here prevents a stale/cross-tenant row from
// becoming a usable private projection.
$vpc = CloudVpc::where('vpc_key', $vpcKey)->first([
'id', 'vpc_key', 'state', 'sync_state', 'desired_revision',
'observed_revision', 'desired_digest', 'observed_digest',
'last_operation_id', 'last_success_at', 'updated_at',
]);
if ($vpc === null) {
return $base;
}
$member = CloudVpcMember::where('instance_id', $instanceId)
->where('vpc_id', (int) $vpc->id)
->where('member_key', $memberKey)
->whereNotIn('state', ['deleted'])
->first([
'id', 'vpc_id', 'subnet_id', 'private_ip_id', 'instance_id',
'private_ip', 'state', 'desired_revision', 'observed_revision',
'desired_digest', 'observed_digest', 'last_error_code',
'last_operation_id', 'updated_at',
]);
if ($member === null || $member->private_ip_id === null) {
return $base;
}
// A member row is not proof that its independent lease exists.
// Read back and bind every lease identity before considering the
// member's revision or digest.
$privateIp = CloudVpcPrivateIp::where('id', (int) $member->private_ip_id)
->where('vpc_id', (int) $vpc->id)
->where('member_id', (int) $member->id)
->where('instance_id', $instanceId)
->first([
'id', 'vpc_id', 'subnet_id', 'member_id', 'instance_id',
'private_ip', 'state', 'desired_revision', 'observed_revision',
'desired_digest', 'observed_digest', 'last_error_code', 'updated_at',
]);
if ($privateIp === null
|| (int) $privateIp->subnet_id !== (int) $member->subnet_id
|| (string) $privateIp->private_ip !== (string) $member->private_ip) {
return $base;
}
$observedRevision = max(0, (int) $member->observed_revision);
$observedDigest = self::safeDigest($member->observed_digest);
$privateObservedRevision = max(0, (int) $privateIp->observed_revision);
$privateObservedDigest = self::safeDigest($privateIp->observed_digest);
$privateDesiredRevision = max(0, (int) $privateIp->desired_revision);
$privateDesiredDigest = self::safeDigest($privateIp->desired_digest);
$rootDesiredRevision = max(0, (int) $vpc->desired_revision);
$rootObservedRevision = max(0, (int) $vpc->observed_revision);
$rootDesiredDigest = self::safeDigest($vpc->desired_digest);
$rootObservedDigest = self::safeDigest($vpc->observed_digest);
$rootOperationId = (string) ($vpc->last_operation_id ?? '');
// A VM job operation ID is not a VPC activation proof. The
// marker must be the current root operation that advanced the
// all-host VPC gate, and that operation must have succeeded.
$proofMarker = preg_match(self::KEY_PATTERN, $rootOperationId) === 1
&& hash_equals($rootOperationId,
(string) ($member->last_operation_id ?? ''));
$operation = $proofMarker
? CloudVpcOperation::where('vpc_id', (int) $vpc->id)
->where('operation_id', $rootOperationId)
->where('status', 'succeeded')
->first(['operation_id', 'desired_revision', 'created_at', 'finished_at'])
: null;
$operationFresh = $operation !== null
&& (int) $operation->desired_revision === (int) $base['desired_revision']
&& (int) ($operation->finished_at ?? 0) > 0
&& (int) ($operation->finished_at ?? 0)
>= (int) ($operation->created_at ?? 0)
&& (int) ($vpc->last_success_at ?? 0) > 0
&& (int) ($vpc->last_success_at ?? 0)
>= (int) ($operation->created_at ?? 0)
&& (int) ($member->updated_at ?? 0) > 0
&& (int) ($privateIp->updated_at ?? 0) > 0
&& (int) ($member->updated_at ?? 0) >= (int) ($operation->created_at ?? 0)
&& (int) ($privateIp->updated_at ?? 0) >= (int) ($operation->created_at ?? 0);
$rootExact = (string) $vpc->state === 'active'
&& (string) $vpc->sync_state === 'applied'
&& $rootDesiredRevision === (int) $base['desired_revision']
&& $rootObservedRevision === $rootDesiredRevision
&& $rootDesiredDigest !== null && $rootObservedDigest !== null
&& self::digestMatches($rootDesiredDigest, $rootObservedDigest)
&& self::digestMatches((string) $base['desired_digest'], $rootDesiredDigest);
$memberExact = (string) $member->state === 'active'
&& $observedRevision === (int) $base['desired_revision']
&& $observedDigest !== null
&& self::digestMatches((string) $base['desired_digest'], $observedDigest)
&& (int) $member->desired_revision === (int) $base['desired_revision']
&& self::digestMatches((string) $base['desired_digest'],
(string) $member->desired_digest);
$privateExact = (string) $privateIp->state === 'active'
&& $privateDesiredRevision === (int) $base['desired_revision']
&& $privateObservedRevision === $privateDesiredRevision
&& $privateDesiredDigest !== null
&& $privateObservedDigest !== null
&& self::digestMatches((string) $base['desired_digest'], $privateDesiredDigest)
&& self::digestMatches($privateDesiredDigest, $privateObservedDigest);
$confirmed = $proofMarker && $operationFresh && $rootExact
&& $memberExact && $privateExact;
$base['observed_revision'] = $observedRevision;
$base['observed_digest'] = $observedDigest;
$base['state'] = $confirmed ? 'active'
: ((string) $member->state === 'degraded'
|| (string) $privateIp->state === 'degraded'
? 'degraded' : 'uncertain');
$base['readback'] = $confirmed ? 'confirmed' : 'missing';
$base['reason_code'] = $confirmed ? null
: (self::safeErrorCode((string) ($privateIp->last_error_code
?? $member->last_error_code ?? ''))
?? 'vpc_private_observation_missing');
$freshness = array_values(array_filter([
(int) ($member->updated_at ?? 0),
(int) ($privateIp->updated_at ?? 0),
(int) ($vpc->last_success_at ?? 0),
], static fn (int $value): bool => $value > 0));
$base['observed_at'] = $freshness === [] ? 0 : min($freshness);
if ($confirmed) {
// The member ledger proves the VPC revision, while the
// instance network policy records whether the last gated
// operation intentionally disconnected all NICs. Pending or
// failed policy transitions must remain uncertain until the
// next engine readback, even when the previous member digest
// was exact.
$networkStatus = (string) (CloudInstance::where('id', $instanceId)
->value('network_status') ?? '');
if (in_array($networkStatus,
['blocking', 'blocking_expired', 'restore_pending', 'restoring'], true)) {
$base['state'] = 'uncertain';
$base['readback'] = 'missing';
$base['reason_code'] = 'vpc_private_observation_missing';
} elseif (in_array($networkStatus,
['blocked_traffic', 'blocked_expired'], true)) {
$base['state'] = 'suspended';
$base['link_state'] = 'down';
} else {
$base['link_state'] = 'up';
}
}
} catch (\Throwable) {
// Keep the fail-closed default projection.
}
return $base;
}
/**
* Validate one engine private_state at a mutation commit boundary.
*
* The engine's XML observation is necessary but not sufficient for an
* attached VPC: the member identity, desired revision and digest must
* also be echoed by a future data-plane reconciler. Until then this
* method intentionally returns an uncertain gate for VPC members.
*/
public static function validatePrivateObservation(
array $expectation, ?array $observation, string $kind
): array {
$fallback = [
'version' => 1, 'required' => (bool) ($expectation['required'] ?? false),
'vm_uuid' => null, 'vpc_id' => $expectation['vpc_id'] ?? null,
'member_id' => $expectation['member_id'] ?? null,
'interface' => null, 'state' => 'uncertain',
'desired_state' => (string) ($expectation['desired_state'] ?? 'disabled'),
'desired_revision' => (int) ($expectation['desired_revision'] ?? 0),
'observed_revision' => 0,
'desired_digest' => $expectation['desired_digest'] ?? null,
'observed_digest' => null, 'readback' => 'missing',
'link_state' => 'absent', 'private_ip_cidr' => null,
'private_mac' => null,
'reason_code' => 'vpc_private_observation_missing',
'observed_at' => time(),
];
if (!is_array($observation)) {
return ['ok' => false, 'state' => $fallback,
'error_code' => $fallback['reason_code']];
}
if ((int) ($observation['version'] ?? 0) !== 1) {
$fallback['reason_code'] = 'vpc_private_observation_mismatch';
return ['ok' => false, 'state' => $fallback,
'error_code' => $fallback['reason_code']];
}
$state = $fallback;
foreach (['version', 'required', 'vm_uuid', 'vpc_id', 'member_id',
'interface', 'state', 'desired_revision', 'observed_revision',
'desired_digest', 'observed_digest', 'readback', 'link_state',
'private_ip_cidr', 'private_mac', 'reason_code', 'observed_at'] as $key) {
if (array_key_exists($key, $observation)) {
$state[$key] = $observation[$key];
}
}
$state['required'] = (bool) $state['required'];
$state['desired_revision'] = max(0, (int) $state['desired_revision']);
$state['observed_revision'] = max(0, (int) $state['observed_revision']);
$state['desired_digest'] = self::safeDigest($state['desired_digest']);
$state['observed_digest'] = self::safeDigest($state['observed_digest']);
$expectedRequired = (bool) ($expectation['required'] ?? false);
$expectedState = (string) ($expectation['desired_state'] ?? 'disabled');
$state['required'] = $expectedRequired;
$state['desired_state'] = $expectedState;
$observedVpcId = is_string($observation['vpc_id'] ?? null)
? (string) $observation['vpc_id'] : null;
$observedMemberId = is_string($observation['member_id'] ?? null)
? (string) $observation['member_id'] : null;
$state['vpc_id'] = $expectation['vpc_id'] ?? null;
$state['member_id'] = $expectation['member_id'] ?? null;
$state['desired_revision'] = (int) ($expectation['desired_revision'] ?? 0);
$state['desired_digest'] = self::safeDigest($expectation['desired_digest'] ?? null);
$state['private_ip_cidr'] = is_string($observation['private_ip_cidr'] ?? null)
? trim($observation['private_ip_cidr']) : null;
$state['private_mac'] = is_string($observation['private_mac'] ?? null)
? strtolower(trim($observation['private_mac'])) : null;
$state['state'] = is_string($observation['state'] ?? null)
? strtolower(trim($observation['state'])) : 'uncertain';
$state['readback'] = is_string($observation['readback'] ?? null)
? strtolower(trim($observation['readback'])) : 'missing';
if (!$expectedRequired) {
// When a structured observation is available, a legacy VM is
// successful only when the host readback proves that no private
// NIC exists. An unexpected private interface, even if its XML
// looks healthy, is topology drift and must not be hidden behind
// a successful public-only operation. The worker compatibility
// gate handles older responses that omit private_state entirely.
$ok = $state['state'] === 'disabled'
&& $state['readback'] === 'confirmed'
&& ($observation['required'] ?? false) === false;
if (!$ok) {
$state['state'] = 'uncertain';
$state['reason_code'] = 'vpc_private_observation_missing';
}
return ['ok' => $ok, 'state' => $state,
'error_code' => $ok ? null : $state['reason_code']];
}
$linkExpected = match ($kind) {
'network_suspend' => 'down',
// service_resume restores the VM power/autostart target. A
// quota-blocked VM may still intentionally keep every NIC down;
// network_resume is the separate operation that proves link-up.
'network_resume' => 'up',
default => null,
};
$identity = is_string($expectation['vpc_id'] ?? null)
&& is_string($expectation['member_id'] ?? null)
&& $observedVpcId !== null && $observedMemberId !== null
&& hash_equals((string) $expectation['vpc_id'], $observedVpcId)
&& hash_equals((string) $expectation['member_id'], $observedMemberId);
$shape = ($observation['required'] ?? false) === true;
if ($expectedState === 'removed') {
// Teardown evidence is allowed to omit the interface/MAC only
// when it explicitly reports the removed state. Identity,
// revision and digest below still bind the tombstone to the
// original VPC member.
$shape = $shape && in_array($state['state'], ['disabled', 'removed'], true);
} else {
$shape = $shape && ($state['interface'] ?? null) === 'private0'
&& is_string($state['private_mac'] ?? null)
&& preg_match('/^[0-9a-f]{2}(?::[0-9a-f]{2}){5}$/Di',
(string) $state['private_mac']) === 1;
}
$revision = $state['observed_revision'] === $state['desired_revision']
&& $state['desired_revision'] > 0;
$digest = is_string($state['observed_digest'])
&& is_string($state['desired_digest'])
&& hash_equals($state['desired_digest'], $state['observed_digest']);
$ip = $expectedState === 'removed'
|| (is_string($expectation['private_ip_cidr'])
&& hash_equals((string) $expectation['private_ip_cidr'], (string) $state['private_ip_cidr']));
$desired = $expectedState === 'removed'
? in_array($state['state'], ['disabled', 'removed'], true)
: in_array($state['state'], ['active', 'suspended'], true);
$link = $linkExpected === null || $state['link_state'] === $linkExpected;
$ok = $shape && $identity && $revision && $digest && $ip && $desired && $link
&& $state['readback'] === 'confirmed';
if (!$ok) {
$state['state'] = $state['state'] === 'degraded' ? 'degraded' : 'uncertain';
$state['readback'] = $state['readback'] === 'mismatch' ? 'mismatch' : 'missing';
$state['reason_code'] = $expectedState === 'removed'
? 'vpc_private_teardown_uncertain' : 'vpc_private_observation_mismatch';
}
return ['ok' => $ok, 'state' => $state,
'error_code' => $ok ? null : $state['reason_code']];
}
/** Persist the latest VM-side observation without advancing a VPC root. */
public static function recordPrivateObservation(
int $instanceId, array $validation, string $operationId
): bool {
if ($instanceId < 1 || preg_match(self::KEY_PATTERN, $operationId) !== 1) {
return false;
}
$connection = (new CloudVpcMember())->getConnection();
$schema = $connection->getSchemaBuilder();
if (!$schema->hasTable('cloud_vpc_members')
|| !$schema->hasTable('cloud_vpc_private_ips')
|| !$schema->hasTable('cloud_vpcs')) {
return false;
}
return (bool) $connection->transaction(function () use (
$instanceId, $validation, $operationId
): bool {
// Match the VPC -> member lock order used by control-plane
// mutations. A lifecycle readback from an older desired revision
// must never be written into a newer member row.
$probe = CloudVpcMember::where('instance_id', $instanceId)
->whereNotIn('state', ['deleted'])->first();
if ($probe === null) {
return false;
}
$vpc = CloudVpc::where('id', (int) $probe->vpc_id)
->lockForUpdate()->first();
if ($vpc === null) {
return false;
}
$member = CloudVpcMember::where('id', (int) $probe->id)
->whereNotIn('state', ['deleted'])->lockForUpdate()->first();
if ($member === null) {
return false;
}
// Lock and validate the independent lease before changing either
// side of the member/lease pair. A VM readback must never copy
// its revision into a lease belonging to another member.
if ($member->private_ip_id === null) {
return false;
}
$private = CloudVpcPrivateIp::where('id', (int) $member->private_ip_id)
->where('vpc_id', (int) $vpc->id)
->where('member_id', (int) $member->id)
->where('instance_id', $instanceId)
->lockForUpdate()->first();
if ($private === null
|| (int) $private->subnet_id !== (int) $member->subnet_id
|| (string) $private->private_ip !== (string) $member->private_ip) {
return false;
}
$state = is_array($validation['state'] ?? null)
? $validation['state'] : [];
$ok = ($validation['ok'] ?? false) === true;
$desired = (string) ($state['desired_state'] ?? 'active');
$expectedRevision = max(0, (int) ($state['desired_revision'] ?? 0));
$expectedDigest = self::safeDigest($state['desired_digest'] ?? null);
$memberDigest = self::safeDigest($member->desired_digest);
if ((int) $member->desired_revision !== $expectedRevision
|| $memberDigest !== $expectedDigest) {
return false;
}
// `last_operation_id` on a member is the VPC proof marker. A
// VM lifecycle job may refresh the private lease, but must not
// replace a successful root operation marker with its unrelated
// job ID; doing so would make an already-proven VPC appear stale.
$rootOperationId = (string) ($vpc->last_operation_id ?? '');
if ($rootOperationId === '' || $rootOperationId === $operationId) {
$member->last_operation_id = $operationId;
}
$member->observed_revision = max(0, (int) ($state['observed_revision'] ?? 0));
$member->observed_digest = self::safeDigest($state['observed_digest'] ?? null);
// Mirror the member outcome into its lease row. A private-IP
// state left at `deleting` or `active` after readback would make
// the next expectation disagree with the member ledger.
$privateState = $ok
? ($desired === 'removed' ? 'deleted' : 'active')
: ($desired === 'removed' ? 'deleting' : 'uncertain');
if ($ok && $desired === 'removed') {
$member->state = 'deleted';
$member->last_error_code = null;
$private->state = $privateState;
$private->observed_revision = $member->observed_revision;
$private->observed_digest = $member->observed_digest;
$private->last_error_code = null;
$private->updated_at = time();
$private->save();
} elseif ($ok) {
$member->state = 'active';
$member->last_error_code = null;
} else {
$member->state = $desired === 'removed' ? 'deleting' : 'uncertain';
$member->last_error_code = is_string($validation['error_code'] ?? null)
? self::safeErrorCode((string) $validation['error_code']) : 'vpc_private_observation_missing';
}
$member->updated_at = time();
$member->save();
$private->state = $privateState;
$private->observed_revision = $member->observed_revision;
$private->observed_digest = $member->observed_digest;
$private->last_error_code = $member->last_error_code;
$private->updated_at = time();
$private->save();
return true;
}, 3);
}
private static function safeDigest($value): ?string
{
return is_string($value) && preg_match('/^[a-f0-9]{64}$/Di', $value) === 1
? strtolower($value) : null;
}
private static function touchDesiredLocked(CloudVpc $vpc, string $kind,
string $requestKey, string $requestHash): array
{
@ -1920,6 +2531,10 @@ final class CloudVpcService
'observed_revision' => max(0, (int) ($host['observed_revision'] ?? $host['revision'] ?? 0)),
'observed_digest' => is_string($host['observed_digest'] ?? $host['digest'] ?? null)
? substr((string) ($host['observed_digest'] ?? $host['digest']), 0, 64) : null,
'transport_verified' => ($host['transport_verified'] ?? false) === true,
'private_dataplane_verified' => (
($host['private_dataplane_verified'] ?? false) === true
),
'next_stage_allowed' => ($host['next_stage_allowed'] ?? false) === true,
];
}

@ -6,6 +6,7 @@ use App\Models\CloudInvoice;
use App\Models\CloudJob;
use App\Models\CloudOrder;
use App\Models\CloudServicePeriod;
use App\Models\CloudVpc;
use App\Models\User;
final class CloudWorkerService
@ -179,7 +180,14 @@ final class CloudWorkerService
$instance = CloudInstance::find($job->instance_id);
$desired = self::policyDesired($instance, (string) $job->kind);
if (!$desired) {
$this->finish($job, ['status' => 'succeeded', 'result' => ['skipped' => true]], $payload);
// Even a no-op policy result crosses the instance
// lifecycle boundary: a stale/missing private readback
// must not be hidden by the early skip path.
$skipped = $this->applyPrivateLifecycleGate(
$job, ['status' => 'succeeded', 'result' => ['skipped' => true]],
$payload, $actor
);
$this->finish($job, $skipped, $payload);
return true;
}
}
@ -280,8 +288,11 @@ final class CloudWorkerService
&& ($operation['service_was_running_hint'] ?? false) === true) {
$result['service_was_running'] = true;
}
$this->finish($job, ['status' => 'succeeded',
'result' => $result], $payload);
$skipped = $this->applyPrivateLifecycleGate(
$job, ['status' => 'succeeded', 'result' => $result],
$payload, $actor
);
$this->finish($job, $skipped, $payload);
return true;
}
$operation = $this->engine->host((int) $job->host_id,
@ -347,6 +358,15 @@ final class CloudWorkerService
$operation = ['status' => $operation['status'] ?? 'succeeded',
'result' => $operation];
}
// Every terminal VM mutation gets the same private-network
// readback gate. The engine's normal VM payload contains only a
// domain/XML observation; an attached VPC additionally requires
// member identity, revision and digest evidence from the future
// data-plane reconciler. A missing or stale observation is kept
// uncertain before finish() can commit public/billing state.
$operation = $this->applyPrivateLifecycleGate(
$job, $operation, $payload, $actor
);
$this->finish($job, $operation, $payload);
} catch (\Throwable $error) {
// Diagnostics deliberately contain no raw payloads, credentials or upstream tracebacks.
@ -504,7 +524,7 @@ final class CloudWorkerService
$this->observedErrors[] = $status === 'failed' ? 'task_failed' : 'task_uncertain';
}
$result = $operation['result'] ?? null;
(new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, $status, $result, $operation) {
(new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, &$status, &$result, &$operation) {
// All financial transactions acquire the wallet first, avoiding wallet/order deadlocks.
$user = User::where('id', $claimed->user_id)->lockForUpdate()->first();
$order = $claimed->order_id === null ? null : CloudOrder::where('id', $claimed->order_id)->lockForUpdate()->first();
@ -513,6 +533,58 @@ final class CloudWorkerService
|| in_array($job->status, ['succeeded', 'failed', 'cancelled'], true)) {
return;
}
if ($status === 'succeeded' && $job->instance_id !== null) {
$capturedPrivateExpectation = $payload['private_expectation'] ?? null;
// VPC mutators lock VPC -> instance. Acquire the same order
// before checking the captured target so detach/attach cannot
// change a member between the readback and this commit.
$privateProbe = is_array($capturedPrivateExpectation)
? $capturedPrivateExpectation
: CloudVpcService::instancePrivateExpectation((int) $job->instance_id);
$vpcKeyToLock = is_string($privateProbe['vpc_id'] ?? null)
? (string) $privateProbe['vpc_id'] : '';
if (preg_match('/^[0-9a-f]{32}$/D', $vpcKeyToLock) === 1) {
CloudVpc::where('vpc_key', $vpcKeyToLock)
->lockForUpdate()->first();
}
$lockedInstance = CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
$privateTargetChanged = $lockedInstance === null;
if (!$privateTargetChanged && is_array($capturedPrivateExpectation)) {
$privateTargetChanged = !CloudVpcService::privateExpectationMatches(
(int) $lockedInstance->id, $capturedPrivateExpectation
);
} elseif (!$privateTargetChanged) {
// Jobs created before the immutable expectation field was
// introduced cannot prove that an attached VPC target
// stayed unchanged. Allow legacy public-only jobs to
// finish, but fail closed for any live private member.
$privateTargetChanged = CloudVpcService::instancePrivateExpectation(
(int) $lockedInstance->id
)['required'] === true;
}
if ($privateTargetChanged) {
// The VPC target changed after the worker's engine
// readback. Keep the operation uncertain while holding
// the same instance row lock used by attach/detach.
$status = 'uncertain';
$operation['status'] = 'uncertain';
$operation['error_code'] = 'vpc_private_observation_mismatch';
$operation['error'] = '实例私网目标已变化,结果正在核对';
if (is_array($operation['result'] ?? null)) {
$operation['result']['private_state'] = [
'version' => 1, 'required' => true,
'state' => 'uncertain', 'desired_state' => 'active',
'readback' => 'mismatch',
'reason_code' => 'vpc_private_observation_mismatch',
'desired_revision' => 0, 'observed_revision' => 0,
'desired_digest' => null, 'observed_digest' => null,
'link_state' => 'absent', 'observed_at' => time(),
];
}
$result = $operation['result'] ?? null;
}
}
$safe = is_array($result)
? CloudOperationErrorCatalog::workerResult($result) : null;
CloudSnapshotService::finishJobLocked($job, $status, $result, $payload);
@ -973,6 +1045,265 @@ final class CloudWorkerService
}, 3);
}
/**
* Failed/uncertain jobs normally must keep their original disposition,
* but a job that crossed the engine mutation boundary still needs the
* same private-state readback as a successful job. The durable engine
* marker avoids treating ordinary request validation failures as if they
* had changed a VM. Network changes retain their narrower checkpoint:
* the child must have claimed the target IP before this gate runs.
*
* The actual bounded private-state read happens outside finish()'s DB
* transaction below, so a slow or failed engine read never holds wallet or
* VPC rows.
*/
private static function privateMutationReadbackRequired(
CloudJob $job, array $operation, string $status
): bool {
if (!in_array($status, ['failed', 'uncertain'], true)) {
return false;
}
$kind = (string) $job->kind;
$result = is_array($operation['result'] ?? null)
? $operation['result'] : [];
$vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
// Public-IP changes already have a commit checkpoint inside the
// child. A generic invocation marker alone would gate a preflight
// rejection before any address or XML mutation took place.
if ($kind === 'network_change') {
if (($operation['network_mutation_started'] ?? false) === true) {
return true;
}
foreach (['network_changed', 'network_mutation_started'] as $key) {
if (($result[$key] ?? false) === true || ($vm[$key] ?? false) === true) {
return true;
}
}
return false;
}
if (($operation['mutation_started'] ?? false) === true
|| ($operation['network_mutation_started'] ?? false) === true) {
return true;
}
if (in_array($kind, ['delete', 'expiry_delete'], true)
&& (($operation['vm_delete_committed'] ?? false) === true
|| array_key_exists('cleanup_pending', $operation))) {
return true;
}
if ($kind === 'reinstall' && self::reinstallNeedsReview($operation)) {
return true;
}
// Preserve existing post-commit markers from older engine versions.
foreach (['reinstall_committed', 'cleanup_pending', 'network_changed',
'settings_changed', 'restored', 'deleted', 'created'] as $key) {
if (($result[$key] ?? false) === true || ($vm[$key] ?? false) === true) {
return true;
}
}
// An uncertain job dispatched by a legacy engine has no durable
// mutation marker. Its dispatched_at timestamp is the conservative
// compatibility signal; failed pre-dispatch jobs keep a false gate.
return $status === 'uncertain' && $job->dispatched_at !== null;
}
private function applyPrivateLifecycleGate(
CloudJob $job, array $operation, array $payload, array $actor
): array {
$status = is_string($operation['status'] ?? null)
? strtolower((string) $operation['status']) : 'uncertain';
$kind = (string) $job->kind;
$postCommitReinstall = $kind === 'reinstall'
&& $status === 'failed' && self::reinstallNeedsReview($operation);
$terminalSuccess = in_array($status, ['succeeded', 'completed', 'done'], true);
if (!$terminalSuccess && !$postCommitReinstall
&& !self::privateMutationReadbackRequired($job, $operation, $status)) {
return $operation;
}
$lifecycleKinds = [
'create', 'reinstall', 'network_change', 'instance_settings', 'rescue',
'snapshot_create', 'snapshot_restore', 'snapshot_delete', 'delete',
'expiry_delete', 'action', 'service_suspend', 'service_resume',
'network_suspend', 'network_resume', 'firewall_apply',
];
if (!in_array($kind, $lifecycleKinds, true)) {
return $operation;
}
$instance = $job->instance_id === null ? null : CloudInstance::find($job->instance_id);
$currentExpectation = CloudVpcService::instancePrivateExpectation(
$instance === null ? null : (int) $instance->id
);
$capturedExpectation = is_array($payload['private_expectation'] ?? null)
? $payload['private_expectation'] : null;
$expectationStale = false;
$expectation = $currentExpectation;
if ($capturedExpectation !== null) {
$expectation = $capturedExpectation;
// `state` is an observed/transient lifecycle value. A member can
// legitimately move applying -> active while this VM job is in
// flight; desired_state plus the revision/digest still bind the
// immutable target. Removal is represented by desired_state and
// therefore remains strict.
foreach (['required', 'desired_state', 'vpc_id', 'member_id',
'private_ip_cidr', 'desired_revision', 'desired_digest'] as $key) {
$left = $capturedExpectation[$key] ?? null;
$right = $currentExpectation[$key] ?? null;
if ((string) $left !== (string) $right) {
$expectationStale = true;
break;
}
}
if ($expectationStale) {
// Keep the original target in the evidence, but make the
// gate fail closed instead of validating it against a newer
// VPC revision that arrived while this job was running.
$expectation['required'] = true;
$expectation['reason_code'] = 'vpc_private_observation_mismatch';
}
}
$result = is_array($operation['result'] ?? null) ? $operation['result'] : [];
$vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
$uuid = is_string($instance?->uuid) ? (string) $instance->uuid
: (is_string($vm['uuid'] ?? null) ? $vm['uuid']
: (is_string($result['uuid'] ?? null) ? $result['uuid'] : null));
// Accept both the normal {vm:{...}} envelope and a rolling-upgrade
// response that places the bounded observation beside `vm`. A
// structured field at either level must stay strict; only a complete
// omission can use the legacy public-only compatibility path.
$hasPrivateState = array_key_exists('private_state', $vm)
|| array_key_exists('private_state', $result);
$observation = is_array($vm['private_state'] ?? null)
? $vm['private_state']
: (is_array($result['private_state'] ?? null)
? $result['private_state'] : null);
// Pre-VPC engines and the existing public-only worker fixtures do not
// return a private_state object at all. There is no private target to
// prove for those jobs, so do not issue a second host request (which
// would also consume the next response in a legacy mock queue). Once
// a VPC member is expected, the gate remains fail-closed below; and a
// structured observation supplied by any engine is still validated,
// including the public-only case where it can reveal an unexpected
// private NIC.
if (($expectation['required'] ?? false) !== true && !$hasPrivateState) {
return $operation;
}
// A delete may legitimately have no readable domain after the
// irreversible boundary. Only a VM without a VPC target can use the
// explicit disabled/confirmed result; a VPC tombstone still requires
// a future removed/readback proof.
if ($uuid !== null) {
try {
$observed = $this->engine->host(
(int) $job->host_id, 'GET',
'vms/' . rawurlencode($uuid) . '/private-state',
$actor, null, (string) $job->operation_id
);
if (is_array($observed)) {
$observation = $observed;
}
} catch (\Throwable $error) {
$absenceProven = in_array($kind, ['delete', 'expiry_delete'], true)
&& $expectation['required'] !== true
&& $error instanceof CloudError
&& $error->httpStatus === 404;
if ($absenceProven) {
// A VM with no VPC member has no independent private
// teardown target. Once the engine has confirmed the
// domain deletion, a 404 is the expected absence proof.
$observation = [
'version' => 1, 'required' => false,
'vm_uuid' => $uuid, 'vpc_id' => null, 'member_id' => null,
'interface' => null, 'state' => 'disabled',
'desired_revision' => 0, 'observed_revision' => 0,
'desired_digest' => null, 'observed_digest' => null,
'readback' => 'confirmed', 'link_state' => 'absent',
'private_ip_cidr' => null, 'private_mac' => null,
'reason_code' => null, 'observed_at' => time(),
];
}
if (!$absenceProven) {
// The operation itself may already be durable. Convert a
// missing observation into a stable uncertain result
// rather than leaking a transport/host diagnostic.
$observation = null;
}
}
}
$validation = $expectationStale
? ['ok' => false, 'state' => [
'version' => 1, 'required' => true,
'vm_uuid' => $uuid, 'vpc_id' => $expectation['vpc_id'] ?? null,
'member_id' => $expectation['member_id'] ?? null,
'interface' => null, 'state' => 'uncertain',
'desired_state' => $expectation['desired_state'] ?? 'active',
'desired_revision' => (int) ($expectation['desired_revision'] ?? 0),
'observed_revision' => 0,
'desired_digest' => $expectation['desired_digest'] ?? null,
'observed_digest' => null, 'readback' => 'mismatch',
'link_state' => 'absent', 'private_ip_cidr' => null,
'private_mac' => null,
'reason_code' => 'vpc_private_observation_mismatch',
'observed_at' => time(),
], 'error_code' => 'vpc_private_observation_mismatch']
: CloudVpcService::validatePrivateObservation(
$expectation, $observation, $kind
);
if ($instance !== null && $expectation['required'] === true && !$expectationStale) {
try {
$recorded = CloudVpcService::recordPrivateObservation(
(int) $instance->id, $validation, (string) $job->operation_id
);
if ($recorded !== true && $validation['ok'] === true) {
$validation = [
'ok' => false,
'state' => array_merge($validation['state'], [
'state' => 'uncertain', 'readback' => 'mismatch',
'reason_code' => 'vpc_private_observation_mismatch',
'observed_at' => time(),
]),
'error_code' => 'vpc_private_observation_mismatch',
];
}
} catch (\Throwable $error) {
// The operation gate remains uncertain even when the
// secondary VPC observation cache cannot be written.
error_log('cloud private observation could not be recorded: '
. get_class($error));
if ($validation['ok'] === true) {
$validation = [
'ok' => false,
'state' => array_merge($validation['state'], [
'state' => 'uncertain', 'readback' => 'missing',
'reason_code' => 'vpc_private_observation_missing',
'observed_at' => time(),
]),
'error_code' => 'vpc_private_observation_missing',
];
}
}
}
$result['private_state'] = $validation['state'];
if ($kind === 'reinstall' && !$validation['ok']) {
// Preserve the replacement credential/spec on a post-commit
// uncertain reinstall; the existing finish() compatibility path
// will not mark the task successful or replay it blindly.
$result['reinstall_committed'] = true;
}
$operation['result'] = $result;
if (!$validation['ok']) {
$operation['status'] = 'uncertain';
$operation['safe_to_refund'] = false;
$operation['error_code'] = $validation['error_code']
?? 'vpc_private_observation_missing';
$operation['error'] = '实例私网状态尚未完成实际回读,请稍后核对';
}
return $operation;
}
private static function validSystemCleanup(CloudJob $job, array $payload): bool
{
if ($job->instance_id === null || ($payload['body'] ?? null) !== []) {

@ -22,6 +22,7 @@ $middleware = $read('app/Middleware/CloudApi.php');
$engineClient = $read('app/Services/Cloud/CloudEngineClient.php');
$accountDeletion = $read('app/Services/UserDataDeletionService.php');
$jobService = $read('app/Services/Cloud/CloudJobService.php');
$orderService = $read('app/Services/Cloud/CloudOrderService.php');
$workerService = $read('app/Services/Cloud/CloudWorkerService.php');
foreach ([
@ -88,6 +89,19 @@ $assert(!str_contains($service, 'CloudJob')
&& !str_contains($service, 'network_change')
&& !str_contains($service, 'publicIpCidr'),
'VPC service must not reuse the public network_change/job payload.');
$assert(str_contains($service, '$orphanLease')
&& str_contains($service, '$stateMatches')
&& str_contains($service, "where('instance_id', \$instanceId)")
&& str_contains($service, "where('vpc_id', (int) \$vpc->id)")
&& str_contains($service, 'hash_equals($rootOperationId')
&& str_contains($service, "where('status', 'succeeded')")
&& str_contains($service, '$operationFresh')
&& str_contains($service, '$vpc->last_success_at')
&& str_contains($service, '$privateIp->observed_revision')
&& str_contains($service, '$privateIp->observed_digest')
&& str_contains($service, "(string) \$privateIp->state === 'active'")
&& str_contains($service, '$private->state = $privateState'),
'VPC private projections must fail closed on orphan/mismatched leases and require current root readback proof.');
$assert(str_contains($service, '当前 VPC 控制面不支持子网网关')
&& str_contains($service, "'private_ip_cidr'")
&& str_contains($service, "'next_hop_member_id'")
@ -107,7 +121,20 @@ $assert(str_contains($engineClient, 'function vpc(')
$assert(str_contains($accountDeletion, "cloud_vpcs")
&& str_contains($accountDeletion, "cloud_private_networks")
&& str_contains($jobService, 'assertInstanceCanBeDeleted')
&& str_contains($orderService, "private_expectation")
&& str_contains($workerService, '旧删除任务缺少 VPC 删除门禁'),
'Account/VM deletion can bypass the independent VPC control-plane ledger.');
'Account/VM creation/deletion can bypass the independent VPC control-plane ledger.');
$legacyPrivateGate = strpos($workerService,
"if ((\$expectation['required'] ?? false) !== true && !\$hasPrivateState)");
$privateReadback = strpos($workerService,
"'vms/' . rawurlencode(\$uuid) . '/private-state'");
$assert(str_contains($workerService, '$hasPrivateState')
&& str_contains($workerService, "&& !\$hasPrivateState")
&& str_contains($workerService, "array_key_exists('private_state', \$result)")
&& str_contains($workerService, 'Pre-VPC engines')
&& $legacyPrivateGate !== false
&& $privateReadback !== false
&& $legacyPrivateGate < $privateReadback,
'Legacy public-only jobs without private_state must remain compatible while structured observations stay gated.');
echo "VPC control-plane schema, API, typed DTO and independent operation contracts passed.\n";

Loading…
Cancel
Save