You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/virsh_panel/vpc.py

4451 lines
219 KiB
Python

"""Structured VPC control-plane state.
This module is deliberately a control-plane boundary. It validates a typed
desired state, stores an encrypted durable copy and keeps a separate observed
revision for the future private-network reconciler. It does not execute
``ip``, ``wg``, libvirt or nftables commands. The host WireGuard Fabric is a
prerequisite and is checked as a read-only gate before a VPC can be released
to the later data-plane phase.
"""
from __future__ import annotations
import asyncio
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from contextlib import AsyncExitStack, asynccontextmanager
import hashlib
import ipaddress
import json
import re
import threading
import time
from dataclasses import dataclass, replace
from typing import Any, Mapping, Sequence
from .fabric import (
FabricCoordinator,
FabricError,
_await_mutation_completion,
)
VPC_SCHEMA_VERSION = 1
VPC_INTERFACE = "private0"
VPC_MAX_HOSTS = 64
VPC_MAX_SUBNETS = 256
VPC_MAX_MEMBERS = 4096
VPC_MAX_ROUTES = 1024
VPC_MAX_ACLS = 2048
VPC_MAX_READBACK = 4096
VPC_MAX_READBACK_VALUE = 512
VPC_EVIDENCE_VERSION = 2
# libvirt rejects an incomplete QGA response after 10 MiB, but its API must
# materialize the complete bounded reply before Python can apply the much
# smaller per-command gate below. Keep the rolling pool deliberately narrow
# so several hostile or broken guests cannot multiply that upstream buffer
# into an avoidable host-memory spike.
VPC_GUEST_PROBE_WORKERS = 4
# Per-request rolling pools must also share one process-wide admission gate.
# Without it, concurrent VPC requests each admit four libvirt QGA replies and
# multiply the upstream response-buffer bound. BoundedSemaphore also turns an
# accidental over-release into an immediate programming error.
_VPC_GUEST_PROBE_SLOTS = threading.BoundedSemaphore(VPC_GUEST_PROBE_WORKERS)
VPC_GUEST_EXEC_KIND = "vpc_guest_exec"
VPC_GUEST_EXEC_MAX_BYTES = 1024
# PHP gives this endpoint 60 seconds and holds the operation lease for 120
# seconds. This monotonic budget stops admission of further hosts/QGA calls at
# 40 seconds, leaving 20 seconds for response/transport and another 60 seconds
# before a second worker may reclaim the operation. A libvirt/seed mutation
# already in flight is not safely preemptible; shutdown waits for bounded QGA
# calls instead of leaving background work after the response.
VPC_REQUEST_DEADLINE_SECONDS = 40.0
VPC_STATES = frozenset({"applying", "active", "degraded", "uncertain"})
VPC_DELETE_RECEIPT_KIND = "vpc_deleted"
VPC_DELETE_DETERMINISTIC_HOST_ERRORS = frozenset({
"vpc_private_remove_requires_stopped",
"vpc_private_remove_unsafe",
"vpc_private_interface_conflict",
"vpc_seed_missing",
"vpc_seed_readback_invalid",
"vpc_seed_network_policy_migration_required",
"vpc_private_xml_invalid",
})
VPC_FABRIC_ADMISSION_ERRORS = frozenset({
"vpc_fabric_region_mismatch",
"vpc_fabric_identity_mismatch",
"vpc_fabric_endpoint_overlap",
})
_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z")
_HOST_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
_DIGEST = re.compile(r"[0-9a-f]{64}\Z")
_OPERATION_ID = re.compile(r"[A-Za-z0-9_-]{8,80}\Z")
_PRIVATE_LIFECYCLE_KIND = re.compile(
r"(?:create|reinstall|network_change|instance_settings|rescue|"
r"snapshot_create|snapshot_restore|snapshot_delete|delete|expiry_delete|"
r"action|service_suspend|service_resume|network_suspend|network_resume|"
r"firewall_apply|"
r"action_(?:start|shutdown|reboot|poweroff|reset)_policy_"
r"(?:active|blocked|unready)|"
r"service_suspend_policy_(?:active|blocked|unready)|"
r"service_resume_(?:start|stopped|unbound)_policy_"
r"(?:active|blocked|unready))\Z"
)
# Compatibility markers used by the asynchronous reinstall worker. A same
# operation private reconcile may pass the lifecycle barrier only after the
# replacement domain has crossed its irreversible commit boundary.
_REINSTALL_COMMITTED_PHASES = frozenset({
"cleaning_up", "cleanup_pending", "result_uncertain", "completed",
"重装已完成,但凭据或任务记录保存失败",
"重装完成,但旧系统盘清理待核对", "重装结果待核对",
})
_REINSTALL_PUBLIC_ERROR_CODES = frozenset({
"reinstall_cleanup_pending", "reinstall_result_uncertain",
})
# CloudInstance's existing identity boundary (and libvirt itself) accepts a
# canonical five-group UUID even when the version/variant nibbles are not
# RFC-4122 values. VPC membership must address the already persisted VM
# identity, so imposing an RFC-only subset here would reject valid legacy or
# operator-supplied instances that the PHP DTO intentionally permits.
_UUID = re.compile(
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z"
)
_MAC = re.compile(r"[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\Z")
_PROTOCOLS = frozenset({"any", "tcp", "udp", "icmp"})
_DIRECTIONS = frozenset({"ingress", "egress"})
_ACTIONS = frozenset({"allow", "deny"})
class VpcError(Exception):
"""Safe error returned by the structured VPC boundary."""
def __init__(self, message: str, status: int = 400,
error_code: str = "vpc_invalid") -> None:
super().__init__(message)
self.status = status
self.error_code = error_code
def _reinstall_job_committed(job: Mapping[str, Any]) -> bool:
"""Whether a durable reinstall row proves the replacement boundary."""
if job.get("reinstall_committed") is True:
return True
if job.get("error_code") in _REINSTALL_PUBLIC_ERROR_CODES:
return True
phase = job.get("phase")
if isinstance(phase, str) and phase.strip() in _REINSTALL_COMMITTED_PHASES:
return True
if job.get("cleanup_pending") is True:
return True
warning = job.get("warning")
return isinstance(warning, str) and bool(warning.strip())
def _string(value: Any, label: str, maximum: int = 128) -> str:
if (not isinstance(value, str) or not value or len(value) > maximum
or value != value.strip()
or any(ord(char) < 32 or ord(char) == 127 for char in value)):
raise VpcError(f"{label}无效")
return value
def _identifier(value: Any, label: str = "标识") -> str:
if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None:
raise VpcError(f"{label}无效")
return value
def _host_id(value: Any) -> str:
if not isinstance(value, str) or _HOST_ID.fullmatch(value) is None:
raise VpcError("VPC 宿主机标识无效")
return value
def _revision(value: Any, label: str = "revision") -> int:
if (isinstance(value, bool) or not isinstance(value, int)
or not 1 <= value <= (1 << 63) - 1):
raise VpcError(f"{label}无效")
return value
def _uuid(value: Any) -> str:
if not isinstance(value, str) or _UUID.fullmatch(value) is None:
raise VpcError("VPC 成员实例 UUID 无效")
return value.lower()
def _mac(value: Any) -> str | None:
if value is None:
return None
if not isinstance(value, str) or _MAC.fullmatch(value) is None:
raise VpcError("VPC 成员 MAC 地址无效")
parsed = int(value.replace(":", ""), 16)
# Multicast and all-zero addresses cannot identify a guest interface.
if parsed == 0 or (int(value[:2], 16) & 1):
raise VpcError("VPC 成员 MAC 地址无效")
return value.lower()
def _effective_private_mac(
vpc_id: str, member_id: str, configured: Any = None,
) -> str:
"""Return the canonical MAC used by the host data-plane adapter.
Older durable VPC records may omit ``private_mac``. The dataplane parser
already derives the same stable locally-administered address in that
case; lifecycle evidence must use that value too, otherwise the worker
receives a false mismatch after a successful host readback. Explicit
values remain accepted for backwards compatibility but are still checked
as a valid unicast MAC by :func:`_mac`.
"""
if (_IDENTIFIER.fullmatch(vpc_id) is None
or _IDENTIFIER.fullmatch(member_id) is None):
raise VpcError("VPC 成员标识无效", 422)
value = _mac(configured)
if value is not None:
return value
# Keep one implementation of the derivation in the dataplane module and
# import lazily to avoid its intentional dependency on this control-plane
# module during module initialization.
from .vpc_dataplane import derive_private_mac
return derive_private_mac(vpc_id, member_id)
def _network(value: Any, label: str, *, rfc1918: bool = False,
min_prefix: int = 0, max_prefix: int = 32) -> ipaddress.IPv4Network:
if not isinstance(value, str) or value != value.strip():
raise VpcError(f"{label}无效")
try:
network = ipaddress.ip_network(value, strict=True)
except ValueError as exc:
raise VpcError(f"{label}无效") from exc
if not isinstance(network, ipaddress.IPv4Network):
raise VpcError(f"{label}必须是 IPv4 网段")
if not min_prefix <= network.prefixlen <= max_prefix:
raise VpcError(f"{label}前缀长度无效")
if rfc1918 and not any(
network.subnet_of(ipaddress.IPv4Network(block))
for block in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")):
raise VpcError("VPC CIDR 必须使用 RFC1918 地址")
return network
def _host_address(value: Any) -> str:
if not isinstance(value, str) or value != value.strip():
raise VpcError("VPC 私网 IP 无效")
raw = value
if "/" in raw:
try:
interface = ipaddress.ip_interface(raw)
except ValueError as exc:
raise VpcError("VPC 私网 IP 必须是 /32") from exc
if not isinstance(interface, ipaddress.IPv4Interface) or interface.network.prefixlen != 32:
raise VpcError("VPC 私网 IP 必须是 /32")
return str(interface.ip)
try:
address = ipaddress.IPv4Address(raw)
except ipaddress.AddressValueError as exc:
raise VpcError("VPC 私网 IP 无效") from exc
return str(address)
def _digest(value: Any, label: str = "desired_digest") -> str:
if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
raise VpcError(f"{label}无效")
return value
def _safe_scalar(value: Any) -> bool:
if isinstance(value, bool) or value is None:
return True
if isinstance(value, int):
return -(1 << 63) <= value <= (1 << 63) - 1
return (isinstance(value, str)
and len(value) <= VPC_MAX_READBACK_VALUE
and all(ord(char) >= 32 and ord(char) != 127 for char in value))
def _guest_exec_core(value: Any) -> dict[str, Any]:
"""Validate one bounded internal QGA launch/PID ledger payload."""
if not isinstance(value, Mapping):
raise VpcError("VPC 来宾执行账本损坏", 500,
"vpc_record_corrupt")
phase = value.get("phase")
fields = {
"version", "phase", "domain_uuid", "domain_runtime_id",
"qemu_runtime_token", "guest_network_generation", "probe_kind",
}
if phase == "pending":
fields.add("pid")
generation = value.get("guest_network_generation")
if (set(value) != fields
or value.get("version") != 1
or phase not in {"launch_intent", "pending"}
or not isinstance(value.get("domain_uuid"), str)
or _UUID.fullmatch(value["domain_uuid"]) is None
or isinstance(value.get("domain_runtime_id"), bool)
or not isinstance(value.get("domain_runtime_id"), int)
or not 0 <= value["domain_runtime_id"] <= 2**31 - 1
or not isinstance(value.get("qemu_runtime_token"), str)
or re.fullmatch(
r"[0-9a-f]{32}", value["qemu_runtime_token"]) is None
or not isinstance(generation, str)
or not 1 <= len(generation) <= 128
or any(ord(char) < 0x20 or ord(char) > 0x7e
for char in generation)
or value.get("probe_kind") not in {"link", "routes"}):
raise VpcError("VPC 来宾执行账本损坏", 500,
"vpc_record_corrupt")
if phase == "pending" and (
isinstance(value.get("pid"), bool)
or not isinstance(value.get("pid"), int)
or not 1 <= value["pid"] <= 2**63 - 1):
raise VpcError("VPC 来宾执行账本损坏", 500,
"vpc_record_corrupt")
return dict(value)
def _canonical_cidr(network: ipaddress.IPv4Network) -> str:
return str(network)
@dataclass(frozen=True)
class VpcSubnet:
subnet_id: str
cidr: str
@dataclass(frozen=True)
class VpcMember:
member_id: str
instance_uuid: str
host_id: str
private_ip: str
subnet_id: str
private_mac: str | None = None
@dataclass(frozen=True)
class VpcDeleteMember:
"""Signed VM identity whose persistent private NIC must be absent."""
member_id: str
instance_uuid: str
host_id: str
private_ip: str
private_mac: str
@dataclass(frozen=True)
class VpcRoute:
route_id: str
destination_cidr: str
next_hop_member_id: str | None
next_hop_host_id: str | None
metric: int
@dataclass(frozen=True)
class VpcAcl:
acl_id: str
direction: str
action: str
protocol: str
source_cidr: str
destination_cidr: str
port_from: int | None
port_to: int | None
priority: int
@dataclass(frozen=True)
class VpcHost:
host_id: str
member_ids: tuple[str, ...]
@dataclass(frozen=True)
class VpcPlan:
vpc_id: str
region: str
fabric_id: str
fabric_revision: int
cidr: str
desired_revision: int
subnets: tuple[VpcSubnet, ...]
members: tuple[VpcMember, ...]
hosts: tuple[VpcHost, ...]
routes: tuple[VpcRoute, ...]
acls: tuple[VpcAcl, ...]
desired_digest: str
operation_id: str
@property
def host_ids(self) -> frozenset[str]:
return frozenset(host.host_id for host in self.hosts)
def member(self, member_id: str) -> VpcMember:
for item in self.members:
if item.member_id == member_id:
return item
raise KeyError(member_id)
def canonical(self) -> dict[str, Any]:
# operation_id is an envelope/idempotency token. PHP computes the
# desired digest before attaching it, so it must never change the
# identity of an otherwise identical typed target.
return {
"schema_version": VPC_SCHEMA_VERSION,
"vpc_id": self.vpc_id,
"region": self.region,
"fabric_id": self.fabric_id,
"fabric_revision": self.fabric_revision,
"cidr": self.cidr,
"desired_revision": self.desired_revision,
"hosts": [
{"host_id": host.host_id,
# ``members`` is the wire key used by the PHP desired DTO.
# Keep the canonical form identical across both sides so a
# digest computed before signing is the digest verified by
# the engine parser.
"members": list(host.member_ids)}
for host in self.hosts
],
"subnets": [
{"subnet_id": item.subnet_id, "cidr": item.cidr}
for item in self.subnets
],
"members": [
{"member_id": item.member_id,
"instance_uuid": item.instance_uuid,
"host_id": item.host_id,
"private_ip_cidr": f"{item.private_ip}/32",
"subnet_id": item.subnet_id,
**({"private_mac": item.private_mac}
if item.private_mac is not None else {})}
for item in self.members
],
"routes": [
{"route_id": item.route_id,
"destination_cidr": item.destination_cidr,
**({"next_hop_member_id": item.next_hop_member_id}
if item.next_hop_member_id is not None else {}),
**({"next_hop_host_id": item.next_hop_host_id}
if item.next_hop_host_id is not None else {}),
"metric": item.metric}
for item in self.routes
],
"acls": [
{"acl_id": item.acl_id,
"direction": item.direction,
"action": item.action,
"protocol": item.protocol,
"source_cidr": item.source_cidr,
"destination_cidr": item.destination_cidr,
**({"port_from": item.port_from}
if item.port_from is not None else {}),
**({"port_to": item.port_to}
if item.port_to is not None else {}),
"priority": item.priority}
for item in self.acls
],
}
def _canonical_digest(canonical: Mapping[str, Any]) -> str:
return hashlib.sha256(
json.dumps(canonical, sort_keys=True, separators=(",", ":"),
ensure_ascii=True).encode("utf-8")
).hexdigest()
def _field_set(raw: Mapping[str, Any], allowed: set[str], label: str) -> None:
if set(raw) - allowed:
raise VpcError(f"{label}包含不支持的字段")
def _alias(raw: Mapping[str, Any], primary: str, legacy: str,
label: str) -> Any:
"""Read a compatibility alias without allowing conflicting values."""
first = raw.get(primary)
second = raw.get(legacy)
if first is not None and second is not None and first != second:
raise VpcError(f"{label}字段不一致")
return first if first is not None else second
def _parse_revision(value: Mapping[str, Any]) -> int:
desired = value.get("desired_revision")
legacy = value.get("revision")
if desired is None:
desired = legacy
elif legacy is not None and desired != legacy:
raise VpcError("desired_revision 与 revision 不一致")
return _revision(desired, "desired_revision")
def _parse_members(
value: Mapping[str, Any],
network: ipaddress.IPv4Network,
subnets: tuple[VpcSubnet, ...],
) -> tuple[tuple[VpcMember, ...], tuple[VpcHost, ...]]:
subnet_networks = {
item.subnet_id: _network(item.cidr, "VPC subnet")
for item in subnets
}
raw_members = value.get("members")
raw_hosts = value.get("hosts")
if raw_members is not None and not isinstance(raw_members, list):
raise VpcError("VPC members 必须是数组")
if raw_hosts is not None and (not isinstance(raw_hosts, list)
or len(raw_hosts) > VPC_MAX_HOSTS):
raise VpcError("VPC hosts 必须是数组")
members: list[VpcMember] = []
member_ids: set[str] = set()
instances: set[str] = set()
private_ips: set[str] = set()
host_member_ids: dict[str, list[str]] = {}
def add_member(raw: Mapping[str, Any], forced_host: str | None = None) -> None:
_field_set(raw, {"member_id", "instance_uuid", "host_id",
"private_ip", "private_ip_cidr", "private_mac",
"subnet_id"}, "VPC member")
instance_uuid = _uuid(raw.get("instance_uuid"))
host_id = _host_id(raw.get("host_id", forced_host))
if forced_host is not None and host_id != forced_host:
raise VpcError("VPC 成员宿主机与 hosts 不一致")
member_id = raw.get("member_id", instance_uuid)
member_id = _identifier(member_id, "VPC member")
if member_id in member_ids or instance_uuid in instances:
# It is useful for callers to send both the normalized top-level
# ``members`` list and the per-host grouping. Accept that only
# when the second copy is byte-for-byte equivalent after
# canonicalization; conflicting copies remain a hard error.
existing = next((item for item in members
if item.member_id == member_id), None)
if existing is None:
raise VpcError("VPC 成员不能重复")
candidate_ip = _host_address(
_alias(raw, "private_ip_cidr", "private_ip", "VPC 私网 IP"))
candidate_subnet = raw.get("subnet_id", existing.subnet_id)
candidate_mac = _mac(raw.get("private_mac"))
if (existing.instance_uuid != instance_uuid
or existing.host_id != host_id
or existing.private_ip != candidate_ip
or existing.subnet_id != candidate_subnet
or existing.private_mac != candidate_mac):
raise VpcError("VPC 成员重复定义不一致")
host_member_ids.setdefault(host_id, []).append(member_id)
return
raw_ip = _alias(raw, "private_ip_cidr", "private_ip", "VPC 私网 IP")
private_ip = _host_address(raw_ip)
address = ipaddress.IPv4Address(private_ip)
if address not in network or address in {
network.network_address, network.broadcast_address}:
raise VpcError("VPC 私网 IP 不在 VPC 可用地址范围内")
if private_ip in private_ips:
raise VpcError("VPC 私网 IP 不能重复")
raw_subnet = raw.get("subnet_id")
if raw_subnet is None:
matching = [sid for sid, subnet in subnet_networks.items()
if address in subnet
and address not in {subnet.network_address,
subnet.broadcast_address}]
if len(matching) != 1:
raise VpcError("VPC 成员必须明确归属一个 subnet")
subnet_id = matching[0]
else:
subnet_id = _identifier(raw_subnet, "VPC subnet")
subnet = subnet_networks.get(subnet_id)
if subnet is None or address not in subnet or address in {
subnet.network_address, subnet.broadcast_address}:
raise VpcError("VPC 成员私网 IP 不属于指定 subnet")
private_mac = _mac(raw.get("private_mac"))
members.append(VpcMember(member_id, instance_uuid, host_id,
private_ip, subnet_id, private_mac))
member_ids.add(member_id)
instances.add(instance_uuid)
private_ips.add(private_ip)
host_member_ids.setdefault(host_id, []).append(member_id)
if raw_members is not None:
for raw in raw_members:
if not isinstance(raw, Mapping):
raise VpcError("VPC member 必须是对象")
add_member(raw)
if raw_hosts is not None:
explicit_hosts: dict[str, list[str]] = {}
for raw in raw_hosts:
if not isinstance(raw, Mapping):
raise VpcError("VPC host 必须是对象")
_field_set(raw, {"host_id", "members"}, "VPC host")
host_id = _host_id(raw.get("host_id"))
if host_id in explicit_hosts:
raise VpcError("VPC 宿主机不能重复")
raw_host_members = raw.get("members", [])
if not isinstance(raw_host_members, list):
raise VpcError("VPC host members 必须是数组")
explicit_hosts[host_id] = []
for item in raw_host_members:
if isinstance(item, str):
member_id = _identifier(item, "VPC member")
existing = next((member for member in members
if member.member_id == member_id), None)
if existing is None or existing.host_id != host_id:
raise VpcError("VPC host member 不存在或宿主机不一致")
explicit_hosts[host_id].append(member_id)
continue
if not isinstance(item, Mapping):
raise VpcError("VPC host member 必须是对象")
add_member(item, host_id)
raw_member_id = item.get("member_id")
explicit_hosts[host_id].append(
_identifier(raw_member_id, "VPC member")
if raw_member_id is not None else _uuid(item.get("instance_uuid")))
# A member may be represented either at top level or under hosts, but
# never twice. Explicit host membership must exactly describe all
# top-level members for that host.
if raw_hosts is not None:
for host_id, member_ids_for_host in host_member_ids.items():
if host_id not in explicit_hosts:
raise VpcError("VPC member 缺少 hosts 归属")
if set(member_ids_for_host) != set(explicit_hosts[host_id]):
raise VpcError("VPC hosts 与 members 不一致")
host_member_ids = explicit_hosts
for host_id in explicit_hosts:
host_member_ids.setdefault(host_id, [])
if any(len(ids) != len(set(ids)) for ids in host_member_ids.values()):
raise VpcError("VPC 宿主机成员不能重复")
if len(members) > VPC_MAX_MEMBERS:
raise VpcError("VPC 成员数量超过限制")
if raw_hosts is None:
# Empty VPCs are valid desired state, but they cannot pass the next
# stage gate until a participating host/member is declared.
hosts = [VpcHost(host_id, tuple(sorted(set(ids))))
for host_id, ids in host_member_ids.items()]
else:
hosts = [VpcHost(host_id, tuple(sorted(set(ids))))
for host_id, ids in host_member_ids.items()]
return tuple(sorted(members, key=lambda item: item.member_id)), tuple(
sorted(hosts, key=lambda item: item.host_id))
def parse_vpc_plan(value: Mapping[str, Any]) -> VpcPlan:
"""Validate and canonicalize a tenant VPC desired state.
The parser intentionally rejects shell-like, host-specific or public
network fields. It accepts the engine contract's ``desired_revision``
spelling and the old ``revision`` spelling only as an equal alias.
"""
if not isinstance(value, Mapping):
raise VpcError("VPC 计划必须是对象")
allowed = {
"schema_version", "vpc_id", "region", "fabric_id", "fabric_revision",
"cidr", "network_cidr", "desired_revision", "revision",
"desired_digest", "operation_id", "hosts", "members", "subnets",
"routes", "acls",
}
_field_set(value, allowed, "VPC 计划")
if value.get("schema_version", VPC_SCHEMA_VERSION) != VPC_SCHEMA_VERSION:
raise VpcError("VPC 计划版本不支持")
vpc_id = _identifier(value.get("vpc_id"), "VPC")
region = _string(value.get("region"), "VPC 地区", 64)
fabric_id = _identifier(value.get("fabric_id"), "Fabric")
fabric_revision = _revision(value.get("fabric_revision"), "Fabric revision")
cidr_value = _alias(value, "cidr", "network_cidr", "VPC CIDR")
network = _network(cidr_value, "VPC CIDR", rfc1918=True,
min_prefix=16, max_prefix=24)
desired_revision = _parse_revision(value)
raw_subnets = value.get("subnets", [])
if not isinstance(raw_subnets, list) or len(raw_subnets) > VPC_MAX_SUBNETS:
raise VpcError("VPC subnets 必须是数组且不超过限制")
subnets: list[VpcSubnet] = []
subnet_ids: set[str] = set()
subnet_networks: list[ipaddress.IPv4Network] = []
for raw in raw_subnets:
if not isinstance(raw, Mapping):
raise VpcError("VPC subnet 必须是对象")
_field_set(raw, {"subnet_id", "id", "cidr", "name"}, "VPC subnet")
subnet_id = _identifier(_alias(raw, "subnet_id", "id", "VPC subnet"),
"VPC subnet")
if subnet_id in subnet_ids:
raise VpcError("VPC subnet 不能重复")
subnet = _network(raw.get("cidr"), "VPC subnet", min_prefix=network.prefixlen,
max_prefix=30)
if not subnet.subnet_of(network):
raise VpcError("VPC subnet 必须位于 VPC CIDR 内")
if any(subnet.overlaps(other) for other in subnet_networks):
raise VpcError("VPC subnet 不能重叠")
subnet_ids.add(subnet_id)
subnet_networks.append(subnet)
subnets.append(VpcSubnet(subnet_id, _canonical_cidr(subnet)))
subnets.sort(key=lambda item: item.subnet_id)
members, hosts = _parse_members(
{**value, "subnets": [item.__dict__ for item in subnets]},
network, tuple(subnets))
host_ids = {host.host_id for host in hosts}
member_by_id = {item.member_id: item for item in members}
raw_routes = value.get("routes", [])
if not isinstance(raw_routes, list) or len(raw_routes) > VPC_MAX_ROUTES:
raise VpcError("VPC routes 必须是数组且不超过限制")
routes: list[VpcRoute] = []
route_ids: set[str] = set()
route_keys: set[tuple[str, str | None, str | None]] = set()
for raw in raw_routes:
if not isinstance(raw, Mapping):
raise VpcError("VPC route 必须是对象")
_field_set(raw, {"route_id", "id", "destination_cidr", "destination",
"next_hop_member_id", "next_hop_host_id", "metric"},
"VPC route")
route_id = _identifier(_alias(raw, "route_id", "id", "VPC route"),
"VPC route")
if route_id in route_ids:
raise VpcError("VPC route 不能重复")
destination = _network(
_alias(raw, "destination_cidr", "destination",
"VPC route destination"),
"VPC route destination", min_prefix=network.prefixlen, max_prefix=32)
# The private data plane is an L3 /32 fabric. A wider route would
# become a cryptokey-routing and nftables bypass (and could overlap a
# different tenant), so reject it at the control-plane boundary rather
# than relying on the host adapter to catch it later.
if destination.prefixlen != 32:
raise VpcError("VPC route destination 必须是精确 /32", 409,
"vpc_route_not_exact")
if not destination.subnet_of(network):
raise VpcError("VPC route destination 必须是 VPC 内的网段")
if destination.network_address in {
network.network_address, network.broadcast_address}:
raise VpcError("VPC route destination 不能使用 VPC 保留地址", 409,
"vpc_route_reserved_address")
member_target = raw.get("next_hop_member_id")
host_target = raw.get("next_hop_host_id")
if (member_target is None) == (host_target is None):
raise VpcError("VPC route 必须指定一个 next hop")
if member_target is not None:
member_target = _identifier(member_target, "VPC route member")
if member_target not in member_by_id:
raise VpcError("VPC route next hop member 不存在")
host_target = None
else:
host_target = _host_id(host_target)
if host_target not in host_ids:
raise VpcError("VPC route next hop host 不存在")
metric = raw.get("metric", 100)
if (isinstance(metric, bool) or not isinstance(metric, int)
or not 1 <= metric <= 65535):
raise VpcError("VPC route metric 无效")
route_key = (_canonical_cidr(destination), member_target, host_target)
if route_key in route_keys:
raise VpcError("VPC route 不能重复")
route_ids.add(route_id)
route_keys.add(route_key)
routes.append(VpcRoute(route_id, _canonical_cidr(destination),
member_target, host_target, metric))
routes.sort(key=lambda item: item.route_id)
raw_acls = value.get("acls", [])
if not isinstance(raw_acls, list) or len(raw_acls) > VPC_MAX_ACLS:
raise VpcError("VPC ACL 必须是数组且不超过限制")
acls: list[VpcAcl] = []
acl_ids: set[str] = set()
acl_priorities: set[int] = set()
for raw in raw_acls:
if not isinstance(raw, Mapping):
raise VpcError("VPC ACL 必须是对象")
_field_set(raw, {"acl_id", "id", "direction", "action", "protocol",
"source_cidr", "source", "destination_cidr",
"destination", "port_from", "port_to", "priority"},
"VPC ACL")
acl_id = _identifier(_alias(raw, "acl_id", "id", "VPC ACL"),
"VPC ACL")
if acl_id in acl_ids:
raise VpcError("VPC ACL 不能重复")
direction = raw.get("direction")
action = raw.get("action")
protocol = raw.get("protocol")
if direction not in _DIRECTIONS or action not in _ACTIONS or protocol not in _PROTOCOLS:
raise VpcError("VPC ACL direction/action/protocol 无效")
source = _network(_alias(raw, "source_cidr", "source", "VPC ACL source"),
"VPC ACL source", min_prefix=network.prefixlen,
max_prefix=32)
destination = _network(
_alias(raw, "destination_cidr", "destination",
"VPC ACL destination"),
"VPC ACL destination", min_prefix=network.prefixlen, max_prefix=32)
if not source.subnet_of(network) or not destination.subnet_of(network):
raise VpcError("VPC ACL 地址必须位于 VPC CIDR 内")
port_from = raw.get("port_from")
port_to = raw.get("port_to")
if protocol in {"tcp", "udp"}:
if port_from is None and port_to is None:
pass
elif (isinstance(port_from, bool) or not isinstance(port_from, int)
or isinstance(port_to, bool) or not isinstance(port_to, int)
or not 1 <= port_from <= port_to <= 65535):
raise VpcError("VPC ACL 端口范围无效")
elif port_from is not None or port_to is not None:
raise VpcError("非 TCP/UDP ACL 不能指定端口")
priority = raw.get("priority", len(acls) + 1)
if (isinstance(priority, bool) or not isinstance(priority, int)
or not 1 <= priority <= 1_000_000 or priority in acl_priorities):
raise VpcError("VPC ACL priority 无效或重复")
acl_ids.add(acl_id)
acl_priorities.add(priority)
acls.append(VpcAcl(
acl_id, direction, action, protocol,
_canonical_cidr(source), _canonical_cidr(destination),
port_from, port_to, priority))
acls.sort(key=lambda item: (item.priority, item.acl_id))
operation_id = value.get("operation_id")
if operation_id is None:
seed = f"{vpc_id}:{desired_revision}:{fabric_id}"
operation_id = "vpc-" + hashlib.sha256(seed.encode()).hexdigest()[:32]
elif not isinstance(operation_id, str) or _OPERATION_ID.fullmatch(operation_id) is None:
raise VpcError("VPC operation_id 无效")
canonical_hosts = tuple(sorted(hosts, key=lambda item: item.host_id))
provisional = VpcPlan(
vpc_id, region, fabric_id, fabric_revision, _canonical_cidr(network),
desired_revision, tuple(subnets), members, canonical_hosts,
tuple(routes), tuple(acls), "", operation_id)
computed_digest = _canonical_digest(provisional.canonical())
supplied_digest = value.get("desired_digest")
if supplied_digest is not None:
supplied_digest = _digest(supplied_digest)
if supplied_digest != computed_digest:
raise VpcError("desired_digest 与 VPC 计划不一致", 409,
"vpc_digest_conflict")
return VpcPlan(
provisional.vpc_id, provisional.region, provisional.fabric_id,
provisional.fabric_revision, provisional.cidr,
provisional.desired_revision, provisional.subnets,
provisional.members, provisional.hosts, provisional.routes,
provisional.acls, computed_digest, provisional.operation_id)
def _canonical_delete_members(
members: Sequence[VpcDeleteMember],
) -> list[dict[str, str]]:
return [{
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"host_id": member.host_id,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": member.private_mac,
} for member in members]
def _parse_delete_members(
value: Any, plan: VpcPlan,
) -> tuple[VpcDeleteMember, ...]:
if not isinstance(value, list) or len(value) > VPC_MAX_MEMBERS:
raise VpcError("VPC 删除成员清单必须是数组且不超过限制", 422,
"vpc_delete_manifest_invalid")
network = ipaddress.ip_network(plan.cidr)
members: list[VpcDeleteMember] = []
member_ids: set[str] = set()
instance_uuids: set[str] = set()
private_ips: set[str] = set()
private_macs: set[str] = set()
try:
for raw in value:
if not isinstance(raw, Mapping):
raise VpcError("VPC 删除成员必须是对象")
_field_set(raw, {
"member_id", "instance_uuid", "host_id",
"private_ip_cidr", "private_mac",
}, "VPC 删除成员")
member_id = _identifier(raw.get("member_id"), "VPC member")
instance_uuid = _uuid(raw.get("instance_uuid"))
host_id = _host_id(raw.get("host_id"))
if host_id not in plan.host_ids:
raise VpcError("VPC 删除成员宿主机不在删除目标中")
private_ip = _host_address(raw.get("private_ip_cidr"))
address = ipaddress.IPv4Address(private_ip)
if address not in network or address in {
network.network_address, network.broadcast_address}:
raise VpcError("VPC 删除成员私网 IP 不在 VPC 可用范围")
private_mac = _mac(raw.get("private_mac"))
if private_mac is None:
raise VpcError("VPC 删除成员 private_mac 缺失")
if (member_id in member_ids or instance_uuid in instance_uuids
or private_ip in private_ips
or private_mac in private_macs):
raise VpcError("VPC 删除成员清单包含重复身份")
member_ids.add(member_id)
instance_uuids.add(instance_uuid)
private_ips.add(private_ip)
private_macs.add(private_mac)
members.append(VpcDeleteMember(
member_id, instance_uuid, host_id, private_ip, private_mac))
except VpcError as exc:
if exc.error_code == "vpc_delete_manifest_invalid":
raise
raise VpcError(str(exc), 422,
"vpc_delete_manifest_invalid") from exc
return tuple(sorted(members, key=lambda item: item.member_id))
def parse_vpc_delete_plan(
value: Mapping[str, Any],
) -> tuple[VpcPlan, tuple[VpcDeleteMember, ...]]:
"""Parse the empty target plus its digest-covered VM cleanup manifest."""
if not isinstance(value, Mapping):
raise VpcError("VPC 删除计划必须是对象", 422,
"vpc_delete_manifest_invalid")
if "retiring_members" not in value:
raise VpcError("VPC 删除计划缺少 retiring_members", 422,
"vpc_delete_manifest_invalid")
base = dict(value)
raw_members = base.pop("retiring_members")
supplied_digest = base.pop("desired_digest", None)
plan = parse_vpc_plan(base)
members = _parse_delete_members(raw_members, plan)
canonical = plan.canonical()
canonical["retiring_members"] = _canonical_delete_members(members)
computed_digest = _canonical_digest(canonical)
if supplied_digest is not None:
supplied_digest = _digest(supplied_digest)
if supplied_digest != computed_digest:
raise VpcError("desired_digest 与 VPC 删除计划不一致", 409,
"vpc_digest_conflict")
return replace(plan, desired_digest=computed_digest), members
def _plan_from_record(value: Mapping[str, Any]) -> VpcPlan:
if not isinstance(value, Mapping):
raise VpcError("VPC 持久化记录损坏", 500, "vpc_record_corrupt")
try:
raw_members = value.get("members", [])
# Durable host rows carry only member IDs plus observed fields. The
# parser's public contract accepts full typed member objects, so
# reconstruct the grouping from the canonical member rows instead of
# feeding internal status fields back into validation.
members_by_host: dict[str, list[dict[str, Any]]] = {}
if isinstance(raw_members, list):
for member in raw_members:
if isinstance(member, Mapping) and isinstance(member.get("host_id"), str):
members_by_host.setdefault(member["host_id"], []).append(dict(member))
raw_host_rows = value.get("hosts", [])
host_ids: list[str] = []
if isinstance(raw_host_rows, list):
for host in raw_host_rows:
if (isinstance(host, Mapping)
and host.get("retiring") is not True
and isinstance(host.get("host_id"), str)):
host_ids.append(host["host_id"])
for host_id in members_by_host:
if host_id not in host_ids:
host_ids.append(host_id)
reconstructed_hosts = [
{"host_id": host_id, "members": members_by_host.get(host_id, [])}
for host_id in sorted(set(host_ids))
]
payload = {
"schema_version": value.get("schema_version", VPC_SCHEMA_VERSION),
"vpc_id": value["vpc_id"], "region": value["region"],
"fabric_id": value["fabric_id"],
"fabric_revision": value["fabric_revision"],
"cidr": value["cidr"],
"desired_revision": value["desired_revision"],
"operation_id": value.get("operation_id"),
"hosts": reconstructed_hosts, "subnets": value.get("subnets", []),
"members": None, "routes": value.get("routes", []),
"acls": value.get("acls", []),
}
except KeyError as exc:
raise VpcError("VPC 持久化记录缺少字段", 500, "vpc_record_corrupt") from exc
try:
plan = parse_vpc_plan(payload)
except VpcError as exc:
raise VpcError("VPC 持久化记录无效", 500, "vpc_record_corrupt") from exc
stored_digest = value.get("desired_digest")
delete_target = value.get("delete_target")
if delete_target is not None:
if (not isinstance(delete_target, Mapping)
or "retiring_members" not in delete_target):
raise VpcError("VPC 删除目标记录无效", 500,
"vpc_record_corrupt")
try:
retiring_members = _parse_delete_members(
delete_target["retiring_members"], plan)
except VpcError as exc:
raise VpcError("VPC 删除成员清单记录无效", 500,
"vpc_record_corrupt") from exc
canonical = plan.canonical()
canonical["retiring_members"] = _canonical_delete_members(
retiring_members)
delete_digest = _canonical_digest(canonical)
if (stored_digest != delete_digest
or delete_target.get("desired_digest") != delete_digest):
raise VpcError("VPC 删除持久化摘要不一致", 500,
"vpc_record_corrupt")
return replace(plan, desired_digest=delete_digest)
if stored_digest not in (None, plan.desired_digest):
raise VpcError("VPC 持久化摘要不一致", 500, "vpc_record_corrupt")
return plan
class VpcCoordinator:
"""Durable desired/observed VPC state with per-VPC and catalog locks."""
def __init__(self, store: Any, fabric: Any, registry: Any | None = None,
vm_lock: Any | None = None,
catalog_lock: Any | None = None):
self.store = store
self.fabric = fabric
self._catalog_lock_provider = catalog_lock
# The registry is optional for pure control-plane/unit callers. The
# production EngineApplication supplies it so a VPC revision can only
# become active after host bridge/WireGuard/nft and libvirt readback.
self.registry = registry
# EngineApplication supplies the same per-VM lock used by ordinary
# lifecycle mutations. Keeping the callback optional preserves the
# pure control-plane test harness and older embedding API.
self.vm_lock = vm_lock
# Per-VPC serialization remains independent from FabricCoordinator's
# per-Fabric locks and from VM/public network mutation locks. A VPC
# catalog mutations (apply and root delete) additionally borrow
# FabricCoordinator's process-wide admission lock so a tenant CIDR
# cannot race an endpoint migration through the two coordinators'
# opposite validation scans.
self._locks: dict[str, Any] = {}
self._fallback_catalog_lock: Any | None = None
@staticmethod
def _key(vpc_id: str, child_id: str) -> str:
return f"{vpc_id}.{child_id}"
def _lock(self, vpc_id: str):
import asyncio
return self._locks.setdefault(vpc_id, asyncio.Lock())
def _catalog_mutation_lock(self):
import asyncio
if callable(self._catalog_lock_provider):
return self._catalog_lock_provider()
# Pure control-plane embeddings and small unit fakes may not provide
# a real FabricCoordinator. They still receive local serialization;
# EngineApplication always supplies the shared production lock.
if self._fallback_catalog_lock is None:
self._fallback_catalog_lock = asyncio.Lock()
return self._fallback_catalog_lock
def _load(self, kind: str, key: str) -> dict[str, Any] | None:
raw = self.store.get(kind, key)
if raw is None:
return None
try:
value = json.loads(raw)
except (TypeError, ValueError) as exc:
raise VpcError("VPC 持久化记录损坏", 500, "vpc_record_corrupt") from exc
if not isinstance(value, dict):
raise VpcError("VPC 持久化记录损坏", 500, "vpc_record_corrupt")
if kind == "vpc_desired" and value.get("vpc_id") != key:
raise VpcError("VPC 持久化记录标识不一致", 500,
"vpc_record_corrupt")
return value
def _save(self, kind: str, key: str, value: Mapping[str, Any]) -> None:
self.store.set(kind, key, json.dumps(value, ensure_ascii=True,
separators=(",", ":")))
def _guest_exec_context(
self, plan: VpcPlan, member: VpcMember,
) -> tuple[
dict[str, Any],
Any,
Any,
]:
"""Load one private durable QGA ledger and build exact callbacks.
The record is deliberately separate from ``vpc_desired``: a PID must
be committed immediately after ``guest-exec`` returns, while rewriting
a multi-thousand-member desired document for every probe would create
quadratic database traffic. Existing VPC/instance locks serialize
these callbacks for the same member.
"""
key = self._key(plan.vpc_id, member.instance_uuid)
raw = self.store.get(VPC_GUEST_EXEC_KIND, key)
state: dict[str, Any] = {}
base = {
"vpc_id": plan.vpc_id,
"host_id": member.host_id,
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
}
if raw is not None:
if (not isinstance(raw, str)
or len(raw.encode("utf-8")) > VPC_GUEST_EXEC_MAX_BYTES):
raise VpcError("VPC 来宾执行账本损坏", 500,
"vpc_record_corrupt")
try:
value = json.loads(raw)
except (TypeError, ValueError) as exc:
raise VpcError("VPC 来宾执行账本损坏", 500,
"vpc_record_corrupt") from exc
if (not isinstance(value, Mapping)
or set(value) != set(base) | {
"version", "phase", "domain_uuid",
"domain_runtime_id", "qemu_runtime_token",
"guest_network_generation",
"probe_kind",
} | ({"pid"} if value.get("phase") == "pending" else set())
or any(value.get(field) != expected
for field, expected in base.items())):
raise VpcError("VPC 来宾执行账本损坏", 500,
"vpc_record_corrupt")
state = _guest_exec_core({
field: value[field] for field in value if field not in base
})
if state.get("domain_uuid") != member.instance_uuid:
raise VpcError("VPC 来宾执行账本损坏", 500,
"vpc_record_corrupt")
def persist(core: Mapping[str, Any]) -> None:
normalized = _guest_exec_core(core)
if normalized.get("domain_uuid") != member.instance_uuid:
raise VpcError("VPC 来宾执行账本损坏", 500,
"vpc_record_corrupt")
record = dict(base)
record.update(normalized)
encoded = json.dumps(
record, ensure_ascii=True, separators=(",", ":"))
if len(encoded.encode("utf-8")) > VPC_GUEST_EXEC_MAX_BYTES:
raise VpcError("VPC 来宾执行账本过大", 500,
"vpc_record_corrupt")
self.store.set(VPC_GUEST_EXEC_KIND, key, encoded)
def clear() -> None:
self.store.delete(VPC_GUEST_EXEC_KIND, key)
return state, persist, clear
@staticmethod
def _confirmed_removal(
host: Mapping[str, Any], revision: int, digest: str,
) -> bool:
"""Whether a durable host row is an exact absence proof.
A boolean ``removed`` marker alone is not enough to retire a host from
a later root-delete target. The proof is useful only when it belongs
to the exact durable revision/digest and every bounded tenant resource
is absent. This also accepts the historical non-``retiring`` shape
produced when PHP kept a deleting deployment in the desired host list.
"""
readback = host.get("readback")
if isinstance(readback, Mapping):
readback_state = str(
readback.get("state", readback.get("status", "")))
else:
readback_state = str(readback or "")
return bool(
host.get("removed") is True
and host.get("state") == "active"
and host.get("transport_verified") is True
and host.get("private_dataplane_verified") is True
and host.get("desired_revision") == revision
and host.get("observed_revision") == revision
and host.get("desired_digest") == digest
and host.get("observed_digest") == digest
and host.get("member_ids", []) == []
and host.get("retiring_members", []) == []
and host.get("member_nics") == []
and host.get("routes") == []
and host.get("wg_allowed_ips") == []
and host.get("proxy_ips") == []
and host.get("bridge") is None
and host.get("bridge_owner") is None
and host.get("nft_table") is None
and host.get("anti_spoof") is False
and host.get("rp_filter") is None
and host.get("proxy_arp") is None
and host.get("proxy_arp_pvlan") is None
and readback_state.lower() == "confirmed"
and host.get("rollback") in {"none", "ok"}
)
@staticmethod
def _confirmed_active_host(host: Mapping[str, Any], plan: VpcPlan) -> bool:
"""Whether this exact host revision already has complete live proof.
A request-wide probe budget is shared by every host. Retrying an
already exact host on every reconcile would let an early slow host
starve later hosts forever. Preserve only the strong same-revision
proof; any tombstone, member-set change, or incomplete guest evidence
puts the host back in the execution closure.
"""
host_id = host.get("host_id")
if not isinstance(host_id, str):
return False
expected_members = tuple(
member for member in plan.members if member.host_id == host_id)
expected_ids = {member.member_id for member in expected_members}
raw_ids = host.get("member_ids")
member_nics = host.get("member_nics")
if (not isinstance(raw_ids, list)
or any(not isinstance(value, str) for value in raw_ids)
or set(raw_ids) != expected_ids
or len(raw_ids) != len(expected_ids)
or not isinstance(member_nics, list)
or len(member_nics) != len(expected_members)):
return False
expected_by_id = {
member.member_id: member for member in expected_members
}
proven_ids: set[str] = set()
for item in member_nics:
if not isinstance(item, Mapping):
continue
member_id = item.get("member_id")
member = expected_by_id.get(member_id)
if (member is None
or item.get("instance_uuid") != member.instance_uuid
or item.get("private_ip_cidr")
!= f"{member.private_ip}/32"
or item.get("private_mac")
!= _effective_private_mac(
plan.vpc_id, member.member_id,
member.private_mac)
or item.get("config_verified") is not True
or item.get("xml_verified") is not True
or item.get("domain_running") is not True
or item.get("guest_network_verified") is not True
or item.get("restart_required") is not False):
continue
proven_ids.add(member_id)
return bool(
host.get("retiring") is not True
and host.get("retiring_members", []) == []
and host.get("state") == "active"
and host.get("transport_verified") is True
and host.get("private_dataplane_verified") is True
and host.get("config_verified") is True
and host.get("desired_revision") == plan.desired_revision
and host.get("observed_revision") == plan.desired_revision
and host.get("config_observed_revision") == plan.desired_revision
and host.get("desired_digest") == plan.desired_digest
and host.get("observed_digest") == plan.desired_digest
and host.get("config_observed_digest") == plan.desired_digest
and proven_ids == expected_ids
)
@classmethod
def _terminal_removal(cls, host: Mapping[str, Any]) -> bool:
"""Validate a self-contained historical host absence proof."""
revision = host.get("desired_revision")
digest = host.get("desired_digest")
return bool(
isinstance(revision, int)
and not isinstance(revision, bool)
and 1 <= revision <= (1 << 63) - 1
and isinstance(digest, str)
and _DIGEST.fullmatch(digest) is not None
and cls._confirmed_removal(host, revision, digest)
)
@staticmethod
def _durable_retiring_members(
host: Mapping[str, Any], *, vpc_id: str, host_id: str,
) -> list[dict[str, Any]]:
"""Validate an unconfirmed normal-apply host tombstone fail-closed."""
raw_members = host.get("retiring_members")
raw_member_ids = host.get("member_ids")
if (not isinstance(raw_members, list)
or len(raw_members) > VPC_MAX_MEMBERS
or not isinstance(raw_member_ids, list)
or len(raw_member_ids) > VPC_MAX_MEMBERS):
raise VpcError("VPC 宿主机清理成员证据损坏", 500,
"vpc_record_corrupt")
try:
member_ids = [_identifier(item, "VPC member")
for item in raw_member_ids]
if len(member_ids) != len(set(member_ids)):
raise VpcError("VPC 宿主机清理成员标识重复")
normalized: list[dict[str, Any]] = []
seen_member_ids: set[str] = set()
seen_instances: set[str] = set()
seen_private_ips: set[str] = set()
seen_private_macs: set[str] = set()
for raw in raw_members:
if not isinstance(raw, Mapping):
raise VpcError("VPC 宿主机清理成员必须是对象")
_field_set(raw, {
"member_id", "instance_uuid", "host_id",
"private_ip_cidr", "subnet_id", "private_mac",
}, "VPC 宿主机清理成员")
if not all(field in raw for field in (
"member_id", "instance_uuid", "host_id",
"private_ip_cidr", "subnet_id")):
raise VpcError("VPC 宿主机清理成员字段不完整")
member_id = _identifier(raw["member_id"], "VPC member")
instance_uuid = _uuid(raw["instance_uuid"])
member_host_id = _host_id(raw["host_id"])
if member_host_id != host_id:
raise VpcError("VPC 宿主机清理成员归属不一致")
private_ip = _host_address(raw["private_ip_cidr"])
subnet_id = _identifier(raw["subnet_id"], "VPC subnet")
private_mac = _effective_private_mac(
vpc_id, member_id, raw.get("private_mac"))
if (member_id in seen_member_ids
or instance_uuid in seen_instances
or private_ip in seen_private_ips
or private_mac in seen_private_macs):
raise VpcError("VPC 宿主机清理成员身份重复")
seen_member_ids.add(member_id)
seen_instances.add(instance_uuid)
seen_private_ips.add(private_ip)
seen_private_macs.add(private_mac)
normalized.append({
"member_id": member_id,
"instance_uuid": instance_uuid,
"host_id": member_host_id,
"private_ip_cidr": f"{private_ip}/32",
"subnet_id": subnet_id,
"private_mac": private_mac,
})
if set(member_ids) != {
item["member_id"] for item in normalized}:
raise VpcError("VPC 宿主机清理成员闭集不一致")
except VpcError as exc:
if exc.error_code == "vpc_record_corrupt":
raise
raise VpcError("VPC 宿主机清理成员证据损坏", 500,
"vpc_record_corrupt") from exc
return normalized
@staticmethod
def _delete_plan(plan: VpcPlan) -> None:
"""Require the signed root-delete target to contain no tenant intent."""
if (plan.subnets or plan.members or plan.routes or plan.acls
or any(host.member_ids for host in plan.hosts)):
raise VpcError(
"VPC 删除目标必须清空全部子资源和宿主机成员",
409, "vpc_delete_target_not_empty")
@staticmethod
def _host_closure(record: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]:
hosts = record.get("hosts", [])
if not isinstance(hosts, list) or len(hosts) > VPC_MAX_HOSTS:
raise VpcError("VPC 删除宿主机闭集无效", 500,
"vpc_record_corrupt")
result: dict[str, Mapping[str, Any]] = {}
for host in hosts:
if not isinstance(host, Mapping):
raise VpcError("VPC 删除宿主机闭集无效", 500,
"vpc_record_corrupt")
host_id = _host_id(host.get("host_id"))
if host_id in result:
raise VpcError("VPC 删除宿主机闭集重复", 500,
"vpc_record_corrupt")
result[host_id] = host
return result
@staticmethod
def _delete_marker(record: Mapping[str, Any]) -> dict[str, Any] | None:
marker = record.get("delete_target")
if marker is None:
return None
if not isinstance(marker, dict):
raise VpcError("VPC 删除目标记录损坏", 500,
"vpc_record_corrupt")
for field in ("operation_id", "desired_revision", "desired_digest",
"all_host_ids", "required_host_ids",
"preverified_host_ids", "retiring_members"):
if field not in marker:
raise VpcError("VPC 删除目标记录不完整", 500,
"vpc_record_corrupt")
if (not isinstance(marker["operation_id"], str)
or _OPERATION_ID.fullmatch(marker["operation_id"]) is None
or isinstance(marker["desired_revision"], bool)
or not isinstance(marker["desired_revision"], int)
or marker["desired_revision"] < 1
or not isinstance(marker["desired_digest"], str)
or _DIGEST.fullmatch(marker["desired_digest"]) is None):
raise VpcError("VPC 删除目标记录无效", 500,
"vpc_record_corrupt")
parsed: dict[str, list[str]] = {}
for field in ("all_host_ids", "required_host_ids",
"preverified_host_ids"):
values = marker[field]
if (not isinstance(values, list) or len(values) > VPC_MAX_HOSTS
or values != sorted(set(values))):
raise VpcError("VPC 删除宿主机闭集无效", 500,
"vpc_record_corrupt")
try:
parsed[field] = [_host_id(value) for value in values]
except VpcError as exc:
raise VpcError("VPC 删除宿主机闭集无效", 500,
"vpc_record_corrupt") from exc
if ((set(parsed["required_host_ids"])
| set(parsed["preverified_host_ids"]))
!= set(parsed["all_host_ids"])
or set(parsed["required_host_ids"])
& set(parsed["preverified_host_ids"])):
raise VpcError("VPC 删除宿主机闭集不一致", 500,
"vpc_record_corrupt")
if (not isinstance(marker["retiring_members"], list)
or len(marker["retiring_members"]) > VPC_MAX_MEMBERS):
raise VpcError("VPC 删除成员清单记录无效", 500,
"vpc_record_corrupt")
return marker
def _load_delete_receipt(self, vpc_id: str) -> dict[str, Any] | None:
receipt = self._load(VPC_DELETE_RECEIPT_KIND, vpc_id)
if receipt is None:
return None
required = {
"vpc_id", "operation_id", "desired_revision", "desired_digest",
"state", "deleted", "readback", "removed_host_ids",
"completed_at",
}
if (not required.issubset(receipt)
or receipt.get("schema_version") != VPC_SCHEMA_VERSION
or receipt.get("vpc_id") != vpc_id
or not isinstance(receipt.get("operation_id"), str)
or _OPERATION_ID.fullmatch(receipt["operation_id"]) is None
or isinstance(receipt.get("desired_revision"), bool)
or not isinstance(receipt.get("desired_revision"), int)
or receipt["desired_revision"] < 1
or not isinstance(receipt.get("desired_digest"), str)
or _DIGEST.fullmatch(receipt["desired_digest"]) is None
or receipt.get("state") != "deleted"
or receipt.get("deleted") is not True
or receipt.get("readback") != "confirmed"
or isinstance(receipt.get("completed_at"), bool)
or not isinstance(receipt.get("completed_at"), int)
or receipt["completed_at"] < 1):
raise VpcError("VPC 删除回执损坏", 500,
"vpc_delete_receipt_corrupt")
host_ids = receipt.get("removed_host_ids")
if (not isinstance(host_ids, list) or len(host_ids) > VPC_MAX_HOSTS
or host_ids != sorted(set(host_ids))):
raise VpcError("VPC 删除回执宿主机闭集损坏", 500,
"vpc_delete_receipt_corrupt")
try:
for host_id in host_ids:
_host_id(host_id)
except VpcError as exc:
raise VpcError("VPC 删除回执宿主机闭集损坏", 500,
"vpc_delete_receipt_corrupt") from exc
# The signed desired digest already commits to the retiring-member
# closure. Once absence is confirmed, replay only needs the operation
# identity, digest, host closure and completion time. Compact an
# older receipt on first read so account deletion does not leave VM
# UUID, private IP and MAC data in the engine indefinitely.
compact = {key: receipt[key] for key in (
"schema_version", "vpc_id", "operation_id", "desired_revision",
"desired_digest", "state", "deleted", "readback",
"removed_host_ids", "completed_at",
)}
compact["evidence_version"] = int(receipt.get("evidence_version", 0))
if receipt != compact:
self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, compact)
return compact
@staticmethod
def _public_delete_receipt(receipt: Mapping[str, Any]) -> dict[str, Any]:
return {key: receipt[key] for key in (
"vpc_id", "operation_id", "evidence_version",
"desired_revision", "desired_digest",
"state", "deleted", "readback", "removed_host_ids",
"completed_at",
)}
def _delete_payload_from_record(
self, record: Mapping[str, Any],
) -> dict[str, Any]:
"""Rebuild the exact signed delete envelope for durable recovery."""
marker = self._delete_marker(record)
if marker is None:
raise VpcError("VPC 删除目标记录缺失", 500,
"vpc_record_corrupt")
plan = _plan_from_record(record)
if (plan.operation_id != marker["operation_id"]
or plan.desired_revision != marker["desired_revision"]
or plan.desired_digest != marker["desired_digest"]):
raise VpcError("VPC 删除目标记录不一致", 500,
"vpc_record_corrupt")
value = plan.canonical()
value["operation_id"] = marker["operation_id"]
value["desired_digest"] = marker["desired_digest"]
value["retiring_members"] = list(marker["retiring_members"])
return value
def _purge_deleted_vpc(self, vpc_id: str) -> None:
prefix = vpc_id + "."
for kind in ("vpc_subnet", "vpc_member", "vpc_route", "vpc_acl",
"vpc_host", VPC_GUEST_EXEC_KIND):
for key in tuple(self.store.list(kind)):
if key.startswith(prefix):
self.store.delete(kind, key)
self.store.delete("vpc_desired", vpc_id)
def _fabric_gate(self, plan: VpcPlan) -> dict[str, Any]:
try:
status = self.fabric.get_status(plan.fabric_id)
except FabricError as exc:
return {
"active": False, "available": False,
"error_code": exc.error_code,
}
except Exception:
return {"active": False, "available": False,
"error_code": "vpc_fabric_status_unavailable"}
if not isinstance(status, Mapping):
return {"active": False, "available": False,
"error_code": "vpc_fabric_status_invalid"}
if status.get("region") != plan.region:
return {
"active": False, "available": True,
"fabric_id": plan.fabric_id,
"fabric_revision": status.get("revision"),
"state": status.get("state"),
"next_stage_allowed": False,
"error_code": "vpc_fabric_region_mismatch",
}
if status.get("fabric_id") != plan.fabric_id:
return {
"active": False, "available": True,
"fabric_id": status.get("fabric_id"),
"fabric_revision": status.get("revision"),
"state": status.get("state"),
"next_stage_allowed": False,
"error_code": "vpc_fabric_identity_mismatch",
}
raw_nodes = status.get("nodes")
try:
vpc_network = ipaddress.IPv4Network(plan.cidr, strict=True)
endpoint_addresses: list[ipaddress.IPv4Address] = []
if not isinstance(raw_nodes, list):
raise ValueError("Fabric node inventory missing")
for item in raw_nodes:
if not isinstance(item, Mapping):
raise ValueError("Fabric node invalid")
endpoint = item.get("endpoint")
if not isinstance(endpoint, str):
raise ValueError("Fabric endpoint missing")
parts = endpoint.rsplit(":", 1)
if len(parts) != 2 or not parts[1].isdigit():
raise ValueError("Fabric endpoint invalid")
port = int(parts[1])
if not 1 <= port <= 65535:
raise ValueError("Fabric endpoint invalid")
endpoint_addresses.append(ipaddress.IPv4Address(parts[0]))
except (TypeError, ValueError, ipaddress.AddressValueError):
return {
"active": False, "available": False,
"error_code": "vpc_fabric_status_invalid",
}
if any(address in vpc_network for address in endpoint_addresses):
return {
"active": False, "available": True,
"fabric_id": plan.fabric_id,
"fabric_revision": status.get("revision"),
"state": status.get("state"),
"next_stage_allowed": False,
"error_code": "vpc_fabric_endpoint_overlap",
}
fabric_hosts = {
item.get("host_id") for item in raw_nodes
if isinstance(item, Mapping) and isinstance(item.get("host_id"), str)
} if isinstance(raw_nodes, list) else set()
requested_hosts = set(plan.host_ids)
raw_revision = status.get("revision")
same_revision = (
isinstance(raw_revision, int) and not isinstance(raw_revision, bool)
and raw_revision == plan.fabric_revision
)
active = (
status.get("state") == "active"
and status.get("next_stage_allowed") is True
and same_revision
and bool(plan.members)
and requested_hosts
and requested_hosts.issubset(fabric_hosts)
)
error_code = None
if not requested_hosts:
error_code = "vpc_no_participating_hosts"
elif not plan.members:
error_code = "vpc_no_members"
elif not requested_hosts.issubset(fabric_hosts):
error_code = "vpc_fabric_host_missing"
elif not same_revision:
error_code = "vpc_fabric_revision_mismatch"
elif status.get("state") != "active" or status.get("next_stage_allowed") is not True:
error_code = "vpc_fabric_not_active"
return {
"active": bool(active), "available": True,
"fabric_id": plan.fabric_id,
"fabric_revision": status.get("revision"),
"state": status.get("state"),
"next_stage_allowed": status.get("next_stage_allowed") is True,
"error_code": error_code,
}
@staticmethod
def _require_fabric_catalog_admission(gate: Mapping[str, Any]) -> None:
"""Reject deterministic Fabric/VPC identity and address conflicts."""
error_code = gate.get("error_code")
if error_code in VPC_FABRIC_ADMISSION_ERRORS:
raise VpcError(
"VPC 与 Fabric 的地区、身份或 underlay 地址冲突",
409, str(error_code))
def _validate_global_fabric_endpoints(self, plan: VpcPlan) -> None:
"""Keep every durable Fabric underlay outside the new tenant CIDR."""
tenant_network = ipaddress.IPv4Network(plan.cidr, strict=True)
for record_key, raw in self.store.list("fabric_desired").items():
try:
value = json.loads(raw)
if (not isinstance(value, Mapping)
or value.get("fabric_id") != record_key):
raise ValueError("Fabric identity mismatch")
fabric_plan = FabricCoordinator._plan_from_record(value)
if value.get("plan_digest") != fabric_plan.digest:
raise ValueError("Fabric plan digest mismatch")
except (TypeError, ValueError, FabricError, KeyError) as exc:
raise VpcError(
"Fabric 资源目录损坏,无法确认全局 underlay 地址",
500, "vpc_fabric_record_corrupt") from exc
for node in fabric_plan.nodes:
try:
endpoint = ipaddress.IPv4Address(
node.endpoint.rsplit(":", 1)[0])
except (ValueError, ipaddress.AddressValueError) as exc:
raise VpcError(
"Fabric endpoint 记录损坏",
500, "vpc_fabric_record_corrupt") from exc
if endpoint in tenant_network:
raise VpcError(
"VPC CIDR 与现有 Fabric underlay endpoint 冲突",
409, "vpc_fabric_endpoint_overlap")
def _fabric_record(self, plan: VpcPlan) -> dict[str, Any]:
"""Load and validate the durable Fabric identity used by the data plane."""
loader = getattr(self.fabric, "_load", None)
raw = loader("fabric_desired", plan.fabric_id) if callable(loader) else None
if not isinstance(raw, Mapping):
raise VpcError("VPC Fabric 记录不可用", 409,
"vpc_fabric_record_unavailable")
try:
fabric_plan = self.fabric._plan_from_record(raw)
except Exception as exc:
raise VpcError("VPC Fabric 记录无效", 500,
"vpc_fabric_record_corrupt") from exc
if (fabric_plan.fabric_id != plan.fabric_id
or fabric_plan.region != plan.region
or fabric_plan.revision != plan.fabric_revision):
raise VpcError("VPC Fabric revision 或地区不匹配", 409,
"vpc_fabric_revision_mismatch")
if raw.get("plan_digest") != fabric_plan.digest:
raise VpcError("VPC Fabric 计划摘要无效", 500,
"vpc_fabric_record_corrupt")
return dict(raw)
def _validate_dataplane_resources(self, plan: VpcPlan) -> None:
"""Reject deterministic resource collisions before touching a host."""
from .vpc_dataplane import resource_names
current = resource_names(plan.vpc_id)
seen: dict[str, str] = {}
for field in ("bridge", "nft_table", "route_table"):
seen[str(getattr(current, field))] = plan.vpc_id
for priority in (current.rule_priority, current.rule_priority + 1):
seen[f"rule:{priority}"] = plan.vpc_id
for key, raw in self.store.list("vpc_desired").items():
if key == plan.vpc_id:
continue
try:
other = _plan_from_record(json.loads(raw))
except (TypeError, ValueError, VpcError) as exc:
raise VpcError("VPC 资源目录损坏", 500,
"vpc_record_corrupt") from exc
resources = resource_names(other.vpc_id)
for field in ("bridge", "nft_table", "route_table"):
marker = str(getattr(resources, field))
owner = seen.get(marker)
if owner is not None and owner != other.vpc_id:
raise VpcError("VPC 数据面资源发生冲突", 409,
"vpc_dataplane_resource_conflict")
seen[marker] = other.vpc_id
for priority in (resources.rule_priority, resources.rule_priority + 1):
marker = f"rule:{priority}"
owner = seen.get(marker)
if owner is not None and owner != other.vpc_id:
raise VpcError("VPC 数据面资源发生冲突", 409,
"vpc_dataplane_resource_conflict")
seen[marker] = other.vpc_id
def _fabric_peers_for_plan(
self, plan: VpcPlan, fabric_record: Mapping[str, Any],
) -> list[dict[str, Any]]:
"""Build per-peer overlay plus tenant /32 AllowedIPs.
WireGuard has one AllowedIPs list per peer. Every VPC sharing a
Fabric therefore contributes only its exact member /32s to the peer
on the member's host. Duplicate addresses are rejected across VPCs
before a host command is rendered.
"""
from .vpc_dataplane import _key
plans: list[VpcPlan] = [plan]
for key, raw in self.store.list("vpc_desired").items():
if key == plan.vpc_id:
continue
try:
candidate = _plan_from_record(json.loads(raw))
except (TypeError, ValueError, VpcError) as exc:
raise VpcError("VPC 资源目录损坏", 500,
"vpc_record_corrupt") from exc
if (candidate.fabric_id == plan.fabric_id
and candidate.region == plan.region):
plans.append(candidate)
address_owner: dict[str, str] = {}
tenant_by_host: dict[str, set[str]] = {}
for candidate in plans:
for member in candidate.members:
cidr = f"{member.private_ip}/32"
owner = address_owner.get(cidr)
if owner is not None and owner != candidate.vpc_id:
raise VpcError("VPC 私网地址跨 VPC 重复", 409,
"vpc_private_ip_conflict")
address_owner[cidr] = candidate.vpc_id
tenant_by_host.setdefault(member.host_id, set()).add(cidr)
# A custom service route is cryptokey-routed to the host that
# owns its next hop. It must be part of that peer's exact
# AllowedIPs set just like a VM /32; otherwise WireGuard sends it
# to no peer and the Fabric quarantine drops it.
members_by_id = {item.member_id: item for item in candidate.members}
for route in candidate.routes:
target = (members_by_id.get(route.next_hop_member_id)
if route.next_hop_member_id is not None else None)
target_host = target.host_id if target is not None else route.next_hop_host_id
if target_host is None:
raise VpcError("VPC route next hop 缺失", 409,
"vpc_route_next_hop_missing")
cidr = route.destination_cidr
owner = address_owner.get(cidr)
if owner is not None and owner != candidate.vpc_id:
raise VpcError("VPC 路由地址跨 VPC 重复", 409,
"vpc_route_address_conflict")
address_owner[cidr] = candidate.vpc_id
tenant_by_host.setdefault(target_host, set()).add(cidr)
raw_nodes = fabric_record.get("nodes")
raw_keys = fabric_record.get("public_keys")
if not isinstance(raw_nodes, list) or not isinstance(raw_keys, Mapping):
raise VpcError("Fabric peer 记录不完整", 500,
"vpc_fabric_record_corrupt")
peers: list[dict[str, Any]] = []
wanted_hosts = set(plan.host_ids)
for raw in raw_nodes:
if not isinstance(raw, Mapping):
continue
host_id = raw.get("host_id")
if not isinstance(host_id, str) or host_id not in wanted_hosts:
continue
# The caller supplies the local host separately when parsing each
# host plan; keep every other participating host in this list.
public_key = raw_keys.get(host_id)
overlay_ip = raw.get("overlay_ip")
if not isinstance(public_key, str) or not isinstance(overlay_ip, str):
raise VpcError("Fabric peer 记录无效", 500,
"vpc_fabric_record_corrupt")
_key(public_key)
peers.append({
"host_id": host_id,
"public_key": public_key,
"overlay_ip": overlay_ip,
"tenant_allowed_ips": sorted(tenant_by_host.get(host_id, set())),
})
return peers
@staticmethod
def _private_routes(plan: VpcPlan, member: VpcMember) -> list[str]:
from .vpc_dataplane import private_guest_routes
return list(private_guest_routes(plan, member))
@staticmethod
def _observation_error(exc: Exception) -> str:
code = getattr(exc, "error_code", None)
if isinstance(code, str) and re.fullmatch(r"[a-z0-9_]{1,64}", code):
return code
return "vpc_private_dataplane_apply_failed"
def _run_host_dataplane(
self, plan: VpcPlan, host_id: str, component: Any,
profile: Mapping[str, Any], fabric_peers: list[dict[str, Any]],
previous_plan: VpcPlan | None,
retiring_members: Sequence[Mapping[str, Any]] | None = None,
fabric_public_keys: Mapping[str, str] | None = None,
fabric_plan_digest: str | None = None,
private_link_states: Mapping[str, str] | None = None,
request_deadline: float | None = None,
) -> dict[str, Any]:
"""Apply and read back one host synchronously behind the VPC lock."""
from .libvirt_service import VMService
from .vpc_dataplane import (
VpcDataplanePlan, VpcPrivateHostAdapter,
parse_vpc_dataplane_plan, resource_names,
)
if host_id in plan.host_ids:
payload = plan.canonical()
payload["fabric_peers"] = [
peer for peer in fabric_peers
if peer.get("host_id") != host_id
]
trusted_peer_tenant_ips = {
str(peer["host_id"]): tuple(peer.get("tenant_allowed_ips", ()))
for peer in payload["fabric_peers"]
if isinstance(peer, Mapping) and isinstance(peer.get("host_id"), str)
}
authoritative_peer_keys = None
if fabric_public_keys is not None:
authoritative_peer_keys = [
key for peer_host, key in fabric_public_keys.items()
if peer_host != host_id
]
dataplane = parse_vpc_dataplane_plan(
payload, host_id,
trusted_tenant_allowed_ips=trusted_peer_tenant_ips,
fabric_peer_keys=authoritative_peer_keys,
fabric_plan_digest=fabric_plan_digest)
# A root-delete plan extends the canonical digest with its signed
# retiring-member manifest. The host projection intentionally
# omits those members from live desired state, so restore the
# authoritative envelope identity after parsing that projection.
dataplane = replace(dataplane, vpc=plan)
else:
# A retiring host is intentionally absent from the new desired
# plan. Construct a typed empty-local plan directly so the
# removal transaction can validate the deterministic VPC
# resources without reintroducing the old host into Fabric.
dataplane = VpcDataplanePlan(
plan.vpc, host_id, tuple(plan.members), tuple(),
resource_names(plan.vpc_id),
fabric_plan_digest)
if request_deadline is None:
request_deadline = (
time.monotonic() + VPC_REQUEST_DEADLINE_SECONDS)
adapter = VpcPrivateHostAdapter(
component.manager, profile, request_deadline=request_deadline)
service = VMService(
profile["libvirt_uri"], profile["storage_pool"],
profile["iso_pool"], profile["default_network"],
rescue_iso_pool=profile.get("rescue_iso_pool", ""),
)
guest_exec_contexts: dict[str, tuple[dict[str, Any], Any, Any]] = {}
def guest_exec_context(
member: VpcMember,
) -> tuple[dict[str, Any], Any, Any]:
context = guest_exec_contexts.get(member.instance_uuid)
if context is None:
context = self._guest_exec_context(plan, member)
guest_exec_contexts[member.instance_uuid] = context
return context
def require_request_budget() -> None:
if time.monotonic() >= request_deadline:
raise VpcError(
"VPC 来宾网络核验超过请求预算", 502,
"vpc_private_guest_probe_deadline_exceeded")
# A lifecycle-triggered reconcile must reach its target before a
# large host's shared probe budget expires. Mapping membership is the
# priority signal even for ``preserve``; ordinary periodic passes have
# no entries and retain canonical member order.
priority_instances = set(private_link_states or {})
local_members = tuple(sorted(
dataplane.local_members,
key=lambda item: item.instance_uuid not in priority_instances,
))
old_members: list[VpcMember] = []
if previous_plan is not None:
old_members.extend(
member for member in previous_plan.members
if member.host_id == host_id)
if retiring_members is not None:
for raw in retiring_members:
if not isinstance(raw, Mapping):
continue
try:
private_ip = str(raw.get("private_ip_cidr", "")).removesuffix("/32")
old_members.append(VpcMember(
str(raw["member_id"]), str(raw["instance_uuid"]),
str(raw.get("host_id", host_id)), private_ip,
str(raw.get("subnet_id", "retiring")),
str(raw.get("private_mac", ""))))
except (KeyError, TypeError, ValueError):
raise VpcError("VPC 宿主机清理成员证据无效", 500,
"vpc_retiring_member_invalid")
current_ids = {member.member_id for member in plan.members}
cleanup_plan = previous_plan or plan
seen_old: set[str] = set()
for old_member in old_members:
if old_member.member_id in seen_old:
continue
seen_old.add(old_member.member_id)
if old_member.host_id != host_id:
continue
if old_member.member_id in current_ids:
continue
require_request_budget()
ledger_state, _persist_ledger, clear_ledger = (
guest_exec_context(old_member))
service.remove_private_interface(
old_member.instance_uuid,
vpc_id=cleanup_plan.vpc_id,
resources=resource_names(cleanup_plan.vpc_id),
desired_revision=cleanup_plan.desired_revision,
desired_digest=cleanup_plan.desired_digest,
expected_member_id=old_member.member_id,
expected_private_mac=_effective_private_mac(
cleanup_plan.vpc_id, old_member.member_id,
old_member.private_mac),
expected_private_ip_cidr=f"{old_member.private_ip}/32",
)
# remove_private_interface positively requires a stopped domain.
# Its old QGA process therefore cannot retain either a launch
# intent or PID record. Delete the ledger only after that proof.
clear_ledger()
ledger_state.clear()
# A host with no local members is a removal target. Run the
# manager-aware teardown and require its positive absence marker;
# returning a normal empty observation would leave stale bridge,
# route, nft, and peer state behind.
if not local_members:
require_request_budget()
removed = adapter.remove(dataplane)
removed.update({
"vpc_id": plan.vpc_id,
"host_id": host_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_revision": plan.desired_revision,
"observed_digest": plan.desired_digest,
"transport_verified": True,
"private_dataplane_verified": True,
"rollback": removed.get("rollback", "none"),
})
return removed
require_request_budget()
adapter.apply(dataplane)
pairs: dict[str, dict[str, Any]] = {}
probe_members: list[VpcMember] = []
for member in local_members:
require_request_budget()
ledger_state, _persist_ledger, _clear_ledger = (
guest_exec_context(member))
configured = service.ensure_private_interface(
member.instance_uuid, member=member,
resources=dataplane.resources, vpc_id=plan.vpc_id,
desired_revision=plan.desired_revision,
desired_digest=plan.desired_digest,
private_routes=self._private_routes(plan, member),
link_state=(private_link_states or {}).get(
member.instance_uuid, "preserve"),
)
pair = {
"inactive": configured.get("xml"),
"live": configured.get("live"),
"seed_verified": configured.get("seed_verified") is True,
}
if configured.get("live") is None:
pair["guest_network"] = {
"state": "stopped", "verified": False,
"reason_code": "vpc_private_guest_stopped",
"observed_at": int(time.time()),
}
if ledger_state:
probe_members.append(member)
elif configured.get("restart_required") is True:
pair["guest_network"] = {
"state": "pending_restart", "verified": False,
"reason_code": "vpc_private_guest_restart_required",
"observed_at": int(time.time()),
}
if ledger_state:
probe_members.append(member)
else:
# This placeholder is deliberately fail-closed. A completed
# QGA future replaces it; an unsubmitted/cancelled future
# remains explicit deadline evidence while preserving the
# persistent/live XML already read back by the transaction.
pair["guest_network"] = {
"state": "deadline_exceeded", "verified": False,
"reason_code":
"vpc_private_guest_probe_deadline_exceeded",
"observed_at": int(time.time()),
}
probe_members.append(member)
pairs[member.instance_uuid] = pair
def read_member(member: VpcMember) -> tuple[VpcMember, dict[str, Any]]:
ledger_state, persist_ledger, clear_ledger = (
guest_exec_context(member))
configured = pairs.get(member.instance_uuid, {})
def deadline_pair() -> dict[str, Any]:
return {
"inactive": configured.get("inactive"),
"live": configured.get("live"),
"seed_verified": bool(configured.get("seed_verified")),
"guest_network": {
"state": "deadline_exceeded", "verified": False,
"reason_code":
"vpc_private_guest_probe_deadline_exceeded",
"observed_at": int(time.time()),
},
}
remaining = max(0.0, request_deadline - time.monotonic())
acquired = _VPC_GUEST_PROBE_SLOTS.acquire(timeout=remaining)
if not acquired:
return member, deadline_pair()
try:
# QGA rounds its command timeout to whole seconds. Do not
# enter libvirt after waiting for a slot if less than one
# useful command second remains in the shared request budget.
if request_deadline - time.monotonic() < 1:
return member, deadline_pair()
pair = service.read_private_domain_pair(
member.instance_uuid,
private_mac=member.private_mac,
private_ip_cidr=f"{member.private_ip}/32",
private_routes=self._private_routes(plan, member),
guest_probe_deadline=request_deadline,
guest_exec_state=ledger_state,
guest_exec_persist=persist_ledger,
guest_exec_clear=clear_ledger,
)
pair["seed_verified"] = bool(
configured.get("seed_verified"))
return member, pair
finally:
_VPC_GUEST_PROBE_SLOTS.release()
# Keep at most one worker-width of QGA calls submitted. This avoids a
# 4,096-future queue, while the process-wide semaphore prevents pools
# belonging to concurrent VPC requests from multiplying that width.
# Both gates share the request deadline, and every unstarted member is
# explicit fail-closed evidence. Shutdown waits for bounded calls, so
# no probe thread survives this request.
if probe_members:
workers = min(VPC_GUEST_PROBE_WORKERS, len(probe_members))
executor = ThreadPoolExecutor(
max_workers=workers,
thread_name_prefix="vpc-guest-probe")
futures: dict[Any, VpcMember] = {}
next_member = 0
probe_error: BaseException | None = None
def submit_available() -> None:
nonlocal next_member
while (len(futures) < workers
and next_member < len(probe_members)
and request_deadline - time.monotonic() >= 1):
member = probe_members[next_member]
next_member += 1
futures[executor.submit(read_member, member)] = member
try:
submit_available()
while futures:
remaining = request_deadline - time.monotonic()
if remaining <= 0:
break
done, _pending = wait(
tuple(futures), timeout=remaining,
return_when=FIRST_COMPLETED)
if not done:
break
for future in done:
futures.pop(future, None)
try:
member, pair = future.result()
except BaseException as exc:
probe_error = exc
break
pairs[member.instance_uuid] = pair
if probe_error is not None:
break
submit_available()
finally:
for future in futures:
future.cancel()
executor.shutdown(wait=True, cancel_futures=True)
if probe_error is not None:
raise probe_error
xml_by_instance: dict[str, Mapping[str, Any]] = {}
for member in local_members:
pair = pairs[member.instance_uuid]
inactive = pair.get("inactive")
if not isinstance(inactive, str):
raise VpcError("私网网卡持久配置回读缺失", 502,
"vpc_private_xml_readback_missing")
live = pair.get("live")
if live is not None and not isinstance(live, str):
raise VpcError("私网网卡实时配置回读无效", 502,
"vpc_private_live_readback_missing")
xml_by_instance[member.instance_uuid] = {
"inactive": inactive, "live": live,
"guest_network": pair.get("guest_network"),
"seed_verified": pair.get("seed_verified") is True,
}
observed = adapter.observe(dataplane, xml_by_instance=xml_by_instance)
observed.update({
"vpc_id": plan.vpc_id,
"host_id": host_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"transport_verified": True,
"private_dataplane_verified": observed.get("state") == "active",
"rollback": observed.get("rollback", "none"),
})
return observed
@asynccontextmanager
async def _instance_locks(
self, plan: VpcPlan, previous_plan: VpcPlan | None = None,
extra_identities: Sequence[tuple[str, str]] | None = None,
):
if not callable(self.vm_lock):
yield
return
identities = {
(member.host_id, member.instance_uuid)
for member in plan.members
}
if previous_plan is not None:
identities.update(
(member.host_id, member.instance_uuid)
for member in previous_plan.members
)
# A removed member is no longer present in either VPC plan. Its
# durable retiring_members row is still a live libvirt target, so the
# cleanup pass must share the ordinary VM lifecycle lock as well.
if extra_identities is not None:
for host_id, instance_uuid in extra_identities:
if (isinstance(host_id, str)
and isinstance(instance_uuid, str)):
identities.add((host_id, instance_uuid))
async with AsyncExitStack() as stack:
for host_id, instance_uuid in sorted(identities):
await stack.enter_async_context(
self.vm_lock(host_id, instance_uuid))
yield
def _lifecycle_conflict(
self, host_id: str, instance_uuid: str,
operation_id: str | None = None,
) -> str | None:
"""Return a fail-closed reason while a host-local VM job owns XML.
Reinstall, rescue and online-snapshot workers are queued by the child
host app and continue after the HTTP operation has returned 202. The
regular target lock therefore cannot cover their whole physical
mutation. Read their durable rows before a VPC pass touches libvirt;
an uncertain row is deliberately treated as active until an operator
reconciles it.
"""
registry = self.registry
if registry is None:
# Pure control-plane callers intentionally have no host job store;
# they cannot race a libvirt worker and keep the historical unit
# harness behavior.
return None
components = getattr(registry, "components", {}).get(host_id)
store = getattr(components, "store", None)
if store is None or not callable(getattr(store, "list", None)):
return "vpc_lifecycle_state_unavailable"
active_states = {"queued", "running", "pending", "uncertain"}
try:
same_uncertain_snapshots: set[tuple[str, str]] = set()
same_uncertain_rescue = False
for kind in ("reinstall_job", "rescue_job", "snapshot_job"):
rows = store.list(kind)
if not isinstance(rows, Mapping):
return "vpc_lifecycle_state_unavailable"
for raw in rows.values():
try:
job = json.loads(raw)
except (TypeError, ValueError, UnicodeError):
return "vpc_lifecycle_state_unavailable"
if not isinstance(job, Mapping):
return "vpc_lifecycle_state_unavailable"
same_uncertain_operation = (
job.get("status") == "uncertain"
and operation_id is not None
and job.get("operation_id") == operation_id
and job.get("vm_id") == instance_uuid
and (
(kind == "snapshot_job"
and isinstance(job.get("snapshot_key"), str))
or (kind == "reinstall_job"
and _reinstall_job_committed(job))
)
)
# A rescue job's uncertain row is not evidence by itself.
# Defer this one same-operation case until the vm_rescue
# state below proves that the rescue XML reached its
# active/staged terminal state.
if (kind == "rescue_job"
and job.get("status") == "uncertain"
and operation_id is not None
and job.get("operation_id") == operation_id
and job.get("vm_id") == instance_uuid):
same_uncertain_rescue = True
continue
# An uncertain reinstall is exempt only after its
# durable post-commit marker proves the replacement
# domain exists. Rescue jobs have no generic exemption;
# their vm_rescue active/staged row below is the only
# same-operation terminal proof.
if (kind == "snapshot_job" and same_uncertain_operation
and job.get("vm_id") == instance_uuid
and isinstance(job.get("snapshot_key"), str)):
# The catalog row has no operation_id of its own; the
# durable snapshot job is the identity that proves a
# lifecycle worker is reconciling this exact row.
same_uncertain_snapshots.add(
(instance_uuid, job["snapshot_key"])
)
if (job.get("vm_id") == instance_uuid
and job.get("status") in active_states
and not same_uncertain_operation):
return "vpc_vm_lifecycle_conflict"
# Rescue mode has a durable state row even after its worker is no
# longer queued. Restoring the original XML while VPC apply is in
# flight would otherwise silently remove private0 again.
rescue_rows = store.list("vm_rescue")
if not isinstance(rescue_rows, Mapping):
return "vpc_lifecycle_state_unavailable"
same_terminal_rescue = False
for key, raw in rescue_rows.items():
try:
state = json.loads(raw)
except (TypeError, ValueError, UnicodeError):
return "vpc_lifecycle_state_unavailable"
if not isinstance(state, Mapping):
return "vpc_lifecycle_state_unavailable"
key_matches = isinstance(key, str) and key.endswith(
"." + instance_uuid)
if state.get("vm_id") == instance_uuid or key_matches:
state_status = state.get("status")
# Enter/force rescue finishes the libvirt XML mutation
# before its lifecycle worker asks for the private
# readback. Keep the durable active/staged marker as
# protection for every other operation, but let the
# authenticated rescue operation reconcile the terminal
# state it just created. Queued/running/exiting/failed
# rows remain blocking even when their operation id
# matches: those states have not proved that the XML
# mutation is complete (or need manual recovery).
same_terminal = (
state_status in {"active", "staged"}
and operation_id is not None
and state.get("operation_id") == operation_id
)
if same_terminal:
same_terminal_rescue = True
elif state_status not in {"completed", "deleted"}:
return "vpc_vm_lifecycle_conflict"
if same_uncertain_rescue and not same_terminal_rescue:
return "vpc_vm_lifecycle_conflict"
# A transitional snapshot catalog row without its job row is a
# durable indication that the previous process died at a physical
# boundary. Do not overwrite its domain XML until reconciliation.
snapshot_rows = store.list("vm_snapshot")
if not isinstance(snapshot_rows, Mapping):
return "vpc_lifecycle_state_unavailable"
for _key, raw in snapshot_rows.items():
try:
snapshot = json.loads(raw)
except (TypeError, ValueError, UnicodeError):
return "vpc_lifecycle_state_unavailable"
if (isinstance(snapshot, Mapping)
and snapshot.get("vm_id") == instance_uuid
and snapshot.get("state") in {
"creating", "restoring", "deleting", "uncertain",
}
and (snapshot.get("vm_id"),
snapshot.get("snapshot_key"))
not in same_uncertain_snapshots):
return "vpc_vm_lifecycle_conflict"
except Exception:
# A database/host-store read failure must not be interpreted as an
# empty job set, because that could race a destructive worker.
return "vpc_lifecycle_state_unavailable"
return None
def _mark_lifecycle_conflicts(
self, plan: VpcPlan, previous_plan: VpcPlan | None,
record: dict[str, Any], operation_id: str | None = None,
) -> bool:
identities: dict[tuple[str, str], None] = {}
for candidate in (plan.members
+ (previous_plan.members if previous_plan else ())):
identities[(candidate.host_id, candidate.instance_uuid)] = None
for host in record.get("hosts", []):
if not isinstance(host, Mapping):
continue
host_id = host.get("host_id")
for row in host.get("retiring_members", []):
if (isinstance(host_id, str) and isinstance(row, Mapping)
and isinstance(row.get("instance_uuid"), str)):
identities[(host_id, row["instance_uuid"])] = None
conflict = False
reasons_by_host: dict[str, str] = {}
for (host_id, instance_uuid) in identities:
reason = self._lifecycle_conflict(
host_id, instance_uuid, operation_id)
if reason is not None:
conflict = True
reasons_by_host[host_id] = reason
if not conflict:
return False
for host in record.get("hosts", []):
if not isinstance(host, dict):
continue
host_id = host.get("host_id")
reason = reasons_by_host.get(host_id)
if reason is not None:
host.update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": reason,
})
return True
async def _reconcile_dataplane(
self, plan: VpcPlan, record: dict[str, Any], gate: Mapping[str, Any],
previous_plan: VpcPlan | None = None,
private_link_states: Mapping[str, str] | None = None,
extra_identities: Sequence[tuple[str, str]] | None = None,
operation_id: str | None = None,
) -> None:
# VPC lock is acquired by each public caller before this method. The
# VM locks are acquired in stable order here, so a VPC apply cannot
# race a normal lifecycle mutation or a targeted private reconcile.
# Derive every durable tombstone identity as well as any explicit
# target supplied by a lifecycle caller. A host pass removes all
# retiring members in one transaction; locking only the requested VM
# would leave its sibling tombstones exposed to delete/reinstall.
derived: list[tuple[str, str]] = []
for host in record.get("hosts", []):
if not isinstance(host, Mapping):
continue
host_id = host.get("host_id")
if not isinstance(host_id, str):
continue
for row in host.get("retiring_members", []):
if (isinstance(row, Mapping)
and isinstance(row.get("instance_uuid"), str)):
derived.append((host_id, row["instance_uuid"]))
if extra_identities is None:
extra_identities = derived
else:
extra_identities = tuple(extra_identities) + tuple(derived)
async with self._instance_locks(
plan, previous_plan, extra_identities):
kwargs: dict[str, Any] = {}
if private_link_states is not None:
kwargs["private_link_states"] = private_link_states
if operation_id is not None:
kwargs["operation_id"] = operation_id
await self._reconcile_dataplane_unlocked(
plan, record, gate, previous_plan, **kwargs)
async def _reconcile_dataplane_unlocked(
self, plan: VpcPlan, record: dict[str, Any], gate: Mapping[str, Any],
previous_plan: VpcPlan | None = None,
private_link_states: Mapping[str, str] | None = None,
operation_id: str | None = None,
) -> None:
"""Apply/read back every participating host; stay closed on failure."""
request_deadline = time.monotonic() + VPC_REQUEST_DEADLINE_SECONDS
retiring_ids = {
item.get("host_id") for item in record.get("hosts", [])
if isinstance(item, Mapping)
and item.get("retiring") is True
and isinstance(item.get("host_id"), str)
}
member_cleanup = any(
isinstance(item, Mapping)
and isinstance(item.get("retiring_members"), list)
and bool(item.get("retiring_members"))
for item in record.get("hosts", [])
)
# A Fabric gate is required for normal applies. Retiring hosts still
# need one final SSH pass even when the new VPC has no members and the
# Fabric coordinator has already marked the tenant gate inactive.
if self.registry is None or (
gate.get("active") is not True
and not retiring_ids and not member_cleanup):
self._finish(record, gate)
return
if self._mark_lifecycle_conflicts(
plan, previous_plan, record, operation_id):
self._finish(record, gate)
return
try:
self._validate_dataplane_resources(plan)
fabric_record = self._fabric_record(plan)
full_fabric_plan = self.fabric._plan_from_record(fabric_record)
fabric_plan_digest = full_fabric_plan.digest
raw_fabric_keys = fabric_record.get("public_keys")
if not isinstance(raw_fabric_keys, Mapping):
raise VpcError("Fabric 公钥记录不完整", 500,
"vpc_fabric_record_corrupt")
fabric_public_keys = {
node.host_id: raw_fabric_keys.get(node.host_id)
for node in full_fabric_plan.nodes
}
if any(not isinstance(key, str) for key in fabric_public_keys.values()):
raise VpcError("Fabric 公钥记录不完整", 500,
"vpc_fabric_record_corrupt")
fabric_plan = full_fabric_plan
# ``record`` is the execution closure for this pass. Root-delete
# retries deliberately project it down to hosts that do not yet
# have an exact durable absence proof; deriving this set from the
# immutable submitted plan would still reprobe an already removed
# host and could block every remaining cleanup when that host is
# now offline. Normal apply records already contain both desired
# and retiring hosts, so the same rule keeps their host closure
# exact as well.
wanted = {
item.get("host_id") for item in record.get("hosts", [])
if isinstance(item, Mapping)
and isinstance(item.get("host_id"), str)
}
fabric_plan = replace(
fabric_plan,
nodes=tuple(node for node in fabric_plan.nodes
if node.host_id in wanted),
)
components = await self.fabric._components(fabric_plan)
all_peers = self._fabric_peers_for_plan(plan, fabric_record)
except Exception as exc:
code = self._observation_error(exc)
for host in record.get("hosts", []):
if isinstance(host, dict):
host.update({"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"last_error_code": code})
self._finish(record, gate)
return
for host in record.get("hosts", []):
if not isinstance(host, dict):
continue
host_id = host.get("host_id")
if time.monotonic() >= request_deadline:
host.update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code":
"vpc_private_guest_probe_deadline_exceeded",
})
continue
if not isinstance(host_id, str) or host_id not in components:
host.update({"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"last_error_code": "vpc_host_unavailable"})
continue
component, profile = components[host_id]
try:
# Persist a fair retry order. If this host consumes the shared
# budget, never-probed/older hosts sort ahead of it on the
# next reconcile even when this runtime proof stayed pending.
host["probe_attempted_at_ns"] = time.time_ns()
observed = await asyncio.to_thread(
self._run_host_dataplane, plan, host_id, component,
profile, all_peers, previous_plan,
host.get("retiring_members", []), fabric_public_keys,
fabric_plan_digest, private_link_states,
request_deadline=request_deadline)
if time.monotonic() >= request_deadline:
member_nics = observed.get("member_nics")
if isinstance(member_nics, list):
for member_nic in member_nics:
if not isinstance(member_nic, dict):
continue
member_nic.update({
"guest_network_verified": False,
"guest_network_readback":
"deadline_exceeded",
"guest_reason_code":
"vpc_private_guest_probe_deadline_exceeded",
})
observed.update({
"state": "applying",
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"reason_code":
"vpc_private_guest_probe_deadline_exceeded",
})
host.update(observed)
host["observed_revision"] = (
plan.desired_revision
if observed.get("state") == "active" else 0)
host["observed_digest"] = (
plan.desired_digest
if observed.get("state") == "active" else None)
host["state"] = observed.get("state", "degraded")
host["last_error_code"] = observed.get("reason_code")
if host.get("retiring") is True:
host["removed"] = observed.get("removed") is True
if (host.get("retiring_members")
and observed.get("config_verified") is True
and observed.get("config_observed_revision")
== plan.desired_revision
and observed.get("config_observed_digest")
== plan.desired_digest):
# The host adapter has positively removed every member
# listed in this tombstone. Guest runtime readiness for a
# different member is deliberately irrelevant to this
# persistent removal proof.
host["retiring_members"] = []
except Exception as exc:
host.update({"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": self._observation_error(exc)})
self._finish(record, gate)
@classmethod
def _desired_record(cls, plan: VpcPlan, gate: Mapping[str, Any], *, state: str,
observed_revision: int = 0,
observed_digest: str | None = None,
previous_hosts: Mapping[str, Mapping[str, Any]] | None = None,
previous_plan: VpcPlan | None = None,
updated_at: int | None = None) -> dict[str, Any]:
now = int(time.time()) if updated_at is None else updated_at
previous_hosts = previous_hosts or {}
host_records = []
current_members_by_host: dict[str, set[str]] = {}
for member in plan.members:
current_members_by_host.setdefault(member.host_id, set()).add(
member.member_id)
for host in plan.hosts:
old = previous_hosts.get(host.host_id)
old_revision = old.get("observed_revision", 0) if isinstance(old, Mapping) else 0
old_digest = old.get("observed_digest") if isinstance(old, Mapping) else None
old_readback = old.get("readback", {}) if isinstance(old, Mapping) else {}
if (isinstance(old_revision, bool) or not isinstance(old_revision, int)
or old_revision < 0 or old_revision > plan.desired_revision):
old_revision = 0
old_digest = None
if old_digest is not None:
try:
old_digest = _digest(old_digest, "observed_digest")
except VpcError:
old_digest = None
if isinstance(old_readback, str) and len(old_readback) <= 128:
safe_readback: Any = old_readback
elif isinstance(old_readback, Mapping):
safe_readback = {
str(key): value for key, value in old_readback.items()
if isinstance(key, str) and len(key) <= 64
and _safe_scalar(value)
}
else:
safe_readback = {}
retiring_members: list[dict[str, Any]] = []
if isinstance(old, Mapping):
prior_retiring = old.get("retiring_members", [])
if isinstance(prior_retiring, list):
retiring_members.extend(
dict(item) for item in prior_retiring
if isinstance(item, Mapping))
if previous_plan is not None:
current_ids = current_members_by_host.get(host.host_id, set())
for member in previous_plan.members:
if (member.host_id == host.host_id
and member.member_id not in current_ids
and not any(
item.get("member_id") == member.member_id
for item in retiring_members)):
retiring_members.append({
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"host_id": member.host_id,
"private_ip_cidr": f"{member.private_ip}/32",
"subnet_id": member.subnet_id,
"private_mac": _effective_private_mac(
plan.vpc_id, member.member_id,
member.private_mac),
})
host_records.append({
"host_id": host.host_id, "member_ids": list(host.member_ids),
"desired_revision": plan.desired_revision,
"vpc_id": plan.vpc_id,
"desired_digest": plan.desired_digest,
"observed_revision": old_revision,
"observed_digest": old_digest,
"state": "applying" if gate.get("active") else "uncertain",
"transport_verified": bool(gate.get("active")),
# Fabric transport is necessary but never sufficient for a
# tenant VPC. Only the future host adapter may set this
# after private0/route/nft/member readback.
"private_dataplane_verified": False,
"config_verified": False,
"config_observed_revision": 0,
"config_observed_digest": None,
"removed": bool(old.get("removed", False)) if isinstance(old, Mapping) else False,
"bridge": old.get("bridge") if isinstance(old, Mapping) else None,
"bridge_owner": old.get("bridge_owner") if isinstance(old, Mapping) else None,
"nft_table": old.get("nft_table") if isinstance(old, Mapping) else None,
"anti_spoof": bool(old.get("anti_spoof")) if isinstance(old, Mapping) else False,
"rp_filter": old.get("rp_filter") if isinstance(old, Mapping) else None,
"proxy_arp": old.get("proxy_arp") if isinstance(old, Mapping) else None,
"proxy_arp_pvlan": old.get("proxy_arp_pvlan") if isinstance(old, Mapping) else None,
"proxy_ips": old.get("proxy_ips", []) if isinstance(old, Mapping) else [],
"acl_revision": old.get("acl_revision") if isinstance(old, Mapping) else None,
"rollback": old.get("rollback", "none") if isinstance(old, Mapping) else "none",
"member_nics": old.get("member_nics", []) if isinstance(old, Mapping) else [],
"routes": old.get("routes", []) if isinstance(old, Mapping) else [],
"wg_allowed_ips": old.get("wg_allowed_ips", []) if isinstance(old, Mapping) else [],
"readback": safe_readback,
"retiring_members": retiring_members,
})
# Keep host tombstones in the durable engine record until the host-side
# removal script returns its typed absence proof. They are excluded
# from the new VPC plan by _plan_from_record(retiring=true), but retain
# the old instance identity so a restart can still remove private0.
current_host_ids = {host.host_id for host in plan.hosts}
retiring_hosts: dict[str, list[dict[str, Any]]] = {}
if previous_plan is not None:
for old_host in previous_plan.hosts:
if old_host.host_id in current_host_ids:
continue
old_members = [
{"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"host_id": member.host_id,
"private_ip_cidr": f"{member.private_ip}/32",
"subnet_id": member.subnet_id,
"private_mac": _effective_private_mac(
plan.vpc_id, member.member_id, member.private_mac)}
for member in previous_plan.members
if member.host_id == old_host.host_id
]
old = previous_hosts.get(old_host.host_id)
if (isinstance(old, Mapping)
and not old_members
and cls._confirmed_removal(
old, previous_plan.desired_revision,
previous_plan.desired_digest)):
# PHP historically kept a deleting deployment in the
# desired host list until this exact empty-host proof was
# durable. Once the next revision omits that host, the
# proof is already terminal: recreating a retiring row
# would recontact a resource that has been positively
# removed and would expose it as an unexpected host again.
continue
retiring_hosts[old_host.host_id] = old_members
# ``_plan_from_record`` deliberately excludes retiring rows from
# the typed live plan. Carry any still-unconfirmed rows directly
# from the durable host closure so a second newer revision cannot
# silently discard the tombstone (and make _save_children publish
# a false removal marker). Exact absence proofs remain terminal
# and are pruned instead of being reprobed.
for old_host_id, old in previous_hosts.items():
if (old_host_id in current_host_ids
or old_host_id in retiring_hosts
or old.get("retiring") is not True):
continue
if cls._confirmed_removal(
old, previous_plan.desired_revision,
previous_plan.desired_digest):
continue
retiring_hosts[old_host_id] = cls._durable_retiring_members(
old, vpc_id=plan.vpc_id, host_id=old_host_id)
for old_host_id, old_members in retiring_hosts.items():
host_records.append({
"host_id": old_host_id,
"member_ids": [item["member_id"] for item in old_members],
"vpc_id": plan.vpc_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_revision": 0,
"observed_digest": None,
"state": "deleting",
"transport_verified": bool(gate.get("active")),
"private_dataplane_verified": False,
"config_verified": False,
"config_observed_revision": 0,
"config_observed_digest": None,
"retiring": True,
"retiring_members": old_members,
"bridge": None, "bridge_owner": None, "nft_table": None,
"anti_spoof": False, "rp_filter": None,
"proxy_arp": None, "proxy_arp_pvlan": None,
"proxy_ips": [], "acl_revision": None,
"rollback": "none", "member_nics": [], "routes": [],
"wg_allowed_ips": [], "readback": {},
})
return {
"schema_version": VPC_SCHEMA_VERSION,
"evidence_version": VPC_EVIDENCE_VERSION,
"vpc_id": plan.vpc_id, "region": plan.region,
"fabric_id": plan.fabric_id,
"fabric_revision": plan.fabric_revision,
"cidr": plan.cidr,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_revision": observed_revision,
"observed_digest": observed_digest,
"operation_id": plan.operation_id,
"state": state,
"next_stage_allowed": False,
"private_dataplane_verified": False,
"fabric_gate": dict(gate),
"hosts": host_records,
"subnets": [item.__dict__ for item in plan.subnets],
"members": [
{"member_id": item.member_id,
"instance_uuid": item.instance_uuid,
"host_id": item.host_id,
"private_ip_cidr": f"{item.private_ip}/32",
"subnet_id": item.subnet_id,
**({"private_mac": item.private_mac}
if item.private_mac else {})}
for item in plan.members
],
"routes": [item.__dict__ for item in plan.routes],
"acls": [item.__dict__ for item in plan.acls],
"updated_at": now,
}
@staticmethod
def _public(value: Mapping[str, Any]) -> dict[str, Any]:
result: dict[str, Any] = {}
for key in (
"vpc_id", "region", "fabric_id", "fabric_revision", "cidr",
"evidence_version",
"desired_revision", "desired_digest", "observed_revision",
"observed_digest", "state", "next_stage_allowed",
"private_dataplane_verified", "operation_id", "updated_at"):
if key in value:
result[key] = value[key]
gate = value.get("fabric_gate")
if isinstance(gate, Mapping):
result["fabric_gate"] = {
key: gate[key] for key in (
"active", "available", "fabric_id", "fabric_revision",
"state", "next_stage_allowed", "error_code")
if key in gate
}
for collection, fields in (
("subnets", ("subnet_id", "cidr")),
("members", ("member_id", "instance_uuid", "host_id",
"private_ip_cidr", "subnet_id", "private_mac")),
("routes", ("route_id", "destination_cidr",
"next_hop_member_id", "next_hop_host_id", "metric")),
("acls", ("acl_id", "direction", "action", "protocol",
"source_cidr", "destination_cidr", "port_from",
"port_to", "priority")),
):
rows = value.get(collection)
if isinstance(rows, list):
result[collection] = [
{key: row[key] for key in fields if key in row}
for row in rows if isinstance(row, Mapping)
][:VPC_MAX_READBACK]
hosts = value.get("hosts")
if isinstance(hosts, list):
result["hosts"] = []
for raw in hosts[:VPC_MAX_HOSTS]:
if not isinstance(raw, Mapping):
continue
item = {key: raw[key] for key in (
"host_id", "member_ids", "vpc_id", "desired_revision",
"desired_digest", "bridge", "bridge_owner", "nft_table",
"observed_revision", "observed_digest", "state",
"transport_verified", "private_dataplane_verified",
"config_verified", "config_observed_revision",
"config_observed_digest",
"last_error_code", "removed", "retiring", "anti_spoof", "rp_filter", "acl_revision", "rollback",
"proxy_arp", "proxy_arp_pvlan", "proxy_ips", "member_nics", "routes", "wg_allowed_ips") if key in raw}
readback = raw.get("readback")
if isinstance(readback, Mapping):
item["readback"] = {
str(key): val for key, val in readback.items()
if isinstance(key, str) and len(key) <= 64
and _safe_scalar(val)
}
elif isinstance(readback, str) and len(readback) <= 128:
item["readback"] = readback
result["hosts"].append(item)
# PHP's typed VPC service consumes an explicit observed projection and
# keys host evidence by stable host ID. Keep this projection bounded
# and scalar-only; the human-facing ``hosts`` list above remains the
# convenient status representation.
observed_hosts: dict[str, dict[str, Any]] = {}
if isinstance(hosts, list):
for raw in hosts[:VPC_MAX_HOSTS]:
if not isinstance(raw, Mapping) or not isinstance(raw.get("host_id"), str):
continue
host_id = raw["host_id"]
item: dict[str, Any] = {}
for key in ("state", "observed_revision", "observed_digest",
"transport_verified", "private_dataplane_verified",
"config_verified", "config_observed_revision",
"config_observed_digest",
"last_error_code", "vpc_id", "host_id",
"desired_revision", "desired_digest", "bridge",
"bridge_owner", "nft_table", "anti_spoof",
"rp_filter", "proxy_arp", "proxy_arp_pvlan", "proxy_ips", "acl_revision", "rollback", "removed", "retiring", "member_nics",
"routes", "wg_allowed_ips", "readback"):
if key in raw:
value_item = raw[key]
if key in {"member_nics", "routes", "wg_allowed_ips", "proxy_ips"}:
if isinstance(value_item, (list, tuple)):
if key == "proxy_ips":
item[key] = [str(ip)[:64] for ip in value_item[:4096]
if isinstance(ip, str)]
else:
item[key] = [dict(row) for row in value_item[:4096]
if isinstance(row, Mapping)]
elif key == "readback":
if isinstance(value_item, str) and len(value_item) <= 128:
item[key] = value_item
elif isinstance(value_item, Mapping):
item[key] = {str(k): v for k, v in value_item.items()
if isinstance(k, str) and _safe_scalar(v)}
elif _safe_scalar(value_item):
item[key] = value_item
item["next_stage_allowed"] = (
raw.get("state") == "active"
and raw.get("transport_verified") is True
and raw.get("private_dataplane_verified") is True
and raw.get("observed_revision") == value.get("desired_revision")
and raw.get("observed_digest") == value.get("desired_digest")
)
observed_hosts[host_id] = item
result["observed"] = {
"evidence_version": value.get("evidence_version", 0),
"revision": value.get("observed_revision", 0),
"digest": value.get("observed_digest"),
"state": value.get("state"),
"next_stage_allowed": value.get("next_stage_allowed") is True,
"private_dataplane_verified": (
value.get("private_dataplane_verified") is True
),
"hosts": observed_hosts,
# PHP consumes this bounded observed projection when deciding
# whether the control-plane result is deliberately waiting for
# the later private data-plane phase. Keep the gate here as well
# as at the top level so callers cannot accidentally discard the
# reason while selecting ``result['observed']``.
"fabric_gate": dict(result.get("fabric_gate", {})),
}
return result
def list_status(self) -> list[dict[str, Any]]:
result: list[dict[str, Any]] = []
for _key, raw in self.store.list("vpc_desired").items():
try:
value = json.loads(raw)
except (TypeError, ValueError):
continue
if (isinstance(value, Mapping)
and value.get("vpc_id") == _key):
result.append(self._public(value))
return sorted(result, key=lambda item: item.get("vpc_id", ""))
def get_status(self, vpc_id: str) -> dict[str, Any]:
_identifier(vpc_id, "VPC")
value = self._load("vpc_desired", vpc_id)
if value is None:
raise VpcError("VPC 不存在", 404, "vpc_not_found")
return self._public(value)
def _save_children(self, record: Mapping[str, Any]) -> None:
vpc_id = record.get("vpc_id")
if not isinstance(vpc_id, str):
return
for kind, collection, identifier in (
("vpc_subnet", "subnets", "subnet_id"),
("vpc_member", "members", "member_id"),
("vpc_route", "routes", "route_id"),
("vpc_acl", "acls", "acl_id")):
rows = record.get(collection)
if not isinstance(rows, list):
continue
live_keys: set[str] = set()
for row in rows:
if not isinstance(row, Mapping) or not isinstance(row.get(identifier), str):
continue
child_id = row[identifier]
key = self._key(vpc_id, child_id)
live_keys.add(key)
child = dict(row)
child.update({"vpc_id": vpc_id,
"desired_revision": record.get("desired_revision"),
"desired_digest": record.get("desired_digest"),
"observed_revision": record.get("observed_revision", 0),
"observed_digest": record.get("observed_digest"),
"state": record.get("state")})
self._save(kind, key, child)
for key in self.store.list(kind):
if key.startswith(vpc_id + ".") and key not in live_keys:
# Keep a tombstone until a later observed revision proves
# that the data-plane removed the child. Deleting the
# row here would make a partial member/route/ACL removal
# impossible to reconcile after a crash.
raw_old = self.store.get(kind, key)
try:
old = json.loads(raw_old) if raw_old is not None else {}
except (TypeError, ValueError):
old = {"record_corrupt": True}
if not isinstance(old, dict):
old = {"record_corrupt": True}
old.update({
"vpc_id": vpc_id,
"state": "removed",
"removed": True,
"removed_at": int(time.time()),
"desired_revision": record.get("desired_revision"),
"desired_digest": record.get("desired_digest"),
})
self._save(kind, key, old)
hosts = record.get("hosts")
if isinstance(hosts, list):
live = set()
for row in hosts:
if isinstance(row, Mapping) and isinstance(row.get("host_id"), str):
key = self._key(vpc_id, row["host_id"])
live.add(key)
child = dict(row)
child.update({"vpc_id": vpc_id,
"desired_revision": record.get("desired_revision"),
"desired_digest": record.get("desired_digest")})
self._save("vpc_host", key, child)
for key in self.store.list("vpc_host"):
if key.startswith(vpc_id + ".") and key not in live:
raw_old = self.store.get("vpc_host", key)
try:
old = json.loads(raw_old) if raw_old is not None else {}
except (TypeError, ValueError):
old = {"record_corrupt": True}
if not isinstance(old, dict):
old = {"record_corrupt": True}
if (old.get("vpc_id") == vpc_id
and isinstance(old.get("host_id"), str)
and key == self._key(vpc_id, old["host_id"])
and self._terminal_removal(old)):
# Keep the exact revision/digest/readback envelope as
# a monotonic historical proof. A later live row for
# the same host overwrites this child normally.
old.setdefault("removed_at", int(time.time()))
self._save("vpc_host", key, old)
continue
old.update({
"vpc_id": vpc_id,
"state": "removed",
"removed": True,
"removed_at": int(time.time()),
"desired_revision": record.get("desired_revision"),
"desired_digest": record.get("desired_digest"),
})
self._save("vpc_host", key, old)
@classmethod
def _finish(cls, record: dict[str, Any], gate: Mapping[str, Any]) -> None:
record["evidence_version"] = VPC_EVIDENCE_VERSION
record["fabric_gate"] = dict(gate)
# ``recover`` marks a durable record as needing one fresh
# control-plane pass. Once the Fabric gate is available again, reset
# only the host control-plane state to applying; never infer active or
# data-plane readiness from stale pre-restart evidence.
if gate.get("active") is True and record.pop("recovery_pending", False) is True:
recovery_revision = record.get("desired_revision")
recovery_digest = record.get("desired_digest")
for item in record.get("hosts", []):
if isinstance(item, dict):
if (isinstance(recovery_revision, int)
and not isinstance(recovery_revision, bool)
and isinstance(recovery_digest, str)
and cls._confirmed_removal(
item, recovery_revision, recovery_digest)):
# An exact absence proof is terminal and monotonic. A
# restart must invalidate live readiness, but must not
# turn an already removed host back into a probe target.
continue
item["state"] = "applying"
item["transport_verified"] = True
item["private_dataplane_verified"] = False
item["config_verified"] = False
item["config_observed_revision"] = 0
item["config_observed_digest"] = None
hosts = record.get("hosts")
desired_revision = record.get("desired_revision")
desired_digest = record.get("desired_digest")
all_observed = bool(hosts) and all(
isinstance(item, Mapping)
and item.get("state") == "active"
and item.get("observed_revision") == desired_revision
and item.get("observed_digest") == desired_digest
and item.get("transport_verified") is True
and item.get("private_dataplane_verified") is True
for item in hosts)
retiring_hosts = [
item for item in hosts if isinstance(item, Mapping)
and item.get("retiring") is True
] if isinstance(hosts, list) else []
live_hosts = [
item for item in hosts if not (isinstance(item, Mapping)
and item.get("retiring") is True)
] if isinstance(hosts, list) else []
all_removed = bool(retiring_hosts) and all(
item.get("state") == "active"
and item.get("removed") is True
and item.get("observed_revision") == desired_revision
and item.get("observed_digest") == desired_digest
and item.get("transport_verified") is True
and item.get("private_dataplane_verified") is True
for item in retiring_hosts)
fabric_active = gate.get("active") is True
if all_removed and not live_hosts:
# A VPC with no remaining members is a terminal removal proof;
# the Fabric gate may correctly be inactive because there is no
# tenant transport left to activate.
record["state"] = "active"
record["next_stage_allowed"] = True
record["private_dataplane_verified"] = True
record["observed_revision"] = desired_revision
record["observed_digest"] = desired_digest
elif fabric_active and all_observed:
record["state"] = "active"
record["next_stage_allowed"] = True
record["private_dataplane_verified"] = True
record["observed_revision"] = desired_revision
record["observed_digest"] = desired_digest
elif not fabric_active:
record["state"] = "degraded" if gate.get("available") else "uncertain"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
for item in hosts if isinstance(hosts, list) else []:
if not isinstance(item, dict):
continue
if (isinstance(desired_revision, int)
and isinstance(desired_digest, str)
and cls._confirmed_removal(
item, desired_revision, desired_digest)):
continue
# Fabric reachability is a prerequisite for both shared host
# configuration and member runtime usability. Never let a
# prior active observation survive an inactive current gate.
item["transport_verified"] = False
item["private_dataplane_verified"] = False
item["config_verified"] = False
item["observed_revision"] = 0
item["observed_digest"] = None
item["config_observed_revision"] = 0
item["config_observed_digest"] = None
elif hosts and any(item.get("state") == "uncertain" for item in hosts if isinstance(item, Mapping)):
record["state"] = "uncertain"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
elif hosts and any(item.get("state") == "degraded" for item in hosts if isinstance(item, Mapping)):
record["state"] = "degraded"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
else:
record["state"] = "applying"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
record["updated_at"] = int(time.time())
async def _reconcile_pending_hosts(
self, plan: VpcPlan, record: dict[str, Any], gate: Mapping[str, Any],
previous_plan: VpcPlan | None,
private_link_states: Mapping[str, str] | None = None,
extra_identities: Sequence[tuple[str, str]] | None = None,
operation_id: str | None = None,
) -> None:
"""Retry only hosts without an exact durable runtime/removal proof."""
self._finish(record, gate)
durable_hosts = self._host_closure(record)
member_host_ids = {member.host_id for member in plan.members}
targeted_host_ids = {
member.host_id for member in plan.members
if (private_link_states is not None
and member.instance_uuid in private_link_states)
}
if extra_identities is not None:
targeted_host_ids.update(
host_id for host_id, _instance_uuid in extra_identities)
pending_host_ids = {
host_id for host_id, host in durable_hosts.items()
if ((host_id in member_host_ids
and (host_id in targeted_host_ids
or not self._confirmed_active_host(host, plan)))
or (host_id not in member_host_ids
and not self._confirmed_removal(
host, plan.desired_revision, plan.desired_digest)))
}
if not pending_host_ids:
def attempted_at(host: Mapping[str, Any]) -> int:
value = host.get("probe_attempted_at_ns", 0)
if (isinstance(value, bool)
or not isinstance(value, int) or value < 0):
return 0
return value
# Exact guest runtime evidence is not timeless: QGA, its route,
# or the guest address may disappear after the durable host row
# became active. Pending and lifecycle-targeted work remains the
# priority; only an otherwise idle periodic pass spends its
# bounded budget revalidating one oldest active member host.
oldest_active = min(
(
(attempted_at(host), host_id)
for host_id, host in durable_hosts.items()
if (host_id in member_host_ids
and self._confirmed_active_host(host, plan))
),
default=None,
)
if oldest_active is None:
return
pending_host_ids.add(oldest_active[1])
execution_record = dict(record)
execution_hosts = [
dict(host) for host in record.get("hosts", [])
if (isinstance(host, Mapping)
and host.get("host_id") in pending_host_ids)
]
def attempt_order(host: Mapping[str, Any]) -> tuple[int, int, str]:
attempted_at = host.get("probe_attempted_at_ns", 0)
if (isinstance(attempted_at, bool)
or not isinstance(attempted_at, int)
or attempted_at < 0):
attempted_at = 0
host_id = str(host.get("host_id", ""))
# A lifecycle request (suspend/resume/delete) must not sit behind
# unrelated periodic guest probes. Fair rotation applies within
# the targeted and ordinary groups independently.
return (0 if host_id in targeted_host_ids else 1,
attempted_at, host_id)
execution_hosts.sort(key=attempt_order)
execution_record["hosts"] = execution_hosts
reconcile_kwargs: dict[str, Any] = {}
if private_link_states is not None:
reconcile_kwargs["private_link_states"] = private_link_states
if extra_identities is not None:
reconcile_kwargs["extra_identities"] = extra_identities
if operation_id is not None:
reconcile_kwargs["operation_id"] = operation_id
await self._reconcile_dataplane(
plan, execution_record, gate, previous_plan, **reconcile_kwargs)
observations = {
host.get("host_id"): dict(host)
for host in execution_record.get("hosts", [])
if (isinstance(host, Mapping)
and isinstance(host.get("host_id"), str))
}
record["hosts"] = [
dict(observations[host["host_id"]])
if (isinstance(host, Mapping)
and host.get("host_id") in observations)
else host
for host in record.get("hosts", [])
]
self._finish(record, gate)
async def delete(self, vpc_id: str, value: Mapping[str, Any]) -> dict[str, Any]:
"""Remove every host resource and persist an idempotent root receipt.
The receipt is written before the desired/child records are purged.
Consequently a lost HTTP response can be retried with the exact same
operation/revision/digest without turning store absence into guessed
success. A signed no-host target is the sole exception: when neither
a desired record nor a receipt exists, PHP's independently locked
deployment ledger supplies the other half of that absence proof.
"""
vpc_id = _identifier(vpc_id, "VPC")
plan, retiring_members = parse_vpc_delete_plan(value)
if plan.vpc_id != vpc_id:
raise VpcError("VPC 删除路径与目标不一致", 409,
"vpc_delete_target_mismatch")
self._delete_plan(plan)
async def delete_locked() -> dict[str, Any]:
receipt = self._load_delete_receipt(vpc_id)
if receipt is not None:
if (receipt["operation_id"] != plan.operation_id
or receipt["desired_revision"] != plan.desired_revision
or receipt["desired_digest"] != plan.desired_digest):
raise VpcError("VPC 已由另一删除操作完成", 409,
"vpc_delete_receipt_conflict")
self._purge_deleted_vpc(vpc_id)
return self._public_delete_receipt(receipt)
previous = self._load("vpc_desired", vpc_id)
if previous is None and not plan.host_ids:
receipt = {
"schema_version": VPC_SCHEMA_VERSION,
"evidence_version": VPC_EVIDENCE_VERSION,
"vpc_id": vpc_id,
"operation_id": plan.operation_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"state": "deleted",
"deleted": True,
"readback": "confirmed",
"removed_host_ids": [],
"completed_at": int(time.time()),
}
self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, receipt)
self._purge_deleted_vpc(vpc_id)
return self._public_delete_receipt(receipt)
# A host-bearing delete without a prior desired record creates a
# temporary durable VPC catalog reservation before contacting any
# host. It must obey the same global Fabric endpoint exclusion as
# an ordinary apply; otherwise it could race an endpoint migration
# and publish a conflicting tenant CIDR under the delete marker.
if previous is None:
self._validate_global_fabric_endpoints(plan)
marker = (self._delete_marker(previous)
if previous is not None else None)
transition_previous: VpcPlan | None = None
if marker is None:
if previous is None:
previous_plan = None
old_hosts: dict[str, Mapping[str, Any]] = {}
previous_observed_revision = 0
previous_observed_digest = None
else:
previous_plan = _plan_from_record(previous)
old_revision = previous.get("desired_revision")
old_digest = previous.get("desired_digest")
if (old_revision != previous_plan.desired_revision
or old_digest != previous_plan.desired_digest):
raise VpcError("VPC 持久化目标摘要不一致", 500,
"vpc_record_corrupt")
if (
plan.region,
plan.fabric_id,
plan.fabric_revision,
plan.cidr,
) != (
previous_plan.region,
previous_plan.fabric_id,
previous_plan.fabric_revision,
previous_plan.cidr,
):
raise VpcError(
"VPC 删除目标不能改变持久化目录身份",
409, "vpc_delete_identity_mismatch")
if plan.desired_revision <= previous_plan.desired_revision:
raise VpcError("VPC 删除版本必须高于当前目标", 409,
"vpc_revision_conflict")
if plan.operation_id == previous_plan.operation_id:
raise VpcError("VPC 删除必须使用新的操作编号", 409,
"vpc_delete_receipt_conflict")
old_hosts = self._host_closure(previous)
previous_observed_revision = previous.get(
"observed_revision", 0)
previous_observed_digest = previous.get("observed_digest")
old_host_ids = set(old_hosts)
requested_host_ids = set(plan.host_ids)
historical_preverified: set[str] = set()
for host_id in requested_host_ids - old_host_ids:
raw_child = self.store.get(
"vpc_host", self._key(vpc_id, host_id))
try:
child = (json.loads(raw_child)
if raw_child is not None else None)
except (TypeError, ValueError):
child = None
proof_revision = (child.get("desired_revision")
if isinstance(child, Mapping) else None)
if (isinstance(child, Mapping)
and child.get("vpc_id") == vpc_id
and child.get("host_id") == host_id
and isinstance(proof_revision, int)
and not isinstance(proof_revision, bool)
and proof_revision < plan.desired_revision
and self._terminal_removal(child)):
historical_preverified.add(host_id)
member_hosts = {
member.host_id for member in (previous_plan.members
if previous_plan else ())
}
manifest_hosts = {
member.host_id for member in retiring_members
}
preverified = set()
if previous_plan is not None:
preverified = {
host_id for host_id, host in old_hosts.items()
if host_id not in member_hosts
and host_id not in manifest_hosts
and self._confirmed_removal(
host, previous_plan.desired_revision,
previous_plan.desired_digest)
}
# A host pruned from a later normal target is absent from the
# current root closure. Its exact child proof remains valid
# unless a new live row overwrote it, and is strong enough to
# cover a historical member in PHP's final delete manifest.
preverified.update(historical_preverified)
missing = old_host_ids - requested_host_ids
if not missing.issubset(preverified):
raise VpcError("VPC 删除遗漏尚未清理的宿主机", 409,
"vpc_delete_host_set_mismatch")
required_host_ids = requested_host_ids - preverified
retiring_by_id = {
member.member_id: member for member in retiring_members
}
for old_member in (previous_plan.members
if previous_plan else ()):
signed = retiring_by_id.get(old_member.member_id)
if (signed is None
or signed.instance_uuid != old_member.instance_uuid
or signed.host_id != old_member.host_id
or signed.private_ip != old_member.private_ip
or (old_member.private_mac is not None
and signed.private_mac
!= old_member.private_mac)):
raise VpcError(
"VPC 删除成员清单未覆盖旧持久化成员", 409,
"vpc_delete_manifest_mismatch")
# Already-proven, omitted hosts must not be contacted again.
# Excluding them from the transition plan prevents the generic
# desired-record builder from recreating a retiring host row.
if previous_plan is not None:
transition_previous = replace(
previous_plan,
hosts=tuple(host for host in previous_plan.hosts
if host.host_id not in missing),
)
previous_hosts = {
host_id: host for host_id, host in old_hosts.items()
if host_id in requested_host_ids
}
gate = self._fabric_gate(plan)
record = self._desired_record(
plan, gate, state="deleting",
observed_revision=previous_observed_revision,
observed_digest=previous_observed_digest,
previous_hosts=previous_hosts,
previous_plan=transition_previous,
)
retiring_rows_by_host: dict[str, list[dict[str, str]]] = {}
for row in _canonical_delete_members(retiring_members):
retiring_rows_by_host.setdefault(
row["host_id"], []).append(row)
for host in record.get("hosts", []):
if (isinstance(host, dict)
and isinstance(host.get("host_id"), str)):
host["retiring_members"] = list(
retiring_rows_by_host.get(host["host_id"], []))
marker = {
"operation_id": plan.operation_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"all_host_ids": sorted(old_host_ids | requested_host_ids),
"required_host_ids": sorted(required_host_ids),
"preverified_host_ids": sorted(preverified),
"retiring_members": _canonical_delete_members(
retiring_members),
}
record["delete_target"] = marker
record["state"] = "deleting"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
record["updated_at"] = int(time.time())
# Persist the target before any SSH/libvirt mutation. A crash
# can then resume the exact same operation and host closure.
self._save("vpc_desired", vpc_id, record)
self._save_children(record)
else:
if (marker["operation_id"] != plan.operation_id
or marker["desired_revision"] != plan.desired_revision
or marker["desired_digest"] != plan.desired_digest):
raise VpcError("VPC 删除目标已由另一操作占用", 409,
"vpc_delete_receipt_conflict")
durable_plan = _plan_from_record(previous)
if (durable_plan.operation_id != plan.operation_id
or durable_plan.desired_revision != plan.desired_revision
or durable_plan.desired_digest != plan.desired_digest
or durable_plan.canonical() != plan.canonical()):
raise VpcError("VPC 删除持久化目标不一致", 500,
"vpc_record_corrupt")
if (not set(marker["required_host_ids"]).issubset(plan.host_ids)
or not plan.host_ids.issubset(
set(marker["all_host_ids"]))):
raise VpcError("VPC 删除持久化宿主机闭集不一致", 500,
"vpc_record_corrupt")
if marker["retiring_members"] != _canonical_delete_members(
retiring_members):
raise VpcError("VPC 删除成员清单与持久化目标不一致", 500,
"vpc_record_corrupt")
record = dict(previous)
transition_previous = durable_plan
required_host_ids = set(marker["required_host_ids"])
observed_hosts = self._host_closure(record)
already_confirmed = {
host_id for host_id in required_host_ids
if (host_id in observed_hosts
and self._confirmed_removal(
observed_hosts[host_id], plan.desired_revision,
plan.desired_digest))
}
pending_host_ids = required_host_ids - already_confirmed
if pending_host_ids:
gate = self._fabric_gate(plan)
# The submitted plan may still contain a deployment whose
# prior remove response was lost by PHP. Its exact durable
# absence proof classifies it as preverified; never contact it
# again. Reconcile a transient projection containing only
# genuinely pending host rows, then merge those observations
# back into the durable submitted plan.
execution_record = dict(record)
execution_hosts: list[dict[str, Any]] = []
retiring_rows_by_host: dict[str, list[dict[str, str]]] = {}
for row in marker["retiring_members"]:
retiring_rows_by_host.setdefault(
row["host_id"], []).append(dict(row))
for host in record.get("hosts", []):
if (not isinstance(host, Mapping)
or host.get("host_id") not in pending_host_ids):
continue
pending = dict(host)
# Persisted delete targets retain the submitted canonical
# host list, while the transient retiring marker forces
# the removal lane even after recover() cleared verified
# state and retiring_members had already been consumed.
pending["retiring"] = True
pending["retiring_members"] = list(
retiring_rows_by_host.get(pending["host_id"], []))
execution_hosts.append(pending)
execution_record["hosts"] = execution_hosts
await self._reconcile_dataplane(
plan, execution_record, gate, transition_previous,
operation_id=plan.operation_id)
observations = {
host.get("host_id"): {
key: value for key, value in host.items()
if key != "retiring"
}
for host in execution_record.get("hosts", [])
if (isinstance(host, Mapping)
and isinstance(host.get("host_id"), str))
}
record["hosts"] = [
dict(observations[host["host_id"]])
if (isinstance(host, Mapping)
and host.get("host_id") in observations)
else host
for host in record.get("hosts", [])
]
# Preserve partial/uncertain evidence before deciding whether
# a receipt may be issued.
self._save("vpc_desired", vpc_id, record)
self._save_children(record)
observed_hosts = self._host_closure(record)
if not required_host_ids.issubset(observed_hosts) or not all(
self._confirmed_removal(
observed_hosts[host_id], plan.desired_revision,
plan.desired_digest)
for host_id in required_host_ids
):
record["state"] = "uncertain"
record["next_stage_allowed"] = False
record["private_dataplane_verified"] = False
record["updated_at"] = int(time.time())
self._save("vpc_desired", vpc_id, record)
deterministic = [
(host_id, observed_hosts[host_id].get("last_error_code"))
for host_id in sorted(required_host_ids)
if (host_id in observed_hosts
and observed_hosts[host_id].get("last_error_code")
in VPC_DELETE_DETERMINISTIC_HOST_ERRORS)
]
if deterministic:
raise VpcError(
"VPC 宿主机存在需要用户处理的确定性清理阻断",
409, str(deterministic[0][1]))
raise VpcError("VPC 宿主机清理尚未全部确认", 503,
"vpc_delete_incomplete")
receipt = {
"schema_version": VPC_SCHEMA_VERSION,
"evidence_version": VPC_EVIDENCE_VERSION,
"vpc_id": vpc_id,
"operation_id": plan.operation_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"state": "deleted",
"deleted": True,
"readback": "confirmed",
"removed_host_ids": list(marker["all_host_ids"]),
"completed_at": int(time.time()),
}
# This ordering is the idempotency boundary. Never erase the
# evidence needed by a retry before its exact receipt is durable.
self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, receipt)
self._purge_deleted_vpc(vpc_id)
return self._public_delete_receipt(receipt)
# Root delete mutates the same CIDR/endpoint catalog as apply. Keep
# the global order per-VPC -> catalog, and do not release either lock
# while a cancelled coroutine still has a blocking host mutation or
# durable receipt commit in flight.
async with self._lock(vpc_id), self._catalog_mutation_lock():
return await _await_mutation_completion(delete_locked())
async def apply(self, value: Mapping[str, Any]) -> dict[str, Any]:
plan = parse_vpc_plan(value)
async with self._lock(plan.vpc_id), self._catalog_mutation_lock():
return await _await_mutation_completion(
self._apply_locked(plan))
async def _apply_locked(self, plan: VpcPlan) -> dict[str, Any]:
if self._load_delete_receipt(plan.vpc_id) is not None:
raise VpcError("VPC 已删除,不能重新应用旧标识", 409,
"vpc_deleted")
self._validate_global_fabric_endpoints(plan)
previous = self._load("vpc_desired", plan.vpc_id)
previous_plan: VpcPlan | None = None
old_revision = plan.desired_revision
if previous is not None:
if self._delete_marker(previous) is not None:
raise VpcError("VPC 正在删除,不能应用普通目标", 409,
"vpc_deleting")
# Validate the durable prior target before accepting a newer
# revision. Otherwise a malformed record could be silently
# overwritten by a tenant retry, erasing the evidence needed
# to reconcile an interrupted operation.
previous_plan = _plan_from_record(previous)
old_revision = previous.get("desired_revision")
old_digest = previous.get("desired_digest")
if (old_revision != previous_plan.desired_revision
or old_digest != previous_plan.desired_digest):
raise VpcError("VPC 持久化目标摘要不一致", 500,
"vpc_record_corrupt")
if isinstance(old_revision, int) and plan.desired_revision < old_revision:
raise VpcError("VPC desired_revision 不能回退", 409,
"vpc_revision_conflict")
if plan.desired_revision == old_revision:
if old_digest != plan.desired_digest:
raise VpcError("相同 desired_revision 的 VPC 计划已存在", 409,
"vpc_revision_conflict")
# Preserve exact host absence proofs monotonically. A
# partial pass may have removed host A while host B was
# unavailable; replay must retry only B instead of making
# the already removed host reachable again as a condition
# of convergence.
gate = self._fabric_gate(plan)
self._require_fabric_catalog_admission(gate)
record = dict(previous)
await self._reconcile_pending_hosts(
plan, record, gate, previous_plan)
self._save("vpc_desired", plan.vpc_id, record)
self._save_children(record)
return self._public(record)
gate = self._fabric_gate(plan)
self._require_fabric_catalog_admission(gate)
state = "applying" if gate.get("active") else (
"degraded" if gate.get("available") else "uncertain")
previous_hosts = {
item.get("host_id"): item
for item in (previous or {}).get("hosts", [])
if isinstance(item, Mapping) and isinstance(item.get("host_id"), str)
}
previous_observed_revision = (previous or {}).get("observed_revision", 0)
previous_observed_digest = (previous or {}).get("observed_digest")
if (isinstance(previous_observed_revision, bool)
or not isinstance(previous_observed_revision, int)
or previous_observed_revision < 0
or previous_observed_revision > old_revision
or previous_observed_revision > plan.desired_revision):
raise VpcError("VPC 持久化观测版本无效", 500,
"vpc_record_corrupt")
if previous_observed_digest is not None:
try:
previous_observed_digest = _digest(
previous_observed_digest, "observed_digest")
except VpcError as exc:
raise VpcError("VPC 持久化观测摘要无效", 500,
"vpc_record_corrupt") from exc
for old_host in previous_hosts.values():
old_host_revision = old_host.get("observed_revision", 0)
if (isinstance(old_host_revision, bool)
or not isinstance(old_host_revision, int)
or old_host_revision < 0
or old_host_revision > old_revision):
raise VpcError("VPC 宿主观测版本无效", 500,
"vpc_record_corrupt")
old_host_digest = old_host.get("observed_digest")
if old_host_digest is not None:
try:
_digest(old_host_digest, "observed_digest")
except VpcError as exc:
raise VpcError("VPC 宿主观测摘要无效", 500,
"vpc_record_corrupt") from exc
record = self._desired_record(
plan, gate, state=state,
observed_revision=previous_observed_revision,
observed_digest=previous_observed_digest,
previous_hosts=previous_hosts,
previous_plan=previous_plan,
)
self._finish(record, gate)
await self._reconcile_dataplane(
plan, record, gate, previous_plan)
self._save("vpc_desired", plan.vpc_id, record)
self._save_children(record)
return self._public(record)
async def observe(
self, vpc_id: str, observations: Mapping[str, Mapping[str, Any]],
) -> dict[str, Any]:
"""Record typed future data-plane readback for one VPC.
This method is intentionally not wired to the public HTTP body. A
later private reconciler may call it after reading host state. It
accepts only revision/digest/state/readback scalars and never shell or
raw nft/libvirt text.
"""
_identifier(vpc_id, "VPC")
if not isinstance(observations, Mapping):
raise VpcError("VPC observed 状态必须是对象")
async def observe_locked() -> dict[str, Any]:
record = self._load("vpc_desired", vpc_id)
if record is None:
raise VpcError("VPC 不存在", 404, "vpc_not_found")
plan = _plan_from_record(record)
expected_hosts = {item.host_id for item in plan.hosts}
if set(observations) - expected_hosts:
raise VpcError("VPC observed 包含未知宿主机", 422,
"vpc_observed_host_invalid")
for host in record.get("hosts", []):
if not isinstance(host, dict):
continue
if (host.get("host_id") not in {
member.host_id for member in plan.members}
and self._confirmed_removal(
host, plan.desired_revision,
plan.desired_digest)):
# A missing or stale external observation cannot revoke a
# durable exact absence proof for the same immutable plan.
continue
raw = observations.get(host.get("host_id"))
if raw is None:
host["state"] = "uncertain"
host["transport_verified"] = False
host["private_dataplane_verified"] = False
continue
if not isinstance(raw, Mapping):
raise VpcError("VPC observed 宿主记录无效")
observed_revision = raw.get("observed_revision")
observed_digest = raw.get("observed_digest")
state = raw.get("state")
transport = raw.get("transport_verified")
private_dataplane = raw.get("private_dataplane_verified", False)
if (isinstance(observed_revision, bool)
or not isinstance(observed_revision, int)
or observed_revision < 0):
raise VpcError("VPC observed_revision 无效")
if observed_revision > plan.desired_revision:
raise VpcError("VPC observed_revision 不能超过 desired_revision",
409, "vpc_revision_conflict")
if observed_digest is not None:
observed_digest = _digest(observed_digest, "observed_digest")
if state not in VPC_STATES:
raise VpcError("VPC observed state 无效")
if not isinstance(transport, bool):
raise VpcError("VPC observed transport_verified 无效")
if not isinstance(private_dataplane, bool):
raise VpcError("VPC observed private_dataplane_verified 无效")
if state == "active" and (transport is not True
or observed_digest is None):
raise VpcError(
"VPC active 观测缺少传输或摘要证据", 422,
"vpc_observation_invalid")
# The current Fabric adapter reports transport/readback before
# the tenant private data-plane adapter exists. Preserve that
# observation as an applying gate instead of rejecting the
# whole reconciliation request: transport is useful progress,
# but it must never open the VPC or be reported as active.
observation_error = raw.get("error_code")
if state == "active" and private_dataplane is not True:
state = "applying"
observation_error = "vpc_private_dataplane_unverified"
readback = raw.get("readback", {})
if isinstance(readback, str) and len(readback) <= 128:
safe_readback: Any = readback
elif isinstance(readback, Mapping) and len(readback) <= 64:
safe_readback = {
str(key): value for key, value in readback.items()
if isinstance(key, str) and len(key) <= 64
and _safe_scalar(value)
}
else:
raise VpcError("VPC readback 无效")
typed_fields = (
"vpc_id", "host_id", "desired_revision", "desired_digest",
"config_verified", "config_observed_revision",
"config_observed_digest",
"bridge", "bridge_owner", "nft_table", "anti_spoof",
"rp_filter", "proxy_arp", "proxy_arp_pvlan", "proxy_ips", "acl_revision", "rollback", "removed", "member_nics", "routes",
"wg_allowed_ips",
)
typed = {}
for field in typed_fields:
if field not in raw:
continue
item_value = raw[field]
if field in {"member_nics", "routes", "wg_allowed_ips", "proxy_ips"}:
if isinstance(item_value, (list, tuple)) and len(item_value) <= VPC_MAX_READBACK:
if field == "proxy_ips":
typed[field] = [str(ip)[:64] for ip in item_value
if isinstance(ip, str)]
else:
typed[field] = [dict(row) for row in item_value
if isinstance(row, Mapping)]
elif field == "readback":
continue
elif _safe_scalar(item_value):
typed[field] = item_value
host.update({"state": state,
"observed_revision": observed_revision,
"observed_digest": observed_digest,
"transport_verified": transport,
"private_dataplane_verified": private_dataplane,
"readback": safe_readback,
"last_error_code": observation_error
if isinstance(observation_error, str)
and len(observation_error) <= 64 else None})
host.update(typed)
gate = self._fabric_gate(plan)
self._finish(record, gate)
self._save("vpc_desired", vpc_id, record)
self._save_children(record)
return self._public(record)
# External observations may publish an active root from existing host
# evidence. Treat that publication as a catalog mutation too: pin the
# Fabric generation and every affected VM identity until the refreshed
# gate, root, and child projections are committed together.
async with self._lock(vpc_id), self._catalog_mutation_lock():
lock_record = self._load("vpc_desired", vpc_id)
if lock_record is None:
raise VpcError("VPC 不存在", 404, "vpc_not_found")
lock_plan = _plan_from_record(lock_record)
extra_identities = [
(host["host_id"], row["instance_uuid"])
for host in lock_record.get("hosts", [])
if (isinstance(host, Mapping)
and isinstance(host.get("host_id"), str))
for row in host.get("retiring_members", [])
if (isinstance(row, Mapping)
and isinstance(row.get("instance_uuid"), str))
]
async with self._instance_locks(
lock_plan, extra_identities=extra_identities):
return await _await_mutation_completion(observe_locked())
@staticmethod
def _instance_private_state(
instance_uuid: str, *, required: bool, vpc_id: str | None,
member_id: str | None, desired_state: str, private_ip_cidr: str | None,
private_mac: str | None, desired_revision: int, desired_digest: str | None,
host: Mapping[str, Any] | None, member_nic: Mapping[str, Any] | None,
reason_code: str | None = None,
) -> dict[str, Any]:
"""Project bounded host/VPC evidence for one lifecycle gate."""
observed_revision = (
host.get("observed_revision", 0) if isinstance(host, Mapping) else 0)
observed_digest = (
host.get("observed_digest") if isinstance(host, Mapping) else None)
config_observed_revision = (
host.get("config_observed_revision", 0)
if isinstance(host, Mapping) else 0)
config_observed_digest = (
host.get("config_observed_digest")
if isinstance(host, Mapping) else None)
host_runtime_ok = bool(
isinstance(host, Mapping)
and host.get("state") == "active"
and host.get("private_dataplane_verified") is True
and observed_revision == desired_revision
and observed_digest == desired_digest
)
host_config_ok = bool(
isinstance(host, Mapping)
and host.get("transport_verified") is True
and host.get("config_verified") is True
and config_observed_revision == desired_revision
and config_observed_digest == desired_digest
)
# Host configuration is shared, while guest readiness is per member.
# A stopped or unobservable sibling must not erase a healthy member's
# independently confirmed runtime state.
host_ok = host_runtime_ok or host_config_ok
desired_mac: str | None = None
if (required and desired_state != "removed"
and isinstance(vpc_id, str) and isinstance(member_id, str)):
try:
desired_mac = _effective_private_mac(
vpc_id, member_id, private_mac)
except (TypeError, ValueError, VpcError):
# Keep the projection bounded and fail closed if a corrupt
# durable row contains an invalid MAC. The caller will
# expose an uncertain observation instead of publishing a
# guessed address.
desired_mac = None
observed_mac: str | None = None
guest_mac: str | None = None
guest_ip: str | None = None
if isinstance(member_nic, Mapping):
try:
observed_mac = _mac(member_nic.get("private_mac"))
except VpcError:
observed_mac = None
try:
guest_mac = _mac(member_nic.get("guest_private_mac"))
except VpcError:
guest_mac = None
if isinstance(member_nic.get("guest_private_ip_cidr"), str):
guest_ip = member_nic.get("guest_private_ip_cidr")
active_nic_ok = bool(
isinstance(member_nic, Mapping)
and member_nic.get("xml_verified") is True
and member_nic.get("seed_verified") is True
and member_nic.get("restart_required") is False
and member_nic.get("domain_running") is True
and member_nic.get("guest_network_verified") is True
and member_nic.get("guest_link_state") == "up"
and desired_mac is not None
and observed_mac == desired_mac
and guest_mac == desired_mac
and guest_ip == private_ip_cidr
)
link_state = (
member_nic.get("link_state") if isinstance(member_nic, Mapping)
else "absent")
if link_state not in {"up", "down"}:
link_state = "absent"
suspended_nic_ok = bool(
isinstance(member_nic, Mapping)
and member_nic.get("xml_verified") is True
and member_nic.get("config_verified") is True
and member_nic.get("seed_verified") is True
and member_nic.get("restart_required") is False
and member_nic.get("domain_running") is True
and link_state == "down"
and desired_mac is not None
and observed_mac == desired_mac
)
stopped_nic_ok = bool(
isinstance(member_nic, Mapping)
and member_nic.get("xml_verified") is True
and member_nic.get("config_verified") is True
and member_nic.get("seed_verified") is True
and member_nic.get("restart_required") is False
and member_nic.get("domain_running") is False
and member_nic.get("private_ip_cidr") == private_ip_cidr
and desired_mac is not None
and observed_mac == desired_mac
)
if not required:
state = "disabled" if not isinstance(member_nic, Mapping) else "uncertain"
return {
"version": 1, "required": False, "vm_uuid": instance_uuid,
"vpc_id": None, "member_id": None, "interface": None,
"state": state, "desired_state": "disabled",
"desired_revision": 0, "observed_revision": 0,
"desired_digest": None, "observed_digest": None,
"readback": "confirmed" if state == "disabled" else "mismatch",
"link_state": "absent", "private_ip_cidr": None,
"private_mac": None,
"reason_code": None if state == "disabled" else reason_code
or "vpc_private_observation_mismatch",
"observed_at": int(time.time()),
"xml_verified": False, "config_verified": False,
"seed_verified": False, "domain_running": False,
"restart_required": False,
"guest_network_verified": False,
"guest_network_readback": "missing",
"guest_link_state": None,
}
removed = desired_state == "removed"
if removed:
shape_ok = not isinstance(member_nic, Mapping)
state = "removed" if host_ok and shape_ok else "uncertain"
interface = None
link = "absent"
ip = None
mac = None
readback = "confirmed" if state == "removed" else "missing"
else:
# An intentional live link-down is an isolation state, not a
# usable-network claim. Exact persistent/live XML plus the host
# config proof is sufficient to confirm suspension; requiring a
# guest link-up probe here would make network_suspend impossible.
shape_ok = (stopped_nic_ok if stopped_nic_ok
else suspended_nic_ok if link_state == "down"
else active_nic_ok)
state = (
"stopped" if stopped_nic_ok
else "suspended" if link_state == "down" else "active"
) if host_ok and shape_ok else "uncertain"
interface = "private0" if shape_ok else None
link = link_state
ip = (private_ip_cidr
if stopped_nic_ok or suspended_nic_ok else guest_ip) \
if shape_ok else None
# Return the value actually present in the bounded host
# observation. Equality with the derived/configured target above
# is part of ``shape_ok``; this prevents a non-empty but stale
# desired MAC from masking host XML drift.
mac = (observed_mac
if stopped_nic_ok or suspended_nic_ok else guest_mac) \
if shape_ok else None
readback = "confirmed" if host_ok and shape_ok else "missing"
return {
"version": 1, "required": True, "vm_uuid": instance_uuid,
"vpc_id": vpc_id, "member_id": member_id,
"interface": interface, "state": state,
"desired_state": desired_state,
"desired_revision": desired_revision,
"observed_revision": (
observed_revision if host_runtime_ok
else config_observed_revision if host_config_ok else 0),
"desired_digest": desired_digest,
"observed_digest": (
observed_digest if host_runtime_ok
else config_observed_digest if host_config_ok else None),
"readback": readback, "link_state": link,
"private_ip_cidr": ip, "private_mac": mac,
"reason_code": (
"vpc_private_guest_stopped"
if state == "stopped" else
None if readback == "confirmed" else reason_code
or ("vpc_private_teardown_uncertain"
if removed else "vpc_private_observation_mismatch")),
"observed_at": int(time.time()),
"xml_verified": (
member_nic.get("xml_verified") is True
if isinstance(member_nic, Mapping) else False),
"config_verified": (
member_nic.get("config_verified") is True
if isinstance(member_nic, Mapping) else False),
"seed_verified": (
member_nic.get("seed_verified") is True
if isinstance(member_nic, Mapping) else False),
"domain_running": (
member_nic.get("domain_running") is True
if isinstance(member_nic, Mapping) else False),
"restart_required": (
member_nic.get("restart_required") is True
if isinstance(member_nic, Mapping) else False),
"guest_network_verified": (
member_nic.get("guest_network_verified") is True
if isinstance(member_nic, Mapping) else False),
"guest_network_readback": (
str(member_nic.get("guest_network_readback"))[:32]
if isinstance(member_nic, Mapping)
and isinstance(member_nic.get("guest_network_readback"), str)
else "missing"),
"guest_link_state": (
member_nic.get("guest_link_state")
if isinstance(member_nic, Mapping)
and member_nic.get("guest_link_state") in {"up", "down"}
else None),
}
async def reconcile_instance(
self, instance_uuid: str, *, host_id: str | None = None,
link_state: str = "preserve", expected: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Converge and prove one VM's private dataplane.
Lifecycle workers call this idempotent lane after a VM mutation. It
reuses the same VPC desired record and host adapter as the periodic
reconciler, so the returned evidence covers the private bridge, exact
routes, WireGuard AllowedIPs, nft ACL/anti-spoof policy and the
persistent/live libvirt NIC XML together. XML alone is never marked
confirmed.
"""
instance_uuid = _uuid(instance_uuid)
if link_state not in {"preserve", "up", "down", "absent"}:
raise VpcError("私网生命周期链路状态无效", 422,
"vpc_private_link_state_invalid")
expected = expected if isinstance(expected, Mapping) else {}
expected_required = expected.get("required")
expected_vpc = expected.get("vpc_id")
expected_member = expected.get("member_id")
expected_state = expected.get("desired_state")
lifecycle_kind = expected.get("lifecycle_kind")
expected_revision = expected.get("desired_revision")
expected_digest = expected.get("desired_digest")
expected_operation_id = expected.get("operation_id")
if lifecycle_kind is None:
lifecycle_kind = "legacy"
elif (not isinstance(lifecycle_kind, str)
or _PRIVATE_LIFECYCLE_KIND.fullmatch(lifecycle_kind) is None):
raise VpcError("私网生命周期类型无效", 422,
"vpc_private_lifecycle_kind_invalid")
lifecycle_link_invalid = (
(lifecycle_kind in {"delete", "expiry_delete"}
and link_state != "absent")
or (lifecycle_kind == "network_suspend"
and link_state not in {"down", "preserve"})
or (lifecycle_kind == "network_resume"
and link_state not in {"up", "preserve"})
or (lifecycle_kind not in {
"delete", "expiry_delete", "network_suspend",
"network_resume"}
and link_state != "preserve")
)
if lifecycle_kind != "legacy" and lifecycle_link_invalid:
raise VpcError("私网生命周期策略与链路目标不一致", 422,
"vpc_private_lifecycle_policy_invalid")
if expected_vpc is not None and not isinstance(expected_vpc, str):
raise VpcError("私网 VPC 标识无效", 422)
if expected_member is not None and not isinstance(expected_member, str):
raise VpcError("私网成员标识无效", 422)
if expected_state is not None and expected_state not in {"active", "removed"}:
raise VpcError("私网目标状态无效", 422)
if expected_revision is not None and (
isinstance(expected_revision, bool)
or not isinstance(expected_revision, int)
or expected_revision < 0):
raise VpcError("私网目标 revision 无效", 422)
if expected_digest is not None and (
not isinstance(expected_digest, str)
or _DIGEST.fullmatch(expected_digest) is None):
raise VpcError("私网目标摘要无效", 422)
if expected_operation_id is not None and (
not isinstance(expected_operation_id, str)
or _OPERATION_ID.fullmatch(expected_operation_id) is None):
raise VpcError("私网操作编号无效", 422)
# Validate the immutable MAC expectation at the engine boundary too;
# PHP validation is not a substitute for a host-side authorization
# check. A missing value is allowed for old ledgers and is filled by
# the same deterministic derivation used by the dataplane parser.
expected_private_mac = _mac(expected.get("private_mac"))
candidates: list[tuple[str, dict[str, Any], VpcPlan, VpcMember, bool]] = []
retiring: list[tuple[str, dict[str, Any], VpcPlan, Mapping[str, Any]]] = []
corrupt = False
for key, raw in self.store.list("vpc_desired").items():
try:
record = json.loads(raw)
plan = _plan_from_record(record)
except (TypeError, ValueError, VpcError):
corrupt = True
continue
for member in plan.members:
if member.instance_uuid == instance_uuid:
candidates.append((key, record, plan, member, False))
for host in record.get("hosts", []):
if not isinstance(host, Mapping):
continue
rows = host.get("retiring_members", [])
if not isinstance(rows, list):
continue
for row in rows:
if (isinstance(row, Mapping)
and row.get("instance_uuid") == instance_uuid):
retiring.append((key, record, plan, row))
if len(candidates) + len(retiring) > 1:
return self._instance_private_state(
instance_uuid, required=True, vpc_id=expected_vpc,
member_id=expected_member, desired_state=expected_state or "active",
private_ip_cidr=expected.get("private_ip_cidr"),
private_mac=expected_private_mac,
desired_revision=int(expected_revision or 0),
desired_digest=expected_digest, host=None, member_nic=None,
reason_code="vpc_private_expectation_ambiguous")
if not candidates and not retiring:
# A malformed VPC ledger must not be hidden as a public-only VM
# when the worker supplied an attached expectation.
if expected_required is True or corrupt:
return self._instance_private_state(
instance_uuid, required=True, vpc_id=expected_vpc,
member_id=expected_member, desired_state=expected_state or "active",
private_ip_cidr=expected.get("private_ip_cidr"),
private_mac=expected_private_mac,
desired_revision=int(expected_revision or 0),
desired_digest=expected_digest, host=None, member_nic=None,
reason_code=("vpc_private_record_corrupt" if corrupt
else "vpc_private_expectation_unavailable"))
return self._instance_private_state(
instance_uuid, required=False, vpc_id=None, member_id=None,
desired_state="disabled", private_ip_cidr=None, private_mac=None,
desired_revision=0, desired_digest=None, host=None, member_nic=None)
if candidates:
key, _old_record, _old_plan, _member, _ = candidates[0]
desired_state = "active"
else:
key, _old_record, _old_plan, _row = retiring[0]
desired_state = "removed"
async def reconcile_instance_locked() -> dict[str, Any]:
record = self._load("vpc_desired", key)
if record is None:
return self._instance_private_state(
instance_uuid, required=True, vpc_id=expected_vpc,
member_id=expected_member, desired_state=desired_state,
private_ip_cidr=expected.get("private_ip_cidr"),
private_mac=expected_private_mac,
desired_revision=int(expected_revision or 0),
desired_digest=expected_digest, host=None, member_nic=None,
reason_code="vpc_private_expectation_unavailable")
plan = _plan_from_record(record)
member = next((item for item in plan.members
if item.instance_uuid == instance_uuid), None)
target_row: Mapping[str, Any] | None = None
target_host_id = host_id
for host in record.get("hosts", []):
if not isinstance(host, Mapping):
continue
if member is not None and host.get("host_id") == member.host_id:
target_host_id = member.host_id
for row in host.get("retiring_members", []):
if (isinstance(row, Mapping)
and row.get("instance_uuid") == instance_uuid):
target_row = row
target_host_id = str(host.get("host_id"))
if member is None and target_row is None:
return self._instance_private_state(
instance_uuid, required=True, vpc_id=expected_vpc,
member_id=expected_member, desired_state="removed",
private_ip_cidr=expected.get("private_ip_cidr"),
private_mac=expected_private_mac,
desired_revision=int(expected_revision or 0),
desired_digest=expected_digest, host=None, member_nic=None,
reason_code="vpc_private_expectation_stale")
target_vpc_id = plan.vpc_id
target_member_id = (member.member_id if member is not None
else target_row.get("member_id") if target_row else None)
target_ip = (f"{member.private_ip}/32" if member is not None
else target_row.get("private_ip_cidr") if target_row else None)
try:
target_mac = (_effective_private_mac(
plan.vpc_id, target_member_id, member.private_mac)
if member is not None else _effective_private_mac(
plan.vpc_id, target_member_id,
target_row.get("private_mac") if target_row else None))
except (TypeError, ValueError, VpcError):
return self._instance_private_state(
instance_uuid, required=True, vpc_id=target_vpc_id,
member_id=target_member_id, desired_state=(
"active" if member is not None else "removed"),
private_ip_cidr=target_ip, private_mac=None,
desired_revision=plan.desired_revision,
desired_digest=plan.desired_digest, host=None, member_nic=None,
reason_code="vpc_private_mac_invalid")
immutable_mismatch = (
(expected_vpc is not None and expected_vpc != target_vpc_id)
or (expected_member is not None and expected_member != target_member_id)
or (expected_revision is not None
and expected_revision != plan.desired_revision)
or (expected_digest is not None
and expected_digest != plan.desired_digest)
or (expected.get("private_ip_cidr") is not None
and expected.get("private_ip_cidr") != target_ip)
or (expected_private_mac is not None
and expected_private_mac != target_mac)
)
if immutable_mismatch:
return self._instance_private_state(
instance_uuid, required=True, vpc_id=target_vpc_id,
member_id=target_member_id,
desired_state="active" if member is not None else "removed",
private_ip_cidr=target_ip, private_mac=target_mac,
desired_revision=plan.desired_revision,
desired_digest=plan.desired_digest, host=None, member_nic=None,
reason_code="vpc_private_expectation_mismatch")
if host_id is not None and target_host_id != host_id:
return self._instance_private_state(
instance_uuid, required=True,
vpc_id=member and plan.vpc_id or expected_vpc,
member_id=member and member.member_id or expected_member,
desired_state="active" if member else "removed",
private_ip_cidr=(f"{member.private_ip}/32" if member
else target_row.get("private_ip_cidr")),
private_mac=target_mac,
desired_revision=plan.desired_revision,
desired_digest=plan.desired_digest, host=None, member_nic=None,
reason_code="vpc_private_host_mismatch")
if member is not None:
if link_state == "absent" or expected_state == "removed":
# The control plane has not yet produced a removal target;
# never reinterpret a delete request as a harmless link
# preservation while the active member is still desired.
return self._instance_private_state(
instance_uuid, required=True, vpc_id=plan.vpc_id,
member_id=member.member_id, desired_state="removed",
private_ip_cidr=f"{member.private_ip}/32",
private_mac=target_mac,
desired_revision=plan.desired_revision,
desired_digest=plan.desired_digest, host=None,
member_nic=None, reason_code="vpc_private_teardown_pending")
if (expected_vpc is not None and expected_vpc != plan.vpc_id
or expected_member is not None
and expected_member != member.member_id):
return self._instance_private_state(
instance_uuid, required=True, vpc_id=plan.vpc_id,
member_id=member.member_id, desired_state="active",
private_ip_cidr=f"{member.private_ip}/32",
private_mac=target_mac,
desired_revision=plan.desired_revision,
desired_digest=plan.desired_digest, host=None,
member_nic=None, reason_code="vpc_private_expectation_mismatch")
# Keep the target UUID in the map even for preserve. The
# host lane uses membership to prioritize this lifecycle VM
# ahead of a large periodic member batch without changing its
# link policy.
link_map = {instance_uuid: link_state}
gate = self._fabric_gate(plan)
reconcile_kwargs: dict[str, Any] = {
"private_link_states": link_map,
}
if expected_operation_id is not None:
reconcile_kwargs["operation_id"] = expected_operation_id
await self._reconcile_pending_hosts(
plan, record, gate, plan, **reconcile_kwargs)
self._save("vpc_desired", key, record)
self._save_children(record)
desired = member
else:
# A removed member is represented by the host's durable
# retiring_members tombstone. The same host adapter pass
# removes private0 and proves the bridge/ACL state afterwards.
gate = self._fabric_gate(plan)
reconcile_kwargs: dict[str, Any] = {}
if expected_operation_id is not None:
reconcile_kwargs["operation_id"] = expected_operation_id
await self._reconcile_pending_hosts(
plan, record, gate, plan, **reconcile_kwargs)
self._save("vpc_desired", key, record)
self._save_children(record)
desired = None
host_row = next((item for item in record.get("hosts", [])
if isinstance(item, Mapping)
and item.get("host_id") == target_host_id), None)
nic = None
if isinstance(host_row, Mapping):
for item in host_row.get("member_nics", []):
if (isinstance(item, Mapping)
and item.get("instance_uuid") == instance_uuid):
nic = item
break
result = self._instance_private_state(
instance_uuid, required=True, vpc_id=plan.vpc_id,
member_id=(desired.member_id if desired is not None
else target_row.get("member_id") if target_row else expected_member),
desired_state="active" if desired is not None else "removed",
private_ip_cidr=(f"{desired.private_ip}/32" if desired is not None
else target_row.get("private_ip_cidr") if target_row else expected.get("private_ip_cidr")),
private_mac=target_mac,
desired_revision=plan.desired_revision,
desired_digest=plan.desired_digest,
host=host_row, member_nic=nic,
reason_code=(host_row.get("last_error_code")
if isinstance(host_row, Mapping) else None),
)
if result.get("readback") == "confirmed":
required_link: str | None = None
if link_state == "down":
required_link = "down"
elif link_state == "up":
required_link = "up"
elif link_state == "absent":
required_link = "absent"
elif lifecycle_kind.endswith("_policy_active"):
required_link = "up"
elif lifecycle_kind.endswith("_policy_blocked"):
required_link = "down"
policy_unready = lifecycle_kind.endswith("_policy_unready")
if (policy_unready
or required_link is not None
and result.get("link_state") != required_link):
result.update({
"state": "uncertain", "readback": "mismatch",
"interface": None, "private_ip_cidr": None,
"private_mac": None,
"reason_code":
"vpc_private_lifecycle_policy_mismatch",
"observed_at": int(time.time()),
})
return result
# Revalidate the optimistically discovered VPC under the canonical
# lock order. _reconcile_pending_hosts acquires the complete sorted
# active/retiring VM lock set before any libvirt or host mutation.
async with self._lock(key), self._catalog_mutation_lock():
return await _await_mutation_completion(
reconcile_instance_locked())
async def reconcile(self, vpc_id: str) -> dict[str, Any]:
_identifier(vpc_id, "VPC")
# Do not run a durable delete target through the ordinary apply
# reconciler: only ``delete`` can issue the exact positive receipt and
# purge the target after every required host reports absence. The
# first read is only an optimization; the row is rechecked under the
# VPC lock below to close the apply/delete race.
record = self._load("vpc_desired", vpc_id)
if record is not None and self._delete_marker(record) is not None:
return await self.delete(
vpc_id, self._delete_payload_from_record(record))
async def reconcile_locked(
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
record = self._load("vpc_desired", vpc_id)
if record is None:
raise VpcError("VPC 不存在", 404, "vpc_not_found")
if self._delete_marker(record) is not None:
return None, self._delete_payload_from_record(record)
plan = _plan_from_record(record)
if record.get("desired_digest") != plan.desired_digest:
raise VpcError("VPC 计划摘要与持久化记录不一致", 500,
"vpc_record_corrupt")
gate = self._fabric_gate(plan)
# Reconcile is the durable retry path after a host response
# was lost or the engine restarted. Refreshing only the
# Fabric gate would leave private0/routes/nft in an unknown
# state forever.
await self._reconcile_pending_hosts(
plan, record, gate, plan)
self._save("vpc_desired", vpc_id, record)
self._save_children(record)
return self._public(record), None
async with self._lock(vpc_id), self._catalog_mutation_lock():
result, delete_payload = await _await_mutation_completion(
reconcile_locked())
# The locked recheck may observe a delete target that was installed
# after the optimistic read. Release the non-reentrant lock before
# entering ``delete``, which acquires the same per-VPC and catalog
# locks itself.
if delete_payload is not None:
return await self.delete(vpc_id, delete_payload)
assert result is not None
return result
def recover(self) -> None:
"""Fail closed after an engine restart until readback is supplied."""
# Successful roots have no desired row left, but old deployments may
# still contain the pre-compaction receipt shape. Validate and reduce
# those tombstones before normal recovery; corrupt identities remain
# fail-closed for operator inspection.
for key in tuple(self.store.list(VPC_DELETE_RECEIPT_KIND)):
try:
self._load_delete_receipt(key)
except VpcError:
continue
for key, raw in self.store.list("vpc_desired").items():
try:
value = json.loads(raw)
except (TypeError, ValueError):
continue
if not isinstance(value, dict):
continue
if value.get("vpc_id") != key:
# Do not move or rewrite children under a different VPC key
# when a durable row has been tampered with.
continue
value["state"] = "uncertain"
value["next_stage_allowed"] = False
value["private_dataplane_verified"] = False
# A previous Fabric gate may have been active before the process
# stopped. Preserve its revision/evidence for audit, but never
# expose that stale gate as usable after restart.
gate = value.get("fabric_gate")
if isinstance(gate, dict):
gate["active"] = False
gate["next_stage_allowed"] = False
gate["error_code"] = "vpc_recovery_pending"
value["recovery_pending"] = True
# Preserve the last durable observed evidence for audit and for a
# future read-only reconciler. The gate is still closed until a
# fresh observation proves that this revision remains converged.
marker = value.get("delete_target")
marker_revision = (marker.get("desired_revision")
if isinstance(marker, Mapping) else None)
marker_digest = (marker.get("desired_digest")
if isinstance(marker, Mapping) else None)
record_revision = value.get("desired_revision")
record_digest = value.get("desired_digest")
for host in value.get("hosts", []):
if isinstance(host, dict):
# An exact absence proof is terminal evidence, not a live
# readiness claim. Preserve both root-delete receipts and
# normal apply empty-host removals across restart so retry
# contacts only hosts whose removal is still pending.
delete_confirmed = (
isinstance(marker_revision, int)
and not isinstance(marker_revision, bool)
and isinstance(marker_digest, str)
and self._confirmed_removal(
host, marker_revision, marker_digest)
)
apply_confirmed = (
isinstance(record_revision, int)
and not isinstance(record_revision, bool)
and isinstance(record_digest, str)
and self._confirmed_removal(
host, record_revision, record_digest)
)
if delete_confirmed or apply_confirmed:
continue
host.update({"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False})
value["updated_at"] = int(time.time())
self._save("vpc_desired", key, value)
self._save_children(value)
async def reconcile_all(self) -> None:
for item in await __import__("asyncio").to_thread(self.list_status):
vpc_id = item.get("vpc_id")
if not isinstance(vpc_id, str):
continue
try:
await self.reconcile(vpc_id)
except Exception:
continue
async def reconcile_loop(self, interval: float = 30.0) -> None:
import asyncio
while True:
await self.reconcile_all()
await asyncio.sleep(interval)