You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
4451 lines
219 KiB
Python
4451 lines
219 KiB
Python
"""Structured VPC control-plane state.
|
|
|
|
This module is deliberately a control-plane boundary. It validates a typed
|
|
desired state, stores an encrypted durable copy and keeps a separate observed
|
|
revision for the future private-network reconciler. It does not execute
|
|
``ip``, ``wg``, libvirt or nftables commands. The host WireGuard Fabric is a
|
|
prerequisite and is checked as a read-only gate before a VPC can be released
|
|
to the later data-plane phase.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
|
from contextlib import AsyncExitStack, asynccontextmanager
|
|
import hashlib
|
|
import ipaddress
|
|
import json
|
|
import re
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass, replace
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
from .fabric import (
|
|
FabricCoordinator,
|
|
FabricError,
|
|
_await_mutation_completion,
|
|
)
|
|
|
|
|
|
VPC_SCHEMA_VERSION = 1
|
|
VPC_INTERFACE = "private0"
|
|
VPC_MAX_HOSTS = 64
|
|
VPC_MAX_SUBNETS = 256
|
|
VPC_MAX_MEMBERS = 4096
|
|
VPC_MAX_ROUTES = 1024
|
|
VPC_MAX_ACLS = 2048
|
|
VPC_MAX_READBACK = 4096
|
|
VPC_MAX_READBACK_VALUE = 512
|
|
VPC_EVIDENCE_VERSION = 2
|
|
# libvirt rejects an incomplete QGA response after 10 MiB, but its API must
|
|
# materialize the complete bounded reply before Python can apply the much
|
|
# smaller per-command gate below. Keep the rolling pool deliberately narrow
|
|
# so several hostile or broken guests cannot multiply that upstream buffer
|
|
# into an avoidable host-memory spike.
|
|
VPC_GUEST_PROBE_WORKERS = 4
|
|
# Per-request rolling pools must also share one process-wide admission gate.
|
|
# Without it, concurrent VPC requests each admit four libvirt QGA replies and
|
|
# multiply the upstream response-buffer bound. BoundedSemaphore also turns an
|
|
# accidental over-release into an immediate programming error.
|
|
_VPC_GUEST_PROBE_SLOTS = threading.BoundedSemaphore(VPC_GUEST_PROBE_WORKERS)
|
|
VPC_GUEST_EXEC_KIND = "vpc_guest_exec"
|
|
VPC_GUEST_EXEC_MAX_BYTES = 1024
|
|
# PHP gives this endpoint 60 seconds and holds the operation lease for 120
|
|
# seconds. This monotonic budget stops admission of further hosts/QGA calls at
|
|
# 40 seconds, leaving 20 seconds for response/transport and another 60 seconds
|
|
# before a second worker may reclaim the operation. A libvirt/seed mutation
|
|
# already in flight is not safely preemptible; shutdown waits for bounded QGA
|
|
# calls instead of leaving background work after the response.
|
|
VPC_REQUEST_DEADLINE_SECONDS = 40.0
|
|
VPC_STATES = frozenset({"applying", "active", "degraded", "uncertain"})
|
|
VPC_DELETE_RECEIPT_KIND = "vpc_deleted"
|
|
VPC_DELETE_DETERMINISTIC_HOST_ERRORS = frozenset({
|
|
"vpc_private_remove_requires_stopped",
|
|
"vpc_private_remove_unsafe",
|
|
"vpc_private_interface_conflict",
|
|
"vpc_seed_missing",
|
|
"vpc_seed_readback_invalid",
|
|
"vpc_seed_network_policy_migration_required",
|
|
"vpc_private_xml_invalid",
|
|
})
|
|
VPC_FABRIC_ADMISSION_ERRORS = frozenset({
|
|
"vpc_fabric_region_mismatch",
|
|
"vpc_fabric_identity_mismatch",
|
|
"vpc_fabric_endpoint_overlap",
|
|
})
|
|
|
|
_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z")
|
|
_HOST_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
|
|
_DIGEST = re.compile(r"[0-9a-f]{64}\Z")
|
|
_OPERATION_ID = re.compile(r"[A-Za-z0-9_-]{8,80}\Z")
|
|
_PRIVATE_LIFECYCLE_KIND = re.compile(
|
|
r"(?:create|reinstall|network_change|instance_settings|rescue|"
|
|
r"snapshot_create|snapshot_restore|snapshot_delete|delete|expiry_delete|"
|
|
r"action|service_suspend|service_resume|network_suspend|network_resume|"
|
|
r"firewall_apply|"
|
|
r"action_(?:start|shutdown|reboot|poweroff|reset)_policy_"
|
|
r"(?:active|blocked|unready)|"
|
|
r"service_suspend_policy_(?:active|blocked|unready)|"
|
|
r"service_resume_(?:start|stopped|unbound)_policy_"
|
|
r"(?:active|blocked|unready))\Z"
|
|
)
|
|
# Compatibility markers used by the asynchronous reinstall worker. A same
|
|
# operation private reconcile may pass the lifecycle barrier only after the
|
|
# replacement domain has crossed its irreversible commit boundary.
|
|
_REINSTALL_COMMITTED_PHASES = frozenset({
|
|
"cleaning_up", "cleanup_pending", "result_uncertain", "completed",
|
|
"重装已完成,但凭据或任务记录保存失败",
|
|
"重装完成,但旧系统盘清理待核对", "重装结果待核对",
|
|
})
|
|
_REINSTALL_PUBLIC_ERROR_CODES = frozenset({
|
|
"reinstall_cleanup_pending", "reinstall_result_uncertain",
|
|
})
|
|
# CloudInstance's existing identity boundary (and libvirt itself) accepts a
|
|
# canonical five-group UUID even when the version/variant nibbles are not
|
|
# RFC-4122 values. VPC membership must address the already persisted VM
|
|
# identity, so imposing an RFC-only subset here would reject valid legacy or
|
|
# operator-supplied instances that the PHP DTO intentionally permits.
|
|
_UUID = re.compile(
|
|
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
|
|
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z"
|
|
)
|
|
_MAC = re.compile(r"[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\Z")
|
|
_PROTOCOLS = frozenset({"any", "tcp", "udp", "icmp"})
|
|
_DIRECTIONS = frozenset({"ingress", "egress"})
|
|
_ACTIONS = frozenset({"allow", "deny"})
|
|
|
|
|
|
class VpcError(Exception):
|
|
"""Safe error returned by the structured VPC boundary."""
|
|
|
|
def __init__(self, message: str, status: int = 400,
|
|
error_code: str = "vpc_invalid") -> None:
|
|
super().__init__(message)
|
|
self.status = status
|
|
self.error_code = error_code
|
|
|
|
|
|
def _reinstall_job_committed(job: Mapping[str, Any]) -> bool:
|
|
"""Whether a durable reinstall row proves the replacement boundary."""
|
|
if job.get("reinstall_committed") is True:
|
|
return True
|
|
if job.get("error_code") in _REINSTALL_PUBLIC_ERROR_CODES:
|
|
return True
|
|
phase = job.get("phase")
|
|
if isinstance(phase, str) and phase.strip() in _REINSTALL_COMMITTED_PHASES:
|
|
return True
|
|
if job.get("cleanup_pending") is True:
|
|
return True
|
|
warning = job.get("warning")
|
|
return isinstance(warning, str) and bool(warning.strip())
|
|
|
|
|
|
def _string(value: Any, label: str, maximum: int = 128) -> str:
|
|
if (not isinstance(value, str) or not value or len(value) > maximum
|
|
or value != value.strip()
|
|
or any(ord(char) < 32 or ord(char) == 127 for char in value)):
|
|
raise VpcError(f"{label}无效")
|
|
return value
|
|
|
|
|
|
def _identifier(value: Any, label: str = "标识") -> str:
|
|
if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None:
|
|
raise VpcError(f"{label}无效")
|
|
return value
|
|
|
|
|
|
def _host_id(value: Any) -> str:
|
|
if not isinstance(value, str) or _HOST_ID.fullmatch(value) is None:
|
|
raise VpcError("VPC 宿主机标识无效")
|
|
return value
|
|
|
|
|
|
def _revision(value: Any, label: str = "revision") -> int:
|
|
if (isinstance(value, bool) or not isinstance(value, int)
|
|
or not 1 <= value <= (1 << 63) - 1):
|
|
raise VpcError(f"{label}无效")
|
|
return value
|
|
|
|
|
|
def _uuid(value: Any) -> str:
|
|
if not isinstance(value, str) or _UUID.fullmatch(value) is None:
|
|
raise VpcError("VPC 成员实例 UUID 无效")
|
|
return value.lower()
|
|
|
|
|
|
def _mac(value: Any) -> str | None:
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, str) or _MAC.fullmatch(value) is None:
|
|
raise VpcError("VPC 成员 MAC 地址无效")
|
|
parsed = int(value.replace(":", ""), 16)
|
|
# Multicast and all-zero addresses cannot identify a guest interface.
|
|
if parsed == 0 or (int(value[:2], 16) & 1):
|
|
raise VpcError("VPC 成员 MAC 地址无效")
|
|
return value.lower()
|
|
|
|
|
|
def _effective_private_mac(
|
|
vpc_id: str, member_id: str, configured: Any = None,
|
|
) -> str:
|
|
"""Return the canonical MAC used by the host data-plane adapter.
|
|
|
|
Older durable VPC records may omit ``private_mac``. The dataplane parser
|
|
already derives the same stable locally-administered address in that
|
|
case; lifecycle evidence must use that value too, otherwise the worker
|
|
receives a false mismatch after a successful host readback. Explicit
|
|
values remain accepted for backwards compatibility but are still checked
|
|
as a valid unicast MAC by :func:`_mac`.
|
|
"""
|
|
if (_IDENTIFIER.fullmatch(vpc_id) is None
|
|
or _IDENTIFIER.fullmatch(member_id) is None):
|
|
raise VpcError("VPC 成员标识无效", 422)
|
|
value = _mac(configured)
|
|
if value is not None:
|
|
return value
|
|
# Keep one implementation of the derivation in the dataplane module and
|
|
# import lazily to avoid its intentional dependency on this control-plane
|
|
# module during module initialization.
|
|
from .vpc_dataplane import derive_private_mac
|
|
|
|
return derive_private_mac(vpc_id, member_id)
|
|
|
|
|
|
def _network(value: Any, label: str, *, rfc1918: bool = False,
|
|
min_prefix: int = 0, max_prefix: int = 32) -> ipaddress.IPv4Network:
|
|
if not isinstance(value, str) or value != value.strip():
|
|
raise VpcError(f"{label}无效")
|
|
try:
|
|
network = ipaddress.ip_network(value, strict=True)
|
|
except ValueError as exc:
|
|
raise VpcError(f"{label}无效") from exc
|
|
if not isinstance(network, ipaddress.IPv4Network):
|
|
raise VpcError(f"{label}必须是 IPv4 网段")
|
|
if not min_prefix <= network.prefixlen <= max_prefix:
|
|
raise VpcError(f"{label}前缀长度无效")
|
|
if rfc1918 and not any(
|
|
network.subnet_of(ipaddress.IPv4Network(block))
|
|
for block in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")):
|
|
raise VpcError("VPC CIDR 必须使用 RFC1918 地址")
|
|
return network
|
|
|
|
|
|
def _host_address(value: Any) -> str:
|
|
if not isinstance(value, str) or value != value.strip():
|
|
raise VpcError("VPC 私网 IP 无效")
|
|
raw = value
|
|
if "/" in raw:
|
|
try:
|
|
interface = ipaddress.ip_interface(raw)
|
|
except ValueError as exc:
|
|
raise VpcError("VPC 私网 IP 必须是 /32") from exc
|
|
if not isinstance(interface, ipaddress.IPv4Interface) or interface.network.prefixlen != 32:
|
|
raise VpcError("VPC 私网 IP 必须是 /32")
|
|
return str(interface.ip)
|
|
try:
|
|
address = ipaddress.IPv4Address(raw)
|
|
except ipaddress.AddressValueError as exc:
|
|
raise VpcError("VPC 私网 IP 无效") from exc
|
|
return str(address)
|
|
|
|
|
|
def _digest(value: Any, label: str = "desired_digest") -> str:
|
|
if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
|
|
raise VpcError(f"{label}无效")
|
|
return value
|
|
|
|
|
|
def _safe_scalar(value: Any) -> bool:
|
|
if isinstance(value, bool) or value is None:
|
|
return True
|
|
if isinstance(value, int):
|
|
return -(1 << 63) <= value <= (1 << 63) - 1
|
|
return (isinstance(value, str)
|
|
and len(value) <= VPC_MAX_READBACK_VALUE
|
|
and all(ord(char) >= 32 and ord(char) != 127 for char in value))
|
|
|
|
|
|
def _guest_exec_core(value: Any) -> dict[str, Any]:
|
|
"""Validate one bounded internal QGA launch/PID ledger payload."""
|
|
if not isinstance(value, Mapping):
|
|
raise VpcError("VPC 来宾执行账本损坏", 500,
|
|
"vpc_record_corrupt")
|
|
phase = value.get("phase")
|
|
fields = {
|
|
"version", "phase", "domain_uuid", "domain_runtime_id",
|
|
"qemu_runtime_token", "guest_network_generation", "probe_kind",
|
|
}
|
|
if phase == "pending":
|
|
fields.add("pid")
|
|
generation = value.get("guest_network_generation")
|
|
if (set(value) != fields
|
|
or value.get("version") != 1
|
|
or phase not in {"launch_intent", "pending"}
|
|
or not isinstance(value.get("domain_uuid"), str)
|
|
or _UUID.fullmatch(value["domain_uuid"]) is None
|
|
or isinstance(value.get("domain_runtime_id"), bool)
|
|
or not isinstance(value.get("domain_runtime_id"), int)
|
|
or not 0 <= value["domain_runtime_id"] <= 2**31 - 1
|
|
or not isinstance(value.get("qemu_runtime_token"), str)
|
|
or re.fullmatch(
|
|
r"[0-9a-f]{32}", value["qemu_runtime_token"]) is None
|
|
or not isinstance(generation, str)
|
|
or not 1 <= len(generation) <= 128
|
|
or any(ord(char) < 0x20 or ord(char) > 0x7e
|
|
for char in generation)
|
|
or value.get("probe_kind") not in {"link", "routes"}):
|
|
raise VpcError("VPC 来宾执行账本损坏", 500,
|
|
"vpc_record_corrupt")
|
|
if phase == "pending" and (
|
|
isinstance(value.get("pid"), bool)
|
|
or not isinstance(value.get("pid"), int)
|
|
or not 1 <= value["pid"] <= 2**63 - 1):
|
|
raise VpcError("VPC 来宾执行账本损坏", 500,
|
|
"vpc_record_corrupt")
|
|
return dict(value)
|
|
|
|
|
|
def _canonical_cidr(network: ipaddress.IPv4Network) -> str:
|
|
return str(network)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcSubnet:
|
|
subnet_id: str
|
|
cidr: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcMember:
|
|
member_id: str
|
|
instance_uuid: str
|
|
host_id: str
|
|
private_ip: str
|
|
subnet_id: str
|
|
private_mac: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcDeleteMember:
|
|
"""Signed VM identity whose persistent private NIC must be absent."""
|
|
|
|
member_id: str
|
|
instance_uuid: str
|
|
host_id: str
|
|
private_ip: str
|
|
private_mac: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcRoute:
|
|
route_id: str
|
|
destination_cidr: str
|
|
next_hop_member_id: str | None
|
|
next_hop_host_id: str | None
|
|
metric: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcAcl:
|
|
acl_id: str
|
|
direction: str
|
|
action: str
|
|
protocol: str
|
|
source_cidr: str
|
|
destination_cidr: str
|
|
port_from: int | None
|
|
port_to: int | None
|
|
priority: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcHost:
|
|
host_id: str
|
|
member_ids: tuple[str, ...]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcPlan:
|
|
vpc_id: str
|
|
region: str
|
|
fabric_id: str
|
|
fabric_revision: int
|
|
cidr: str
|
|
desired_revision: int
|
|
subnets: tuple[VpcSubnet, ...]
|
|
members: tuple[VpcMember, ...]
|
|
hosts: tuple[VpcHost, ...]
|
|
routes: tuple[VpcRoute, ...]
|
|
acls: tuple[VpcAcl, ...]
|
|
desired_digest: str
|
|
operation_id: str
|
|
|
|
@property
|
|
def host_ids(self) -> frozenset[str]:
|
|
return frozenset(host.host_id for host in self.hosts)
|
|
|
|
def member(self, member_id: str) -> VpcMember:
|
|
for item in self.members:
|
|
if item.member_id == member_id:
|
|
return item
|
|
raise KeyError(member_id)
|
|
|
|
def canonical(self) -> dict[str, Any]:
|
|
# operation_id is an envelope/idempotency token. PHP computes the
|
|
# desired digest before attaching it, so it must never change the
|
|
# identity of an otherwise identical typed target.
|
|
return {
|
|
"schema_version": VPC_SCHEMA_VERSION,
|
|
"vpc_id": self.vpc_id,
|
|
"region": self.region,
|
|
"fabric_id": self.fabric_id,
|
|
"fabric_revision": self.fabric_revision,
|
|
"cidr": self.cidr,
|
|
"desired_revision": self.desired_revision,
|
|
"hosts": [
|
|
{"host_id": host.host_id,
|
|
# ``members`` is the wire key used by the PHP desired DTO.
|
|
# Keep the canonical form identical across both sides so a
|
|
# digest computed before signing is the digest verified by
|
|
# the engine parser.
|
|
"members": list(host.member_ids)}
|
|
for host in self.hosts
|
|
],
|
|
"subnets": [
|
|
{"subnet_id": item.subnet_id, "cidr": item.cidr}
|
|
for item in self.subnets
|
|
],
|
|
"members": [
|
|
{"member_id": item.member_id,
|
|
"instance_uuid": item.instance_uuid,
|
|
"host_id": item.host_id,
|
|
"private_ip_cidr": f"{item.private_ip}/32",
|
|
"subnet_id": item.subnet_id,
|
|
**({"private_mac": item.private_mac}
|
|
if item.private_mac is not None else {})}
|
|
for item in self.members
|
|
],
|
|
"routes": [
|
|
{"route_id": item.route_id,
|
|
"destination_cidr": item.destination_cidr,
|
|
**({"next_hop_member_id": item.next_hop_member_id}
|
|
if item.next_hop_member_id is not None else {}),
|
|
**({"next_hop_host_id": item.next_hop_host_id}
|
|
if item.next_hop_host_id is not None else {}),
|
|
"metric": item.metric}
|
|
for item in self.routes
|
|
],
|
|
"acls": [
|
|
{"acl_id": item.acl_id,
|
|
"direction": item.direction,
|
|
"action": item.action,
|
|
"protocol": item.protocol,
|
|
"source_cidr": item.source_cidr,
|
|
"destination_cidr": item.destination_cidr,
|
|
**({"port_from": item.port_from}
|
|
if item.port_from is not None else {}),
|
|
**({"port_to": item.port_to}
|
|
if item.port_to is not None else {}),
|
|
"priority": item.priority}
|
|
for item in self.acls
|
|
],
|
|
}
|
|
|
|
|
|
def _canonical_digest(canonical: Mapping[str, Any]) -> str:
|
|
return hashlib.sha256(
|
|
json.dumps(canonical, sort_keys=True, separators=(",", ":"),
|
|
ensure_ascii=True).encode("utf-8")
|
|
).hexdigest()
|
|
|
|
|
|
def _field_set(raw: Mapping[str, Any], allowed: set[str], label: str) -> None:
|
|
if set(raw) - allowed:
|
|
raise VpcError(f"{label}包含不支持的字段")
|
|
|
|
|
|
def _alias(raw: Mapping[str, Any], primary: str, legacy: str,
|
|
label: str) -> Any:
|
|
"""Read a compatibility alias without allowing conflicting values."""
|
|
first = raw.get(primary)
|
|
second = raw.get(legacy)
|
|
if first is not None and second is not None and first != second:
|
|
raise VpcError(f"{label}字段不一致")
|
|
return first if first is not None else second
|
|
|
|
|
|
def _parse_revision(value: Mapping[str, Any]) -> int:
|
|
desired = value.get("desired_revision")
|
|
legacy = value.get("revision")
|
|
if desired is None:
|
|
desired = legacy
|
|
elif legacy is not None and desired != legacy:
|
|
raise VpcError("desired_revision 与 revision 不一致")
|
|
return _revision(desired, "desired_revision")
|
|
|
|
|
|
def _parse_members(
|
|
value: Mapping[str, Any],
|
|
network: ipaddress.IPv4Network,
|
|
subnets: tuple[VpcSubnet, ...],
|
|
) -> tuple[tuple[VpcMember, ...], tuple[VpcHost, ...]]:
|
|
subnet_networks = {
|
|
item.subnet_id: _network(item.cidr, "VPC subnet")
|
|
for item in subnets
|
|
}
|
|
raw_members = value.get("members")
|
|
raw_hosts = value.get("hosts")
|
|
if raw_members is not None and not isinstance(raw_members, list):
|
|
raise VpcError("VPC members 必须是数组")
|
|
if raw_hosts is not None and (not isinstance(raw_hosts, list)
|
|
or len(raw_hosts) > VPC_MAX_HOSTS):
|
|
raise VpcError("VPC hosts 必须是数组")
|
|
members: list[VpcMember] = []
|
|
member_ids: set[str] = set()
|
|
instances: set[str] = set()
|
|
private_ips: set[str] = set()
|
|
host_member_ids: dict[str, list[str]] = {}
|
|
|
|
def add_member(raw: Mapping[str, Any], forced_host: str | None = None) -> None:
|
|
_field_set(raw, {"member_id", "instance_uuid", "host_id",
|
|
"private_ip", "private_ip_cidr", "private_mac",
|
|
"subnet_id"}, "VPC member")
|
|
instance_uuid = _uuid(raw.get("instance_uuid"))
|
|
host_id = _host_id(raw.get("host_id", forced_host))
|
|
if forced_host is not None and host_id != forced_host:
|
|
raise VpcError("VPC 成员宿主机与 hosts 不一致")
|
|
member_id = raw.get("member_id", instance_uuid)
|
|
member_id = _identifier(member_id, "VPC member")
|
|
if member_id in member_ids or instance_uuid in instances:
|
|
# It is useful for callers to send both the normalized top-level
|
|
# ``members`` list and the per-host grouping. Accept that only
|
|
# when the second copy is byte-for-byte equivalent after
|
|
# canonicalization; conflicting copies remain a hard error.
|
|
existing = next((item for item in members
|
|
if item.member_id == member_id), None)
|
|
if existing is None:
|
|
raise VpcError("VPC 成员不能重复")
|
|
candidate_ip = _host_address(
|
|
_alias(raw, "private_ip_cidr", "private_ip", "VPC 私网 IP"))
|
|
candidate_subnet = raw.get("subnet_id", existing.subnet_id)
|
|
candidate_mac = _mac(raw.get("private_mac"))
|
|
if (existing.instance_uuid != instance_uuid
|
|
or existing.host_id != host_id
|
|
or existing.private_ip != candidate_ip
|
|
or existing.subnet_id != candidate_subnet
|
|
or existing.private_mac != candidate_mac):
|
|
raise VpcError("VPC 成员重复定义不一致")
|
|
host_member_ids.setdefault(host_id, []).append(member_id)
|
|
return
|
|
raw_ip = _alias(raw, "private_ip_cidr", "private_ip", "VPC 私网 IP")
|
|
private_ip = _host_address(raw_ip)
|
|
address = ipaddress.IPv4Address(private_ip)
|
|
if address not in network or address in {
|
|
network.network_address, network.broadcast_address}:
|
|
raise VpcError("VPC 私网 IP 不在 VPC 可用地址范围内")
|
|
if private_ip in private_ips:
|
|
raise VpcError("VPC 私网 IP 不能重复")
|
|
raw_subnet = raw.get("subnet_id")
|
|
if raw_subnet is None:
|
|
matching = [sid for sid, subnet in subnet_networks.items()
|
|
if address in subnet
|
|
and address not in {subnet.network_address,
|
|
subnet.broadcast_address}]
|
|
if len(matching) != 1:
|
|
raise VpcError("VPC 成员必须明确归属一个 subnet")
|
|
subnet_id = matching[0]
|
|
else:
|
|
subnet_id = _identifier(raw_subnet, "VPC subnet")
|
|
subnet = subnet_networks.get(subnet_id)
|
|
if subnet is None or address not in subnet or address in {
|
|
subnet.network_address, subnet.broadcast_address}:
|
|
raise VpcError("VPC 成员私网 IP 不属于指定 subnet")
|
|
private_mac = _mac(raw.get("private_mac"))
|
|
members.append(VpcMember(member_id, instance_uuid, host_id,
|
|
private_ip, subnet_id, private_mac))
|
|
member_ids.add(member_id)
|
|
instances.add(instance_uuid)
|
|
private_ips.add(private_ip)
|
|
host_member_ids.setdefault(host_id, []).append(member_id)
|
|
|
|
if raw_members is not None:
|
|
for raw in raw_members:
|
|
if not isinstance(raw, Mapping):
|
|
raise VpcError("VPC member 必须是对象")
|
|
add_member(raw)
|
|
if raw_hosts is not None:
|
|
explicit_hosts: dict[str, list[str]] = {}
|
|
for raw in raw_hosts:
|
|
if not isinstance(raw, Mapping):
|
|
raise VpcError("VPC host 必须是对象")
|
|
_field_set(raw, {"host_id", "members"}, "VPC host")
|
|
host_id = _host_id(raw.get("host_id"))
|
|
if host_id in explicit_hosts:
|
|
raise VpcError("VPC 宿主机不能重复")
|
|
raw_host_members = raw.get("members", [])
|
|
if not isinstance(raw_host_members, list):
|
|
raise VpcError("VPC host members 必须是数组")
|
|
explicit_hosts[host_id] = []
|
|
for item in raw_host_members:
|
|
if isinstance(item, str):
|
|
member_id = _identifier(item, "VPC member")
|
|
existing = next((member for member in members
|
|
if member.member_id == member_id), None)
|
|
if existing is None or existing.host_id != host_id:
|
|
raise VpcError("VPC host member 不存在或宿主机不一致")
|
|
explicit_hosts[host_id].append(member_id)
|
|
continue
|
|
if not isinstance(item, Mapping):
|
|
raise VpcError("VPC host member 必须是对象")
|
|
add_member(item, host_id)
|
|
raw_member_id = item.get("member_id")
|
|
explicit_hosts[host_id].append(
|
|
_identifier(raw_member_id, "VPC member")
|
|
if raw_member_id is not None else _uuid(item.get("instance_uuid")))
|
|
# A member may be represented either at top level or under hosts, but
|
|
# never twice. Explicit host membership must exactly describe all
|
|
# top-level members for that host.
|
|
if raw_hosts is not None:
|
|
for host_id, member_ids_for_host in host_member_ids.items():
|
|
if host_id not in explicit_hosts:
|
|
raise VpcError("VPC member 缺少 hosts 归属")
|
|
if set(member_ids_for_host) != set(explicit_hosts[host_id]):
|
|
raise VpcError("VPC hosts 与 members 不一致")
|
|
host_member_ids = explicit_hosts
|
|
for host_id in explicit_hosts:
|
|
host_member_ids.setdefault(host_id, [])
|
|
if any(len(ids) != len(set(ids)) for ids in host_member_ids.values()):
|
|
raise VpcError("VPC 宿主机成员不能重复")
|
|
if len(members) > VPC_MAX_MEMBERS:
|
|
raise VpcError("VPC 成员数量超过限制")
|
|
if raw_hosts is None:
|
|
# Empty VPCs are valid desired state, but they cannot pass the next
|
|
# stage gate until a participating host/member is declared.
|
|
hosts = [VpcHost(host_id, tuple(sorted(set(ids))))
|
|
for host_id, ids in host_member_ids.items()]
|
|
else:
|
|
hosts = [VpcHost(host_id, tuple(sorted(set(ids))))
|
|
for host_id, ids in host_member_ids.items()]
|
|
return tuple(sorted(members, key=lambda item: item.member_id)), tuple(
|
|
sorted(hosts, key=lambda item: item.host_id))
|
|
|
|
|
|
def parse_vpc_plan(value: Mapping[str, Any]) -> VpcPlan:
|
|
"""Validate and canonicalize a tenant VPC desired state.
|
|
|
|
The parser intentionally rejects shell-like, host-specific or public
|
|
network fields. It accepts the engine contract's ``desired_revision``
|
|
spelling and the old ``revision`` spelling only as an equal alias.
|
|
"""
|
|
if not isinstance(value, Mapping):
|
|
raise VpcError("VPC 计划必须是对象")
|
|
allowed = {
|
|
"schema_version", "vpc_id", "region", "fabric_id", "fabric_revision",
|
|
"cidr", "network_cidr", "desired_revision", "revision",
|
|
"desired_digest", "operation_id", "hosts", "members", "subnets",
|
|
"routes", "acls",
|
|
}
|
|
_field_set(value, allowed, "VPC 计划")
|
|
if value.get("schema_version", VPC_SCHEMA_VERSION) != VPC_SCHEMA_VERSION:
|
|
raise VpcError("VPC 计划版本不支持")
|
|
vpc_id = _identifier(value.get("vpc_id"), "VPC")
|
|
region = _string(value.get("region"), "VPC 地区", 64)
|
|
fabric_id = _identifier(value.get("fabric_id"), "Fabric")
|
|
fabric_revision = _revision(value.get("fabric_revision"), "Fabric revision")
|
|
cidr_value = _alias(value, "cidr", "network_cidr", "VPC CIDR")
|
|
network = _network(cidr_value, "VPC CIDR", rfc1918=True,
|
|
min_prefix=16, max_prefix=24)
|
|
desired_revision = _parse_revision(value)
|
|
raw_subnets = value.get("subnets", [])
|
|
if not isinstance(raw_subnets, list) or len(raw_subnets) > VPC_MAX_SUBNETS:
|
|
raise VpcError("VPC subnets 必须是数组且不超过限制")
|
|
subnets: list[VpcSubnet] = []
|
|
subnet_ids: set[str] = set()
|
|
subnet_networks: list[ipaddress.IPv4Network] = []
|
|
for raw in raw_subnets:
|
|
if not isinstance(raw, Mapping):
|
|
raise VpcError("VPC subnet 必须是对象")
|
|
_field_set(raw, {"subnet_id", "id", "cidr", "name"}, "VPC subnet")
|
|
subnet_id = _identifier(_alias(raw, "subnet_id", "id", "VPC subnet"),
|
|
"VPC subnet")
|
|
if subnet_id in subnet_ids:
|
|
raise VpcError("VPC subnet 不能重复")
|
|
subnet = _network(raw.get("cidr"), "VPC subnet", min_prefix=network.prefixlen,
|
|
max_prefix=30)
|
|
if not subnet.subnet_of(network):
|
|
raise VpcError("VPC subnet 必须位于 VPC CIDR 内")
|
|
if any(subnet.overlaps(other) for other in subnet_networks):
|
|
raise VpcError("VPC subnet 不能重叠")
|
|
subnet_ids.add(subnet_id)
|
|
subnet_networks.append(subnet)
|
|
subnets.append(VpcSubnet(subnet_id, _canonical_cidr(subnet)))
|
|
subnets.sort(key=lambda item: item.subnet_id)
|
|
members, hosts = _parse_members(
|
|
{**value, "subnets": [item.__dict__ for item in subnets]},
|
|
network, tuple(subnets))
|
|
host_ids = {host.host_id for host in hosts}
|
|
member_by_id = {item.member_id: item for item in members}
|
|
raw_routes = value.get("routes", [])
|
|
if not isinstance(raw_routes, list) or len(raw_routes) > VPC_MAX_ROUTES:
|
|
raise VpcError("VPC routes 必须是数组且不超过限制")
|
|
routes: list[VpcRoute] = []
|
|
route_ids: set[str] = set()
|
|
route_keys: set[tuple[str, str | None, str | None]] = set()
|
|
for raw in raw_routes:
|
|
if not isinstance(raw, Mapping):
|
|
raise VpcError("VPC route 必须是对象")
|
|
_field_set(raw, {"route_id", "id", "destination_cidr", "destination",
|
|
"next_hop_member_id", "next_hop_host_id", "metric"},
|
|
"VPC route")
|
|
route_id = _identifier(_alias(raw, "route_id", "id", "VPC route"),
|
|
"VPC route")
|
|
if route_id in route_ids:
|
|
raise VpcError("VPC route 不能重复")
|
|
destination = _network(
|
|
_alias(raw, "destination_cidr", "destination",
|
|
"VPC route destination"),
|
|
"VPC route destination", min_prefix=network.prefixlen, max_prefix=32)
|
|
# The private data plane is an L3 /32 fabric. A wider route would
|
|
# become a cryptokey-routing and nftables bypass (and could overlap a
|
|
# different tenant), so reject it at the control-plane boundary rather
|
|
# than relying on the host adapter to catch it later.
|
|
if destination.prefixlen != 32:
|
|
raise VpcError("VPC route destination 必须是精确 /32", 409,
|
|
"vpc_route_not_exact")
|
|
if not destination.subnet_of(network):
|
|
raise VpcError("VPC route destination 必须是 VPC 内的网段")
|
|
if destination.network_address in {
|
|
network.network_address, network.broadcast_address}:
|
|
raise VpcError("VPC route destination 不能使用 VPC 保留地址", 409,
|
|
"vpc_route_reserved_address")
|
|
member_target = raw.get("next_hop_member_id")
|
|
host_target = raw.get("next_hop_host_id")
|
|
if (member_target is None) == (host_target is None):
|
|
raise VpcError("VPC route 必须指定一个 next hop")
|
|
if member_target is not None:
|
|
member_target = _identifier(member_target, "VPC route member")
|
|
if member_target not in member_by_id:
|
|
raise VpcError("VPC route next hop member 不存在")
|
|
host_target = None
|
|
else:
|
|
host_target = _host_id(host_target)
|
|
if host_target not in host_ids:
|
|
raise VpcError("VPC route next hop host 不存在")
|
|
metric = raw.get("metric", 100)
|
|
if (isinstance(metric, bool) or not isinstance(metric, int)
|
|
or not 1 <= metric <= 65535):
|
|
raise VpcError("VPC route metric 无效")
|
|
route_key = (_canonical_cidr(destination), member_target, host_target)
|
|
if route_key in route_keys:
|
|
raise VpcError("VPC route 不能重复")
|
|
route_ids.add(route_id)
|
|
route_keys.add(route_key)
|
|
routes.append(VpcRoute(route_id, _canonical_cidr(destination),
|
|
member_target, host_target, metric))
|
|
routes.sort(key=lambda item: item.route_id)
|
|
raw_acls = value.get("acls", [])
|
|
if not isinstance(raw_acls, list) or len(raw_acls) > VPC_MAX_ACLS:
|
|
raise VpcError("VPC ACL 必须是数组且不超过限制")
|
|
acls: list[VpcAcl] = []
|
|
acl_ids: set[str] = set()
|
|
acl_priorities: set[int] = set()
|
|
for raw in raw_acls:
|
|
if not isinstance(raw, Mapping):
|
|
raise VpcError("VPC ACL 必须是对象")
|
|
_field_set(raw, {"acl_id", "id", "direction", "action", "protocol",
|
|
"source_cidr", "source", "destination_cidr",
|
|
"destination", "port_from", "port_to", "priority"},
|
|
"VPC ACL")
|
|
acl_id = _identifier(_alias(raw, "acl_id", "id", "VPC ACL"),
|
|
"VPC ACL")
|
|
if acl_id in acl_ids:
|
|
raise VpcError("VPC ACL 不能重复")
|
|
direction = raw.get("direction")
|
|
action = raw.get("action")
|
|
protocol = raw.get("protocol")
|
|
if direction not in _DIRECTIONS or action not in _ACTIONS or protocol not in _PROTOCOLS:
|
|
raise VpcError("VPC ACL direction/action/protocol 无效")
|
|
source = _network(_alias(raw, "source_cidr", "source", "VPC ACL source"),
|
|
"VPC ACL source", min_prefix=network.prefixlen,
|
|
max_prefix=32)
|
|
destination = _network(
|
|
_alias(raw, "destination_cidr", "destination",
|
|
"VPC ACL destination"),
|
|
"VPC ACL destination", min_prefix=network.prefixlen, max_prefix=32)
|
|
if not source.subnet_of(network) or not destination.subnet_of(network):
|
|
raise VpcError("VPC ACL 地址必须位于 VPC CIDR 内")
|
|
port_from = raw.get("port_from")
|
|
port_to = raw.get("port_to")
|
|
if protocol in {"tcp", "udp"}:
|
|
if port_from is None and port_to is None:
|
|
pass
|
|
elif (isinstance(port_from, bool) or not isinstance(port_from, int)
|
|
or isinstance(port_to, bool) or not isinstance(port_to, int)
|
|
or not 1 <= port_from <= port_to <= 65535):
|
|
raise VpcError("VPC ACL 端口范围无效")
|
|
elif port_from is not None or port_to is not None:
|
|
raise VpcError("非 TCP/UDP ACL 不能指定端口")
|
|
priority = raw.get("priority", len(acls) + 1)
|
|
if (isinstance(priority, bool) or not isinstance(priority, int)
|
|
or not 1 <= priority <= 1_000_000 or priority in acl_priorities):
|
|
raise VpcError("VPC ACL priority 无效或重复")
|
|
acl_ids.add(acl_id)
|
|
acl_priorities.add(priority)
|
|
acls.append(VpcAcl(
|
|
acl_id, direction, action, protocol,
|
|
_canonical_cidr(source), _canonical_cidr(destination),
|
|
port_from, port_to, priority))
|
|
acls.sort(key=lambda item: (item.priority, item.acl_id))
|
|
operation_id = value.get("operation_id")
|
|
if operation_id is None:
|
|
seed = f"{vpc_id}:{desired_revision}:{fabric_id}"
|
|
operation_id = "vpc-" + hashlib.sha256(seed.encode()).hexdigest()[:32]
|
|
elif not isinstance(operation_id, str) or _OPERATION_ID.fullmatch(operation_id) is None:
|
|
raise VpcError("VPC operation_id 无效")
|
|
canonical_hosts = tuple(sorted(hosts, key=lambda item: item.host_id))
|
|
provisional = VpcPlan(
|
|
vpc_id, region, fabric_id, fabric_revision, _canonical_cidr(network),
|
|
desired_revision, tuple(subnets), members, canonical_hosts,
|
|
tuple(routes), tuple(acls), "", operation_id)
|
|
computed_digest = _canonical_digest(provisional.canonical())
|
|
supplied_digest = value.get("desired_digest")
|
|
if supplied_digest is not None:
|
|
supplied_digest = _digest(supplied_digest)
|
|
if supplied_digest != computed_digest:
|
|
raise VpcError("desired_digest 与 VPC 计划不一致", 409,
|
|
"vpc_digest_conflict")
|
|
return VpcPlan(
|
|
provisional.vpc_id, provisional.region, provisional.fabric_id,
|
|
provisional.fabric_revision, provisional.cidr,
|
|
provisional.desired_revision, provisional.subnets,
|
|
provisional.members, provisional.hosts, provisional.routes,
|
|
provisional.acls, computed_digest, provisional.operation_id)
|
|
|
|
|
|
def _canonical_delete_members(
|
|
members: Sequence[VpcDeleteMember],
|
|
) -> list[dict[str, str]]:
|
|
return [{
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"host_id": member.host_id,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": member.private_mac,
|
|
} for member in members]
|
|
|
|
|
|
def _parse_delete_members(
|
|
value: Any, plan: VpcPlan,
|
|
) -> tuple[VpcDeleteMember, ...]:
|
|
if not isinstance(value, list) or len(value) > VPC_MAX_MEMBERS:
|
|
raise VpcError("VPC 删除成员清单必须是数组且不超过限制", 422,
|
|
"vpc_delete_manifest_invalid")
|
|
network = ipaddress.ip_network(plan.cidr)
|
|
members: list[VpcDeleteMember] = []
|
|
member_ids: set[str] = set()
|
|
instance_uuids: set[str] = set()
|
|
private_ips: set[str] = set()
|
|
private_macs: set[str] = set()
|
|
try:
|
|
for raw in value:
|
|
if not isinstance(raw, Mapping):
|
|
raise VpcError("VPC 删除成员必须是对象")
|
|
_field_set(raw, {
|
|
"member_id", "instance_uuid", "host_id",
|
|
"private_ip_cidr", "private_mac",
|
|
}, "VPC 删除成员")
|
|
member_id = _identifier(raw.get("member_id"), "VPC member")
|
|
instance_uuid = _uuid(raw.get("instance_uuid"))
|
|
host_id = _host_id(raw.get("host_id"))
|
|
if host_id not in plan.host_ids:
|
|
raise VpcError("VPC 删除成员宿主机不在删除目标中")
|
|
private_ip = _host_address(raw.get("private_ip_cidr"))
|
|
address = ipaddress.IPv4Address(private_ip)
|
|
if address not in network or address in {
|
|
network.network_address, network.broadcast_address}:
|
|
raise VpcError("VPC 删除成员私网 IP 不在 VPC 可用范围")
|
|
private_mac = _mac(raw.get("private_mac"))
|
|
if private_mac is None:
|
|
raise VpcError("VPC 删除成员 private_mac 缺失")
|
|
if (member_id in member_ids or instance_uuid in instance_uuids
|
|
or private_ip in private_ips
|
|
or private_mac in private_macs):
|
|
raise VpcError("VPC 删除成员清单包含重复身份")
|
|
member_ids.add(member_id)
|
|
instance_uuids.add(instance_uuid)
|
|
private_ips.add(private_ip)
|
|
private_macs.add(private_mac)
|
|
members.append(VpcDeleteMember(
|
|
member_id, instance_uuid, host_id, private_ip, private_mac))
|
|
except VpcError as exc:
|
|
if exc.error_code == "vpc_delete_manifest_invalid":
|
|
raise
|
|
raise VpcError(str(exc), 422,
|
|
"vpc_delete_manifest_invalid") from exc
|
|
return tuple(sorted(members, key=lambda item: item.member_id))
|
|
|
|
|
|
def parse_vpc_delete_plan(
|
|
value: Mapping[str, Any],
|
|
) -> tuple[VpcPlan, tuple[VpcDeleteMember, ...]]:
|
|
"""Parse the empty target plus its digest-covered VM cleanup manifest."""
|
|
if not isinstance(value, Mapping):
|
|
raise VpcError("VPC 删除计划必须是对象", 422,
|
|
"vpc_delete_manifest_invalid")
|
|
if "retiring_members" not in value:
|
|
raise VpcError("VPC 删除计划缺少 retiring_members", 422,
|
|
"vpc_delete_manifest_invalid")
|
|
base = dict(value)
|
|
raw_members = base.pop("retiring_members")
|
|
supplied_digest = base.pop("desired_digest", None)
|
|
plan = parse_vpc_plan(base)
|
|
members = _parse_delete_members(raw_members, plan)
|
|
canonical = plan.canonical()
|
|
canonical["retiring_members"] = _canonical_delete_members(members)
|
|
computed_digest = _canonical_digest(canonical)
|
|
if supplied_digest is not None:
|
|
supplied_digest = _digest(supplied_digest)
|
|
if supplied_digest != computed_digest:
|
|
raise VpcError("desired_digest 与 VPC 删除计划不一致", 409,
|
|
"vpc_digest_conflict")
|
|
return replace(plan, desired_digest=computed_digest), members
|
|
|
|
|
|
def _plan_from_record(value: Mapping[str, Any]) -> VpcPlan:
|
|
if not isinstance(value, Mapping):
|
|
raise VpcError("VPC 持久化记录损坏", 500, "vpc_record_corrupt")
|
|
try:
|
|
raw_members = value.get("members", [])
|
|
# Durable host rows carry only member IDs plus observed fields. The
|
|
# parser's public contract accepts full typed member objects, so
|
|
# reconstruct the grouping from the canonical member rows instead of
|
|
# feeding internal status fields back into validation.
|
|
members_by_host: dict[str, list[dict[str, Any]]] = {}
|
|
if isinstance(raw_members, list):
|
|
for member in raw_members:
|
|
if isinstance(member, Mapping) and isinstance(member.get("host_id"), str):
|
|
members_by_host.setdefault(member["host_id"], []).append(dict(member))
|
|
raw_host_rows = value.get("hosts", [])
|
|
host_ids: list[str] = []
|
|
if isinstance(raw_host_rows, list):
|
|
for host in raw_host_rows:
|
|
if (isinstance(host, Mapping)
|
|
and host.get("retiring") is not True
|
|
and isinstance(host.get("host_id"), str)):
|
|
host_ids.append(host["host_id"])
|
|
for host_id in members_by_host:
|
|
if host_id not in host_ids:
|
|
host_ids.append(host_id)
|
|
reconstructed_hosts = [
|
|
{"host_id": host_id, "members": members_by_host.get(host_id, [])}
|
|
for host_id in sorted(set(host_ids))
|
|
]
|
|
payload = {
|
|
"schema_version": value.get("schema_version", VPC_SCHEMA_VERSION),
|
|
"vpc_id": value["vpc_id"], "region": value["region"],
|
|
"fabric_id": value["fabric_id"],
|
|
"fabric_revision": value["fabric_revision"],
|
|
"cidr": value["cidr"],
|
|
"desired_revision": value["desired_revision"],
|
|
"operation_id": value.get("operation_id"),
|
|
"hosts": reconstructed_hosts, "subnets": value.get("subnets", []),
|
|
"members": None, "routes": value.get("routes", []),
|
|
"acls": value.get("acls", []),
|
|
}
|
|
except KeyError as exc:
|
|
raise VpcError("VPC 持久化记录缺少字段", 500, "vpc_record_corrupt") from exc
|
|
try:
|
|
plan = parse_vpc_plan(payload)
|
|
except VpcError as exc:
|
|
raise VpcError("VPC 持久化记录无效", 500, "vpc_record_corrupt") from exc
|
|
stored_digest = value.get("desired_digest")
|
|
delete_target = value.get("delete_target")
|
|
if delete_target is not None:
|
|
if (not isinstance(delete_target, Mapping)
|
|
or "retiring_members" not in delete_target):
|
|
raise VpcError("VPC 删除目标记录无效", 500,
|
|
"vpc_record_corrupt")
|
|
try:
|
|
retiring_members = _parse_delete_members(
|
|
delete_target["retiring_members"], plan)
|
|
except VpcError as exc:
|
|
raise VpcError("VPC 删除成员清单记录无效", 500,
|
|
"vpc_record_corrupt") from exc
|
|
canonical = plan.canonical()
|
|
canonical["retiring_members"] = _canonical_delete_members(
|
|
retiring_members)
|
|
delete_digest = _canonical_digest(canonical)
|
|
if (stored_digest != delete_digest
|
|
or delete_target.get("desired_digest") != delete_digest):
|
|
raise VpcError("VPC 删除持久化摘要不一致", 500,
|
|
"vpc_record_corrupt")
|
|
return replace(plan, desired_digest=delete_digest)
|
|
if stored_digest not in (None, plan.desired_digest):
|
|
raise VpcError("VPC 持久化摘要不一致", 500, "vpc_record_corrupt")
|
|
return plan
|
|
|
|
|
|
class VpcCoordinator:
|
|
"""Durable desired/observed VPC state with per-VPC and catalog locks."""
|
|
|
|
def __init__(self, store: Any, fabric: Any, registry: Any | None = None,
|
|
vm_lock: Any | None = None,
|
|
catalog_lock: Any | None = None):
|
|
self.store = store
|
|
self.fabric = fabric
|
|
self._catalog_lock_provider = catalog_lock
|
|
# The registry is optional for pure control-plane/unit callers. The
|
|
# production EngineApplication supplies it so a VPC revision can only
|
|
# become active after host bridge/WireGuard/nft and libvirt readback.
|
|
self.registry = registry
|
|
# EngineApplication supplies the same per-VM lock used by ordinary
|
|
# lifecycle mutations. Keeping the callback optional preserves the
|
|
# pure control-plane test harness and older embedding API.
|
|
self.vm_lock = vm_lock
|
|
# Per-VPC serialization remains independent from FabricCoordinator's
|
|
# per-Fabric locks and from VM/public network mutation locks. A VPC
|
|
# catalog mutations (apply and root delete) additionally borrow
|
|
# FabricCoordinator's process-wide admission lock so a tenant CIDR
|
|
# cannot race an endpoint migration through the two coordinators'
|
|
# opposite validation scans.
|
|
self._locks: dict[str, Any] = {}
|
|
self._fallback_catalog_lock: Any | None = None
|
|
|
|
@staticmethod
|
|
def _key(vpc_id: str, child_id: str) -> str:
|
|
return f"{vpc_id}.{child_id}"
|
|
|
|
def _lock(self, vpc_id: str):
|
|
import asyncio
|
|
return self._locks.setdefault(vpc_id, asyncio.Lock())
|
|
|
|
def _catalog_mutation_lock(self):
|
|
import asyncio
|
|
|
|
if callable(self._catalog_lock_provider):
|
|
return self._catalog_lock_provider()
|
|
# Pure control-plane embeddings and small unit fakes may not provide
|
|
# a real FabricCoordinator. They still receive local serialization;
|
|
# EngineApplication always supplies the shared production lock.
|
|
if self._fallback_catalog_lock is None:
|
|
self._fallback_catalog_lock = asyncio.Lock()
|
|
return self._fallback_catalog_lock
|
|
|
|
def _load(self, kind: str, key: str) -> dict[str, Any] | None:
|
|
raw = self.store.get(kind, key)
|
|
if raw is None:
|
|
return None
|
|
try:
|
|
value = json.loads(raw)
|
|
except (TypeError, ValueError) as exc:
|
|
raise VpcError("VPC 持久化记录损坏", 500, "vpc_record_corrupt") from exc
|
|
if not isinstance(value, dict):
|
|
raise VpcError("VPC 持久化记录损坏", 500, "vpc_record_corrupt")
|
|
if kind == "vpc_desired" and value.get("vpc_id") != key:
|
|
raise VpcError("VPC 持久化记录标识不一致", 500,
|
|
"vpc_record_corrupt")
|
|
return value
|
|
|
|
def _save(self, kind: str, key: str, value: Mapping[str, Any]) -> None:
|
|
self.store.set(kind, key, json.dumps(value, ensure_ascii=True,
|
|
separators=(",", ":")))
|
|
|
|
def _guest_exec_context(
|
|
self, plan: VpcPlan, member: VpcMember,
|
|
) -> tuple[
|
|
dict[str, Any],
|
|
Any,
|
|
Any,
|
|
]:
|
|
"""Load one private durable QGA ledger and build exact callbacks.
|
|
|
|
The record is deliberately separate from ``vpc_desired``: a PID must
|
|
be committed immediately after ``guest-exec`` returns, while rewriting
|
|
a multi-thousand-member desired document for every probe would create
|
|
quadratic database traffic. Existing VPC/instance locks serialize
|
|
these callbacks for the same member.
|
|
"""
|
|
key = self._key(plan.vpc_id, member.instance_uuid)
|
|
raw = self.store.get(VPC_GUEST_EXEC_KIND, key)
|
|
state: dict[str, Any] = {}
|
|
base = {
|
|
"vpc_id": plan.vpc_id,
|
|
"host_id": member.host_id,
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
}
|
|
if raw is not None:
|
|
if (not isinstance(raw, str)
|
|
or len(raw.encode("utf-8")) > VPC_GUEST_EXEC_MAX_BYTES):
|
|
raise VpcError("VPC 来宾执行账本损坏", 500,
|
|
"vpc_record_corrupt")
|
|
try:
|
|
value = json.loads(raw)
|
|
except (TypeError, ValueError) as exc:
|
|
raise VpcError("VPC 来宾执行账本损坏", 500,
|
|
"vpc_record_corrupt") from exc
|
|
if (not isinstance(value, Mapping)
|
|
or set(value) != set(base) | {
|
|
"version", "phase", "domain_uuid",
|
|
"domain_runtime_id", "qemu_runtime_token",
|
|
"guest_network_generation",
|
|
"probe_kind",
|
|
} | ({"pid"} if value.get("phase") == "pending" else set())
|
|
or any(value.get(field) != expected
|
|
for field, expected in base.items())):
|
|
raise VpcError("VPC 来宾执行账本损坏", 500,
|
|
"vpc_record_corrupt")
|
|
state = _guest_exec_core({
|
|
field: value[field] for field in value if field not in base
|
|
})
|
|
if state.get("domain_uuid") != member.instance_uuid:
|
|
raise VpcError("VPC 来宾执行账本损坏", 500,
|
|
"vpc_record_corrupt")
|
|
|
|
def persist(core: Mapping[str, Any]) -> None:
|
|
normalized = _guest_exec_core(core)
|
|
if normalized.get("domain_uuid") != member.instance_uuid:
|
|
raise VpcError("VPC 来宾执行账本损坏", 500,
|
|
"vpc_record_corrupt")
|
|
record = dict(base)
|
|
record.update(normalized)
|
|
encoded = json.dumps(
|
|
record, ensure_ascii=True, separators=(",", ":"))
|
|
if len(encoded.encode("utf-8")) > VPC_GUEST_EXEC_MAX_BYTES:
|
|
raise VpcError("VPC 来宾执行账本过大", 500,
|
|
"vpc_record_corrupt")
|
|
self.store.set(VPC_GUEST_EXEC_KIND, key, encoded)
|
|
|
|
def clear() -> None:
|
|
self.store.delete(VPC_GUEST_EXEC_KIND, key)
|
|
|
|
return state, persist, clear
|
|
|
|
@staticmethod
|
|
def _confirmed_removal(
|
|
host: Mapping[str, Any], revision: int, digest: str,
|
|
) -> bool:
|
|
"""Whether a durable host row is an exact absence proof.
|
|
|
|
A boolean ``removed`` marker alone is not enough to retire a host from
|
|
a later root-delete target. The proof is useful only when it belongs
|
|
to the exact durable revision/digest and every bounded tenant resource
|
|
is absent. This also accepts the historical non-``retiring`` shape
|
|
produced when PHP kept a deleting deployment in the desired host list.
|
|
"""
|
|
readback = host.get("readback")
|
|
if isinstance(readback, Mapping):
|
|
readback_state = str(
|
|
readback.get("state", readback.get("status", "")))
|
|
else:
|
|
readback_state = str(readback or "")
|
|
return bool(
|
|
host.get("removed") is True
|
|
and host.get("state") == "active"
|
|
and host.get("transport_verified") is True
|
|
and host.get("private_dataplane_verified") is True
|
|
and host.get("desired_revision") == revision
|
|
and host.get("observed_revision") == revision
|
|
and host.get("desired_digest") == digest
|
|
and host.get("observed_digest") == digest
|
|
and host.get("member_ids", []) == []
|
|
and host.get("retiring_members", []) == []
|
|
and host.get("member_nics") == []
|
|
and host.get("routes") == []
|
|
and host.get("wg_allowed_ips") == []
|
|
and host.get("proxy_ips") == []
|
|
and host.get("bridge") is None
|
|
and host.get("bridge_owner") is None
|
|
and host.get("nft_table") is None
|
|
and host.get("anti_spoof") is False
|
|
and host.get("rp_filter") is None
|
|
and host.get("proxy_arp") is None
|
|
and host.get("proxy_arp_pvlan") is None
|
|
and readback_state.lower() == "confirmed"
|
|
and host.get("rollback") in {"none", "ok"}
|
|
)
|
|
|
|
@staticmethod
|
|
def _confirmed_active_host(host: Mapping[str, Any], plan: VpcPlan) -> bool:
|
|
"""Whether this exact host revision already has complete live proof.
|
|
|
|
A request-wide probe budget is shared by every host. Retrying an
|
|
already exact host on every reconcile would let an early slow host
|
|
starve later hosts forever. Preserve only the strong same-revision
|
|
proof; any tombstone, member-set change, or incomplete guest evidence
|
|
puts the host back in the execution closure.
|
|
"""
|
|
host_id = host.get("host_id")
|
|
if not isinstance(host_id, str):
|
|
return False
|
|
expected_members = tuple(
|
|
member for member in plan.members if member.host_id == host_id)
|
|
expected_ids = {member.member_id for member in expected_members}
|
|
raw_ids = host.get("member_ids")
|
|
member_nics = host.get("member_nics")
|
|
if (not isinstance(raw_ids, list)
|
|
or any(not isinstance(value, str) for value in raw_ids)
|
|
or set(raw_ids) != expected_ids
|
|
or len(raw_ids) != len(expected_ids)
|
|
or not isinstance(member_nics, list)
|
|
or len(member_nics) != len(expected_members)):
|
|
return False
|
|
expected_by_id = {
|
|
member.member_id: member for member in expected_members
|
|
}
|
|
proven_ids: set[str] = set()
|
|
for item in member_nics:
|
|
if not isinstance(item, Mapping):
|
|
continue
|
|
member_id = item.get("member_id")
|
|
member = expected_by_id.get(member_id)
|
|
if (member is None
|
|
or item.get("instance_uuid") != member.instance_uuid
|
|
or item.get("private_ip_cidr")
|
|
!= f"{member.private_ip}/32"
|
|
or item.get("private_mac")
|
|
!= _effective_private_mac(
|
|
plan.vpc_id, member.member_id,
|
|
member.private_mac)
|
|
or item.get("config_verified") is not True
|
|
or item.get("xml_verified") is not True
|
|
or item.get("domain_running") is not True
|
|
or item.get("guest_network_verified") is not True
|
|
or item.get("restart_required") is not False):
|
|
continue
|
|
proven_ids.add(member_id)
|
|
return bool(
|
|
host.get("retiring") is not True
|
|
and host.get("retiring_members", []) == []
|
|
and host.get("state") == "active"
|
|
and host.get("transport_verified") is True
|
|
and host.get("private_dataplane_verified") is True
|
|
and host.get("config_verified") is True
|
|
and host.get("desired_revision") == plan.desired_revision
|
|
and host.get("observed_revision") == plan.desired_revision
|
|
and host.get("config_observed_revision") == plan.desired_revision
|
|
and host.get("desired_digest") == plan.desired_digest
|
|
and host.get("observed_digest") == plan.desired_digest
|
|
and host.get("config_observed_digest") == plan.desired_digest
|
|
and proven_ids == expected_ids
|
|
)
|
|
|
|
@classmethod
|
|
def _terminal_removal(cls, host: Mapping[str, Any]) -> bool:
|
|
"""Validate a self-contained historical host absence proof."""
|
|
revision = host.get("desired_revision")
|
|
digest = host.get("desired_digest")
|
|
return bool(
|
|
isinstance(revision, int)
|
|
and not isinstance(revision, bool)
|
|
and 1 <= revision <= (1 << 63) - 1
|
|
and isinstance(digest, str)
|
|
and _DIGEST.fullmatch(digest) is not None
|
|
and cls._confirmed_removal(host, revision, digest)
|
|
)
|
|
|
|
@staticmethod
|
|
def _durable_retiring_members(
|
|
host: Mapping[str, Any], *, vpc_id: str, host_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
"""Validate an unconfirmed normal-apply host tombstone fail-closed."""
|
|
raw_members = host.get("retiring_members")
|
|
raw_member_ids = host.get("member_ids")
|
|
if (not isinstance(raw_members, list)
|
|
or len(raw_members) > VPC_MAX_MEMBERS
|
|
or not isinstance(raw_member_ids, list)
|
|
or len(raw_member_ids) > VPC_MAX_MEMBERS):
|
|
raise VpcError("VPC 宿主机清理成员证据损坏", 500,
|
|
"vpc_record_corrupt")
|
|
try:
|
|
member_ids = [_identifier(item, "VPC member")
|
|
for item in raw_member_ids]
|
|
if len(member_ids) != len(set(member_ids)):
|
|
raise VpcError("VPC 宿主机清理成员标识重复")
|
|
normalized: list[dict[str, Any]] = []
|
|
seen_member_ids: set[str] = set()
|
|
seen_instances: set[str] = set()
|
|
seen_private_ips: set[str] = set()
|
|
seen_private_macs: set[str] = set()
|
|
for raw in raw_members:
|
|
if not isinstance(raw, Mapping):
|
|
raise VpcError("VPC 宿主机清理成员必须是对象")
|
|
_field_set(raw, {
|
|
"member_id", "instance_uuid", "host_id",
|
|
"private_ip_cidr", "subnet_id", "private_mac",
|
|
}, "VPC 宿主机清理成员")
|
|
if not all(field in raw for field in (
|
|
"member_id", "instance_uuid", "host_id",
|
|
"private_ip_cidr", "subnet_id")):
|
|
raise VpcError("VPC 宿主机清理成员字段不完整")
|
|
member_id = _identifier(raw["member_id"], "VPC member")
|
|
instance_uuid = _uuid(raw["instance_uuid"])
|
|
member_host_id = _host_id(raw["host_id"])
|
|
if member_host_id != host_id:
|
|
raise VpcError("VPC 宿主机清理成员归属不一致")
|
|
private_ip = _host_address(raw["private_ip_cidr"])
|
|
subnet_id = _identifier(raw["subnet_id"], "VPC subnet")
|
|
private_mac = _effective_private_mac(
|
|
vpc_id, member_id, raw.get("private_mac"))
|
|
if (member_id in seen_member_ids
|
|
or instance_uuid in seen_instances
|
|
or private_ip in seen_private_ips
|
|
or private_mac in seen_private_macs):
|
|
raise VpcError("VPC 宿主机清理成员身份重复")
|
|
seen_member_ids.add(member_id)
|
|
seen_instances.add(instance_uuid)
|
|
seen_private_ips.add(private_ip)
|
|
seen_private_macs.add(private_mac)
|
|
normalized.append({
|
|
"member_id": member_id,
|
|
"instance_uuid": instance_uuid,
|
|
"host_id": member_host_id,
|
|
"private_ip_cidr": f"{private_ip}/32",
|
|
"subnet_id": subnet_id,
|
|
"private_mac": private_mac,
|
|
})
|
|
if set(member_ids) != {
|
|
item["member_id"] for item in normalized}:
|
|
raise VpcError("VPC 宿主机清理成员闭集不一致")
|
|
except VpcError as exc:
|
|
if exc.error_code == "vpc_record_corrupt":
|
|
raise
|
|
raise VpcError("VPC 宿主机清理成员证据损坏", 500,
|
|
"vpc_record_corrupt") from exc
|
|
return normalized
|
|
|
|
@staticmethod
|
|
def _delete_plan(plan: VpcPlan) -> None:
|
|
"""Require the signed root-delete target to contain no tenant intent."""
|
|
if (plan.subnets or plan.members or plan.routes or plan.acls
|
|
or any(host.member_ids for host in plan.hosts)):
|
|
raise VpcError(
|
|
"VPC 删除目标必须清空全部子资源和宿主机成员",
|
|
409, "vpc_delete_target_not_empty")
|
|
|
|
@staticmethod
|
|
def _host_closure(record: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]:
|
|
hosts = record.get("hosts", [])
|
|
if not isinstance(hosts, list) or len(hosts) > VPC_MAX_HOSTS:
|
|
raise VpcError("VPC 删除宿主机闭集无效", 500,
|
|
"vpc_record_corrupt")
|
|
result: dict[str, Mapping[str, Any]] = {}
|
|
for host in hosts:
|
|
if not isinstance(host, Mapping):
|
|
raise VpcError("VPC 删除宿主机闭集无效", 500,
|
|
"vpc_record_corrupt")
|
|
host_id = _host_id(host.get("host_id"))
|
|
if host_id in result:
|
|
raise VpcError("VPC 删除宿主机闭集重复", 500,
|
|
"vpc_record_corrupt")
|
|
result[host_id] = host
|
|
return result
|
|
|
|
@staticmethod
|
|
def _delete_marker(record: Mapping[str, Any]) -> dict[str, Any] | None:
|
|
marker = record.get("delete_target")
|
|
if marker is None:
|
|
return None
|
|
if not isinstance(marker, dict):
|
|
raise VpcError("VPC 删除目标记录损坏", 500,
|
|
"vpc_record_corrupt")
|
|
for field in ("operation_id", "desired_revision", "desired_digest",
|
|
"all_host_ids", "required_host_ids",
|
|
"preverified_host_ids", "retiring_members"):
|
|
if field not in marker:
|
|
raise VpcError("VPC 删除目标记录不完整", 500,
|
|
"vpc_record_corrupt")
|
|
if (not isinstance(marker["operation_id"], str)
|
|
or _OPERATION_ID.fullmatch(marker["operation_id"]) is None
|
|
or isinstance(marker["desired_revision"], bool)
|
|
or not isinstance(marker["desired_revision"], int)
|
|
or marker["desired_revision"] < 1
|
|
or not isinstance(marker["desired_digest"], str)
|
|
or _DIGEST.fullmatch(marker["desired_digest"]) is None):
|
|
raise VpcError("VPC 删除目标记录无效", 500,
|
|
"vpc_record_corrupt")
|
|
parsed: dict[str, list[str]] = {}
|
|
for field in ("all_host_ids", "required_host_ids",
|
|
"preverified_host_ids"):
|
|
values = marker[field]
|
|
if (not isinstance(values, list) or len(values) > VPC_MAX_HOSTS
|
|
or values != sorted(set(values))):
|
|
raise VpcError("VPC 删除宿主机闭集无效", 500,
|
|
"vpc_record_corrupt")
|
|
try:
|
|
parsed[field] = [_host_id(value) for value in values]
|
|
except VpcError as exc:
|
|
raise VpcError("VPC 删除宿主机闭集无效", 500,
|
|
"vpc_record_corrupt") from exc
|
|
if ((set(parsed["required_host_ids"])
|
|
| set(parsed["preverified_host_ids"]))
|
|
!= set(parsed["all_host_ids"])
|
|
or set(parsed["required_host_ids"])
|
|
& set(parsed["preverified_host_ids"])):
|
|
raise VpcError("VPC 删除宿主机闭集不一致", 500,
|
|
"vpc_record_corrupt")
|
|
if (not isinstance(marker["retiring_members"], list)
|
|
or len(marker["retiring_members"]) > VPC_MAX_MEMBERS):
|
|
raise VpcError("VPC 删除成员清单记录无效", 500,
|
|
"vpc_record_corrupt")
|
|
return marker
|
|
|
|
def _load_delete_receipt(self, vpc_id: str) -> dict[str, Any] | None:
|
|
receipt = self._load(VPC_DELETE_RECEIPT_KIND, vpc_id)
|
|
if receipt is None:
|
|
return None
|
|
required = {
|
|
"vpc_id", "operation_id", "desired_revision", "desired_digest",
|
|
"state", "deleted", "readback", "removed_host_ids",
|
|
"completed_at",
|
|
}
|
|
if (not required.issubset(receipt)
|
|
or receipt.get("schema_version") != VPC_SCHEMA_VERSION
|
|
or receipt.get("vpc_id") != vpc_id
|
|
or not isinstance(receipt.get("operation_id"), str)
|
|
or _OPERATION_ID.fullmatch(receipt["operation_id"]) is None
|
|
or isinstance(receipt.get("desired_revision"), bool)
|
|
or not isinstance(receipt.get("desired_revision"), int)
|
|
or receipt["desired_revision"] < 1
|
|
or not isinstance(receipt.get("desired_digest"), str)
|
|
or _DIGEST.fullmatch(receipt["desired_digest"]) is None
|
|
or receipt.get("state") != "deleted"
|
|
or receipt.get("deleted") is not True
|
|
or receipt.get("readback") != "confirmed"
|
|
or isinstance(receipt.get("completed_at"), bool)
|
|
or not isinstance(receipt.get("completed_at"), int)
|
|
or receipt["completed_at"] < 1):
|
|
raise VpcError("VPC 删除回执损坏", 500,
|
|
"vpc_delete_receipt_corrupt")
|
|
host_ids = receipt.get("removed_host_ids")
|
|
if (not isinstance(host_ids, list) or len(host_ids) > VPC_MAX_HOSTS
|
|
or host_ids != sorted(set(host_ids))):
|
|
raise VpcError("VPC 删除回执宿主机闭集损坏", 500,
|
|
"vpc_delete_receipt_corrupt")
|
|
try:
|
|
for host_id in host_ids:
|
|
_host_id(host_id)
|
|
except VpcError as exc:
|
|
raise VpcError("VPC 删除回执宿主机闭集损坏", 500,
|
|
"vpc_delete_receipt_corrupt") from exc
|
|
# The signed desired digest already commits to the retiring-member
|
|
# closure. Once absence is confirmed, replay only needs the operation
|
|
# identity, digest, host closure and completion time. Compact an
|
|
# older receipt on first read so account deletion does not leave VM
|
|
# UUID, private IP and MAC data in the engine indefinitely.
|
|
compact = {key: receipt[key] for key in (
|
|
"schema_version", "vpc_id", "operation_id", "desired_revision",
|
|
"desired_digest", "state", "deleted", "readback",
|
|
"removed_host_ids", "completed_at",
|
|
)}
|
|
compact["evidence_version"] = int(receipt.get("evidence_version", 0))
|
|
if receipt != compact:
|
|
self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, compact)
|
|
return compact
|
|
|
|
@staticmethod
|
|
def _public_delete_receipt(receipt: Mapping[str, Any]) -> dict[str, Any]:
|
|
return {key: receipt[key] for key in (
|
|
"vpc_id", "operation_id", "evidence_version",
|
|
"desired_revision", "desired_digest",
|
|
"state", "deleted", "readback", "removed_host_ids",
|
|
"completed_at",
|
|
)}
|
|
|
|
def _delete_payload_from_record(
|
|
self, record: Mapping[str, Any],
|
|
) -> dict[str, Any]:
|
|
"""Rebuild the exact signed delete envelope for durable recovery."""
|
|
marker = self._delete_marker(record)
|
|
if marker is None:
|
|
raise VpcError("VPC 删除目标记录缺失", 500,
|
|
"vpc_record_corrupt")
|
|
plan = _plan_from_record(record)
|
|
if (plan.operation_id != marker["operation_id"]
|
|
or plan.desired_revision != marker["desired_revision"]
|
|
or plan.desired_digest != marker["desired_digest"]):
|
|
raise VpcError("VPC 删除目标记录不一致", 500,
|
|
"vpc_record_corrupt")
|
|
value = plan.canonical()
|
|
value["operation_id"] = marker["operation_id"]
|
|
value["desired_digest"] = marker["desired_digest"]
|
|
value["retiring_members"] = list(marker["retiring_members"])
|
|
return value
|
|
|
|
def _purge_deleted_vpc(self, vpc_id: str) -> None:
|
|
prefix = vpc_id + "."
|
|
for kind in ("vpc_subnet", "vpc_member", "vpc_route", "vpc_acl",
|
|
"vpc_host", VPC_GUEST_EXEC_KIND):
|
|
for key in tuple(self.store.list(kind)):
|
|
if key.startswith(prefix):
|
|
self.store.delete(kind, key)
|
|
self.store.delete("vpc_desired", vpc_id)
|
|
|
|
def _fabric_gate(self, plan: VpcPlan) -> dict[str, Any]:
|
|
try:
|
|
status = self.fabric.get_status(plan.fabric_id)
|
|
except FabricError as exc:
|
|
return {
|
|
"active": False, "available": False,
|
|
"error_code": exc.error_code,
|
|
}
|
|
except Exception:
|
|
return {"active": False, "available": False,
|
|
"error_code": "vpc_fabric_status_unavailable"}
|
|
if not isinstance(status, Mapping):
|
|
return {"active": False, "available": False,
|
|
"error_code": "vpc_fabric_status_invalid"}
|
|
if status.get("region") != plan.region:
|
|
return {
|
|
"active": False, "available": True,
|
|
"fabric_id": plan.fabric_id,
|
|
"fabric_revision": status.get("revision"),
|
|
"state": status.get("state"),
|
|
"next_stage_allowed": False,
|
|
"error_code": "vpc_fabric_region_mismatch",
|
|
}
|
|
if status.get("fabric_id") != plan.fabric_id:
|
|
return {
|
|
"active": False, "available": True,
|
|
"fabric_id": status.get("fabric_id"),
|
|
"fabric_revision": status.get("revision"),
|
|
"state": status.get("state"),
|
|
"next_stage_allowed": False,
|
|
"error_code": "vpc_fabric_identity_mismatch",
|
|
}
|
|
raw_nodes = status.get("nodes")
|
|
try:
|
|
vpc_network = ipaddress.IPv4Network(plan.cidr, strict=True)
|
|
endpoint_addresses: list[ipaddress.IPv4Address] = []
|
|
if not isinstance(raw_nodes, list):
|
|
raise ValueError("Fabric node inventory missing")
|
|
for item in raw_nodes:
|
|
if not isinstance(item, Mapping):
|
|
raise ValueError("Fabric node invalid")
|
|
endpoint = item.get("endpoint")
|
|
if not isinstance(endpoint, str):
|
|
raise ValueError("Fabric endpoint missing")
|
|
parts = endpoint.rsplit(":", 1)
|
|
if len(parts) != 2 or not parts[1].isdigit():
|
|
raise ValueError("Fabric endpoint invalid")
|
|
port = int(parts[1])
|
|
if not 1 <= port <= 65535:
|
|
raise ValueError("Fabric endpoint invalid")
|
|
endpoint_addresses.append(ipaddress.IPv4Address(parts[0]))
|
|
except (TypeError, ValueError, ipaddress.AddressValueError):
|
|
return {
|
|
"active": False, "available": False,
|
|
"error_code": "vpc_fabric_status_invalid",
|
|
}
|
|
if any(address in vpc_network for address in endpoint_addresses):
|
|
return {
|
|
"active": False, "available": True,
|
|
"fabric_id": plan.fabric_id,
|
|
"fabric_revision": status.get("revision"),
|
|
"state": status.get("state"),
|
|
"next_stage_allowed": False,
|
|
"error_code": "vpc_fabric_endpoint_overlap",
|
|
}
|
|
fabric_hosts = {
|
|
item.get("host_id") for item in raw_nodes
|
|
if isinstance(item, Mapping) and isinstance(item.get("host_id"), str)
|
|
} if isinstance(raw_nodes, list) else set()
|
|
requested_hosts = set(plan.host_ids)
|
|
raw_revision = status.get("revision")
|
|
same_revision = (
|
|
isinstance(raw_revision, int) and not isinstance(raw_revision, bool)
|
|
and raw_revision == plan.fabric_revision
|
|
)
|
|
active = (
|
|
status.get("state") == "active"
|
|
and status.get("next_stage_allowed") is True
|
|
and same_revision
|
|
and bool(plan.members)
|
|
and requested_hosts
|
|
and requested_hosts.issubset(fabric_hosts)
|
|
)
|
|
error_code = None
|
|
if not requested_hosts:
|
|
error_code = "vpc_no_participating_hosts"
|
|
elif not plan.members:
|
|
error_code = "vpc_no_members"
|
|
elif not requested_hosts.issubset(fabric_hosts):
|
|
error_code = "vpc_fabric_host_missing"
|
|
elif not same_revision:
|
|
error_code = "vpc_fabric_revision_mismatch"
|
|
elif status.get("state") != "active" or status.get("next_stage_allowed") is not True:
|
|
error_code = "vpc_fabric_not_active"
|
|
return {
|
|
"active": bool(active), "available": True,
|
|
"fabric_id": plan.fabric_id,
|
|
"fabric_revision": status.get("revision"),
|
|
"state": status.get("state"),
|
|
"next_stage_allowed": status.get("next_stage_allowed") is True,
|
|
"error_code": error_code,
|
|
}
|
|
|
|
@staticmethod
|
|
def _require_fabric_catalog_admission(gate: Mapping[str, Any]) -> None:
|
|
"""Reject deterministic Fabric/VPC identity and address conflicts."""
|
|
error_code = gate.get("error_code")
|
|
if error_code in VPC_FABRIC_ADMISSION_ERRORS:
|
|
raise VpcError(
|
|
"VPC 与 Fabric 的地区、身份或 underlay 地址冲突",
|
|
409, str(error_code))
|
|
|
|
def _validate_global_fabric_endpoints(self, plan: VpcPlan) -> None:
|
|
"""Keep every durable Fabric underlay outside the new tenant CIDR."""
|
|
tenant_network = ipaddress.IPv4Network(plan.cidr, strict=True)
|
|
for record_key, raw in self.store.list("fabric_desired").items():
|
|
try:
|
|
value = json.loads(raw)
|
|
if (not isinstance(value, Mapping)
|
|
or value.get("fabric_id") != record_key):
|
|
raise ValueError("Fabric identity mismatch")
|
|
fabric_plan = FabricCoordinator._plan_from_record(value)
|
|
if value.get("plan_digest") != fabric_plan.digest:
|
|
raise ValueError("Fabric plan digest mismatch")
|
|
except (TypeError, ValueError, FabricError, KeyError) as exc:
|
|
raise VpcError(
|
|
"Fabric 资源目录损坏,无法确认全局 underlay 地址",
|
|
500, "vpc_fabric_record_corrupt") from exc
|
|
for node in fabric_plan.nodes:
|
|
try:
|
|
endpoint = ipaddress.IPv4Address(
|
|
node.endpoint.rsplit(":", 1)[0])
|
|
except (ValueError, ipaddress.AddressValueError) as exc:
|
|
raise VpcError(
|
|
"Fabric endpoint 记录损坏",
|
|
500, "vpc_fabric_record_corrupt") from exc
|
|
if endpoint in tenant_network:
|
|
raise VpcError(
|
|
"VPC CIDR 与现有 Fabric underlay endpoint 冲突",
|
|
409, "vpc_fabric_endpoint_overlap")
|
|
|
|
def _fabric_record(self, plan: VpcPlan) -> dict[str, Any]:
|
|
"""Load and validate the durable Fabric identity used by the data plane."""
|
|
loader = getattr(self.fabric, "_load", None)
|
|
raw = loader("fabric_desired", plan.fabric_id) if callable(loader) else None
|
|
if not isinstance(raw, Mapping):
|
|
raise VpcError("VPC Fabric 记录不可用", 409,
|
|
"vpc_fabric_record_unavailable")
|
|
try:
|
|
fabric_plan = self.fabric._plan_from_record(raw)
|
|
except Exception as exc:
|
|
raise VpcError("VPC Fabric 记录无效", 500,
|
|
"vpc_fabric_record_corrupt") from exc
|
|
if (fabric_plan.fabric_id != plan.fabric_id
|
|
or fabric_plan.region != plan.region
|
|
or fabric_plan.revision != plan.fabric_revision):
|
|
raise VpcError("VPC Fabric revision 或地区不匹配", 409,
|
|
"vpc_fabric_revision_mismatch")
|
|
if raw.get("plan_digest") != fabric_plan.digest:
|
|
raise VpcError("VPC Fabric 计划摘要无效", 500,
|
|
"vpc_fabric_record_corrupt")
|
|
return dict(raw)
|
|
|
|
def _validate_dataplane_resources(self, plan: VpcPlan) -> None:
|
|
"""Reject deterministic resource collisions before touching a host."""
|
|
from .vpc_dataplane import resource_names
|
|
|
|
current = resource_names(plan.vpc_id)
|
|
seen: dict[str, str] = {}
|
|
for field in ("bridge", "nft_table", "route_table"):
|
|
seen[str(getattr(current, field))] = plan.vpc_id
|
|
for priority in (current.rule_priority, current.rule_priority + 1):
|
|
seen[f"rule:{priority}"] = plan.vpc_id
|
|
for key, raw in self.store.list("vpc_desired").items():
|
|
if key == plan.vpc_id:
|
|
continue
|
|
try:
|
|
other = _plan_from_record(json.loads(raw))
|
|
except (TypeError, ValueError, VpcError) as exc:
|
|
raise VpcError("VPC 资源目录损坏", 500,
|
|
"vpc_record_corrupt") from exc
|
|
resources = resource_names(other.vpc_id)
|
|
for field in ("bridge", "nft_table", "route_table"):
|
|
marker = str(getattr(resources, field))
|
|
owner = seen.get(marker)
|
|
if owner is not None and owner != other.vpc_id:
|
|
raise VpcError("VPC 数据面资源发生冲突", 409,
|
|
"vpc_dataplane_resource_conflict")
|
|
seen[marker] = other.vpc_id
|
|
for priority in (resources.rule_priority, resources.rule_priority + 1):
|
|
marker = f"rule:{priority}"
|
|
owner = seen.get(marker)
|
|
if owner is not None and owner != other.vpc_id:
|
|
raise VpcError("VPC 数据面资源发生冲突", 409,
|
|
"vpc_dataplane_resource_conflict")
|
|
seen[marker] = other.vpc_id
|
|
|
|
def _fabric_peers_for_plan(
|
|
self, plan: VpcPlan, fabric_record: Mapping[str, Any],
|
|
) -> list[dict[str, Any]]:
|
|
"""Build per-peer overlay plus tenant /32 AllowedIPs.
|
|
|
|
WireGuard has one AllowedIPs list per peer. Every VPC sharing a
|
|
Fabric therefore contributes only its exact member /32s to the peer
|
|
on the member's host. Duplicate addresses are rejected across VPCs
|
|
before a host command is rendered.
|
|
"""
|
|
from .vpc_dataplane import _key
|
|
|
|
plans: list[VpcPlan] = [plan]
|
|
for key, raw in self.store.list("vpc_desired").items():
|
|
if key == plan.vpc_id:
|
|
continue
|
|
try:
|
|
candidate = _plan_from_record(json.loads(raw))
|
|
except (TypeError, ValueError, VpcError) as exc:
|
|
raise VpcError("VPC 资源目录损坏", 500,
|
|
"vpc_record_corrupt") from exc
|
|
if (candidate.fabric_id == plan.fabric_id
|
|
and candidate.region == plan.region):
|
|
plans.append(candidate)
|
|
address_owner: dict[str, str] = {}
|
|
tenant_by_host: dict[str, set[str]] = {}
|
|
for candidate in plans:
|
|
for member in candidate.members:
|
|
cidr = f"{member.private_ip}/32"
|
|
owner = address_owner.get(cidr)
|
|
if owner is not None and owner != candidate.vpc_id:
|
|
raise VpcError("VPC 私网地址跨 VPC 重复", 409,
|
|
"vpc_private_ip_conflict")
|
|
address_owner[cidr] = candidate.vpc_id
|
|
tenant_by_host.setdefault(member.host_id, set()).add(cidr)
|
|
# A custom service route is cryptokey-routed to the host that
|
|
# owns its next hop. It must be part of that peer's exact
|
|
# AllowedIPs set just like a VM /32; otherwise WireGuard sends it
|
|
# to no peer and the Fabric quarantine drops it.
|
|
members_by_id = {item.member_id: item for item in candidate.members}
|
|
for route in candidate.routes:
|
|
target = (members_by_id.get(route.next_hop_member_id)
|
|
if route.next_hop_member_id is not None else None)
|
|
target_host = target.host_id if target is not None else route.next_hop_host_id
|
|
if target_host is None:
|
|
raise VpcError("VPC route next hop 缺失", 409,
|
|
"vpc_route_next_hop_missing")
|
|
cidr = route.destination_cidr
|
|
owner = address_owner.get(cidr)
|
|
if owner is not None and owner != candidate.vpc_id:
|
|
raise VpcError("VPC 路由地址跨 VPC 重复", 409,
|
|
"vpc_route_address_conflict")
|
|
address_owner[cidr] = candidate.vpc_id
|
|
tenant_by_host.setdefault(target_host, set()).add(cidr)
|
|
|
|
raw_nodes = fabric_record.get("nodes")
|
|
raw_keys = fabric_record.get("public_keys")
|
|
if not isinstance(raw_nodes, list) or not isinstance(raw_keys, Mapping):
|
|
raise VpcError("Fabric peer 记录不完整", 500,
|
|
"vpc_fabric_record_corrupt")
|
|
peers: list[dict[str, Any]] = []
|
|
wanted_hosts = set(plan.host_ids)
|
|
for raw in raw_nodes:
|
|
if not isinstance(raw, Mapping):
|
|
continue
|
|
host_id = raw.get("host_id")
|
|
if not isinstance(host_id, str) or host_id not in wanted_hosts:
|
|
continue
|
|
# The caller supplies the local host separately when parsing each
|
|
# host plan; keep every other participating host in this list.
|
|
public_key = raw_keys.get(host_id)
|
|
overlay_ip = raw.get("overlay_ip")
|
|
if not isinstance(public_key, str) or not isinstance(overlay_ip, str):
|
|
raise VpcError("Fabric peer 记录无效", 500,
|
|
"vpc_fabric_record_corrupt")
|
|
_key(public_key)
|
|
peers.append({
|
|
"host_id": host_id,
|
|
"public_key": public_key,
|
|
"overlay_ip": overlay_ip,
|
|
"tenant_allowed_ips": sorted(tenant_by_host.get(host_id, set())),
|
|
})
|
|
return peers
|
|
|
|
@staticmethod
|
|
def _private_routes(plan: VpcPlan, member: VpcMember) -> list[str]:
|
|
from .vpc_dataplane import private_guest_routes
|
|
|
|
return list(private_guest_routes(plan, member))
|
|
|
|
@staticmethod
|
|
def _observation_error(exc: Exception) -> str:
|
|
code = getattr(exc, "error_code", None)
|
|
if isinstance(code, str) and re.fullmatch(r"[a-z0-9_]{1,64}", code):
|
|
return code
|
|
return "vpc_private_dataplane_apply_failed"
|
|
|
|
def _run_host_dataplane(
|
|
self, plan: VpcPlan, host_id: str, component: Any,
|
|
profile: Mapping[str, Any], fabric_peers: list[dict[str, Any]],
|
|
previous_plan: VpcPlan | None,
|
|
retiring_members: Sequence[Mapping[str, Any]] | None = None,
|
|
fabric_public_keys: Mapping[str, str] | None = None,
|
|
fabric_plan_digest: str | None = None,
|
|
private_link_states: Mapping[str, str] | None = None,
|
|
request_deadline: float | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Apply and read back one host synchronously behind the VPC lock."""
|
|
from .libvirt_service import VMService
|
|
from .vpc_dataplane import (
|
|
VpcDataplanePlan, VpcPrivateHostAdapter,
|
|
parse_vpc_dataplane_plan, resource_names,
|
|
)
|
|
|
|
if host_id in plan.host_ids:
|
|
payload = plan.canonical()
|
|
payload["fabric_peers"] = [
|
|
peer for peer in fabric_peers
|
|
if peer.get("host_id") != host_id
|
|
]
|
|
trusted_peer_tenant_ips = {
|
|
str(peer["host_id"]): tuple(peer.get("tenant_allowed_ips", ()))
|
|
for peer in payload["fabric_peers"]
|
|
if isinstance(peer, Mapping) and isinstance(peer.get("host_id"), str)
|
|
}
|
|
authoritative_peer_keys = None
|
|
if fabric_public_keys is not None:
|
|
authoritative_peer_keys = [
|
|
key for peer_host, key in fabric_public_keys.items()
|
|
if peer_host != host_id
|
|
]
|
|
dataplane = parse_vpc_dataplane_plan(
|
|
payload, host_id,
|
|
trusted_tenant_allowed_ips=trusted_peer_tenant_ips,
|
|
fabric_peer_keys=authoritative_peer_keys,
|
|
fabric_plan_digest=fabric_plan_digest)
|
|
# A root-delete plan extends the canonical digest with its signed
|
|
# retiring-member manifest. The host projection intentionally
|
|
# omits those members from live desired state, so restore the
|
|
# authoritative envelope identity after parsing that projection.
|
|
dataplane = replace(dataplane, vpc=plan)
|
|
else:
|
|
# A retiring host is intentionally absent from the new desired
|
|
# plan. Construct a typed empty-local plan directly so the
|
|
# removal transaction can validate the deterministic VPC
|
|
# resources without reintroducing the old host into Fabric.
|
|
dataplane = VpcDataplanePlan(
|
|
plan.vpc, host_id, tuple(plan.members), tuple(),
|
|
resource_names(plan.vpc_id),
|
|
fabric_plan_digest)
|
|
if request_deadline is None:
|
|
request_deadline = (
|
|
time.monotonic() + VPC_REQUEST_DEADLINE_SECONDS)
|
|
adapter = VpcPrivateHostAdapter(
|
|
component.manager, profile, request_deadline=request_deadline)
|
|
service = VMService(
|
|
profile["libvirt_uri"], profile["storage_pool"],
|
|
profile["iso_pool"], profile["default_network"],
|
|
rescue_iso_pool=profile.get("rescue_iso_pool", ""),
|
|
)
|
|
guest_exec_contexts: dict[str, tuple[dict[str, Any], Any, Any]] = {}
|
|
|
|
def guest_exec_context(
|
|
member: VpcMember,
|
|
) -> tuple[dict[str, Any], Any, Any]:
|
|
context = guest_exec_contexts.get(member.instance_uuid)
|
|
if context is None:
|
|
context = self._guest_exec_context(plan, member)
|
|
guest_exec_contexts[member.instance_uuid] = context
|
|
return context
|
|
|
|
def require_request_budget() -> None:
|
|
if time.monotonic() >= request_deadline:
|
|
raise VpcError(
|
|
"VPC 来宾网络核验超过请求预算", 502,
|
|
"vpc_private_guest_probe_deadline_exceeded")
|
|
|
|
# A lifecycle-triggered reconcile must reach its target before a
|
|
# large host's shared probe budget expires. Mapping membership is the
|
|
# priority signal even for ``preserve``; ordinary periodic passes have
|
|
# no entries and retain canonical member order.
|
|
priority_instances = set(private_link_states or {})
|
|
local_members = tuple(sorted(
|
|
dataplane.local_members,
|
|
key=lambda item: item.instance_uuid not in priority_instances,
|
|
))
|
|
|
|
old_members: list[VpcMember] = []
|
|
if previous_plan is not None:
|
|
old_members.extend(
|
|
member for member in previous_plan.members
|
|
if member.host_id == host_id)
|
|
if retiring_members is not None:
|
|
for raw in retiring_members:
|
|
if not isinstance(raw, Mapping):
|
|
continue
|
|
try:
|
|
private_ip = str(raw.get("private_ip_cidr", "")).removesuffix("/32")
|
|
old_members.append(VpcMember(
|
|
str(raw["member_id"]), str(raw["instance_uuid"]),
|
|
str(raw.get("host_id", host_id)), private_ip,
|
|
str(raw.get("subnet_id", "retiring")),
|
|
str(raw.get("private_mac", ""))))
|
|
except (KeyError, TypeError, ValueError):
|
|
raise VpcError("VPC 宿主机清理成员证据无效", 500,
|
|
"vpc_retiring_member_invalid")
|
|
current_ids = {member.member_id for member in plan.members}
|
|
cleanup_plan = previous_plan or plan
|
|
seen_old: set[str] = set()
|
|
for old_member in old_members:
|
|
if old_member.member_id in seen_old:
|
|
continue
|
|
seen_old.add(old_member.member_id)
|
|
if old_member.host_id != host_id:
|
|
continue
|
|
if old_member.member_id in current_ids:
|
|
continue
|
|
require_request_budget()
|
|
ledger_state, _persist_ledger, clear_ledger = (
|
|
guest_exec_context(old_member))
|
|
service.remove_private_interface(
|
|
old_member.instance_uuid,
|
|
vpc_id=cleanup_plan.vpc_id,
|
|
resources=resource_names(cleanup_plan.vpc_id),
|
|
desired_revision=cleanup_plan.desired_revision,
|
|
desired_digest=cleanup_plan.desired_digest,
|
|
expected_member_id=old_member.member_id,
|
|
expected_private_mac=_effective_private_mac(
|
|
cleanup_plan.vpc_id, old_member.member_id,
|
|
old_member.private_mac),
|
|
expected_private_ip_cidr=f"{old_member.private_ip}/32",
|
|
)
|
|
# remove_private_interface positively requires a stopped domain.
|
|
# Its old QGA process therefore cannot retain either a launch
|
|
# intent or PID record. Delete the ledger only after that proof.
|
|
clear_ledger()
|
|
ledger_state.clear()
|
|
# A host with no local members is a removal target. Run the
|
|
# manager-aware teardown and require its positive absence marker;
|
|
# returning a normal empty observation would leave stale bridge,
|
|
# route, nft, and peer state behind.
|
|
if not local_members:
|
|
require_request_budget()
|
|
removed = adapter.remove(dataplane)
|
|
removed.update({
|
|
"vpc_id": plan.vpc_id,
|
|
"host_id": host_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_revision": plan.desired_revision,
|
|
"observed_digest": plan.desired_digest,
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"rollback": removed.get("rollback", "none"),
|
|
})
|
|
return removed
|
|
require_request_budget()
|
|
adapter.apply(dataplane)
|
|
pairs: dict[str, dict[str, Any]] = {}
|
|
probe_members: list[VpcMember] = []
|
|
for member in local_members:
|
|
require_request_budget()
|
|
ledger_state, _persist_ledger, _clear_ledger = (
|
|
guest_exec_context(member))
|
|
configured = service.ensure_private_interface(
|
|
member.instance_uuid, member=member,
|
|
resources=dataplane.resources, vpc_id=plan.vpc_id,
|
|
desired_revision=plan.desired_revision,
|
|
desired_digest=plan.desired_digest,
|
|
private_routes=self._private_routes(plan, member),
|
|
link_state=(private_link_states or {}).get(
|
|
member.instance_uuid, "preserve"),
|
|
)
|
|
pair = {
|
|
"inactive": configured.get("xml"),
|
|
"live": configured.get("live"),
|
|
"seed_verified": configured.get("seed_verified") is True,
|
|
}
|
|
if configured.get("live") is None:
|
|
pair["guest_network"] = {
|
|
"state": "stopped", "verified": False,
|
|
"reason_code": "vpc_private_guest_stopped",
|
|
"observed_at": int(time.time()),
|
|
}
|
|
if ledger_state:
|
|
probe_members.append(member)
|
|
elif configured.get("restart_required") is True:
|
|
pair["guest_network"] = {
|
|
"state": "pending_restart", "verified": False,
|
|
"reason_code": "vpc_private_guest_restart_required",
|
|
"observed_at": int(time.time()),
|
|
}
|
|
if ledger_state:
|
|
probe_members.append(member)
|
|
else:
|
|
# This placeholder is deliberately fail-closed. A completed
|
|
# QGA future replaces it; an unsubmitted/cancelled future
|
|
# remains explicit deadline evidence while preserving the
|
|
# persistent/live XML already read back by the transaction.
|
|
pair["guest_network"] = {
|
|
"state": "deadline_exceeded", "verified": False,
|
|
"reason_code":
|
|
"vpc_private_guest_probe_deadline_exceeded",
|
|
"observed_at": int(time.time()),
|
|
}
|
|
probe_members.append(member)
|
|
pairs[member.instance_uuid] = pair
|
|
|
|
def read_member(member: VpcMember) -> tuple[VpcMember, dict[str, Any]]:
|
|
ledger_state, persist_ledger, clear_ledger = (
|
|
guest_exec_context(member))
|
|
configured = pairs.get(member.instance_uuid, {})
|
|
|
|
def deadline_pair() -> dict[str, Any]:
|
|
return {
|
|
"inactive": configured.get("inactive"),
|
|
"live": configured.get("live"),
|
|
"seed_verified": bool(configured.get("seed_verified")),
|
|
"guest_network": {
|
|
"state": "deadline_exceeded", "verified": False,
|
|
"reason_code":
|
|
"vpc_private_guest_probe_deadline_exceeded",
|
|
"observed_at": int(time.time()),
|
|
},
|
|
}
|
|
|
|
remaining = max(0.0, request_deadline - time.monotonic())
|
|
acquired = _VPC_GUEST_PROBE_SLOTS.acquire(timeout=remaining)
|
|
if not acquired:
|
|
return member, deadline_pair()
|
|
try:
|
|
# QGA rounds its command timeout to whole seconds. Do not
|
|
# enter libvirt after waiting for a slot if less than one
|
|
# useful command second remains in the shared request budget.
|
|
if request_deadline - time.monotonic() < 1:
|
|
return member, deadline_pair()
|
|
pair = service.read_private_domain_pair(
|
|
member.instance_uuid,
|
|
private_mac=member.private_mac,
|
|
private_ip_cidr=f"{member.private_ip}/32",
|
|
private_routes=self._private_routes(plan, member),
|
|
guest_probe_deadline=request_deadline,
|
|
guest_exec_state=ledger_state,
|
|
guest_exec_persist=persist_ledger,
|
|
guest_exec_clear=clear_ledger,
|
|
)
|
|
pair["seed_verified"] = bool(
|
|
configured.get("seed_verified"))
|
|
return member, pair
|
|
finally:
|
|
_VPC_GUEST_PROBE_SLOTS.release()
|
|
|
|
# Keep at most one worker-width of QGA calls submitted. This avoids a
|
|
# 4,096-future queue, while the process-wide semaphore prevents pools
|
|
# belonging to concurrent VPC requests from multiplying that width.
|
|
# Both gates share the request deadline, and every unstarted member is
|
|
# explicit fail-closed evidence. Shutdown waits for bounded calls, so
|
|
# no probe thread survives this request.
|
|
if probe_members:
|
|
workers = min(VPC_GUEST_PROBE_WORKERS, len(probe_members))
|
|
executor = ThreadPoolExecutor(
|
|
max_workers=workers,
|
|
thread_name_prefix="vpc-guest-probe")
|
|
futures: dict[Any, VpcMember] = {}
|
|
next_member = 0
|
|
probe_error: BaseException | None = None
|
|
|
|
def submit_available() -> None:
|
|
nonlocal next_member
|
|
while (len(futures) < workers
|
|
and next_member < len(probe_members)
|
|
and request_deadline - time.monotonic() >= 1):
|
|
member = probe_members[next_member]
|
|
next_member += 1
|
|
futures[executor.submit(read_member, member)] = member
|
|
|
|
try:
|
|
submit_available()
|
|
while futures:
|
|
remaining = request_deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
break
|
|
done, _pending = wait(
|
|
tuple(futures), timeout=remaining,
|
|
return_when=FIRST_COMPLETED)
|
|
if not done:
|
|
break
|
|
for future in done:
|
|
futures.pop(future, None)
|
|
try:
|
|
member, pair = future.result()
|
|
except BaseException as exc:
|
|
probe_error = exc
|
|
break
|
|
pairs[member.instance_uuid] = pair
|
|
if probe_error is not None:
|
|
break
|
|
submit_available()
|
|
finally:
|
|
for future in futures:
|
|
future.cancel()
|
|
executor.shutdown(wait=True, cancel_futures=True)
|
|
if probe_error is not None:
|
|
raise probe_error
|
|
|
|
xml_by_instance: dict[str, Mapping[str, Any]] = {}
|
|
for member in local_members:
|
|
pair = pairs[member.instance_uuid]
|
|
inactive = pair.get("inactive")
|
|
if not isinstance(inactive, str):
|
|
raise VpcError("私网网卡持久配置回读缺失", 502,
|
|
"vpc_private_xml_readback_missing")
|
|
live = pair.get("live")
|
|
if live is not None and not isinstance(live, str):
|
|
raise VpcError("私网网卡实时配置回读无效", 502,
|
|
"vpc_private_live_readback_missing")
|
|
xml_by_instance[member.instance_uuid] = {
|
|
"inactive": inactive, "live": live,
|
|
"guest_network": pair.get("guest_network"),
|
|
"seed_verified": pair.get("seed_verified") is True,
|
|
}
|
|
observed = adapter.observe(dataplane, xml_by_instance=xml_by_instance)
|
|
observed.update({
|
|
"vpc_id": plan.vpc_id,
|
|
"host_id": host_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": observed.get("state") == "active",
|
|
"rollback": observed.get("rollback", "none"),
|
|
})
|
|
return observed
|
|
|
|
@asynccontextmanager
|
|
async def _instance_locks(
|
|
self, plan: VpcPlan, previous_plan: VpcPlan | None = None,
|
|
extra_identities: Sequence[tuple[str, str]] | None = None,
|
|
):
|
|
if not callable(self.vm_lock):
|
|
yield
|
|
return
|
|
identities = {
|
|
(member.host_id, member.instance_uuid)
|
|
for member in plan.members
|
|
}
|
|
if previous_plan is not None:
|
|
identities.update(
|
|
(member.host_id, member.instance_uuid)
|
|
for member in previous_plan.members
|
|
)
|
|
# A removed member is no longer present in either VPC plan. Its
|
|
# durable retiring_members row is still a live libvirt target, so the
|
|
# cleanup pass must share the ordinary VM lifecycle lock as well.
|
|
if extra_identities is not None:
|
|
for host_id, instance_uuid in extra_identities:
|
|
if (isinstance(host_id, str)
|
|
and isinstance(instance_uuid, str)):
|
|
identities.add((host_id, instance_uuid))
|
|
async with AsyncExitStack() as stack:
|
|
for host_id, instance_uuid in sorted(identities):
|
|
await stack.enter_async_context(
|
|
self.vm_lock(host_id, instance_uuid))
|
|
yield
|
|
|
|
def _lifecycle_conflict(
|
|
self, host_id: str, instance_uuid: str,
|
|
operation_id: str | None = None,
|
|
) -> str | None:
|
|
"""Return a fail-closed reason while a host-local VM job owns XML.
|
|
|
|
Reinstall, rescue and online-snapshot workers are queued by the child
|
|
host app and continue after the HTTP operation has returned 202. The
|
|
regular target lock therefore cannot cover their whole physical
|
|
mutation. Read their durable rows before a VPC pass touches libvirt;
|
|
an uncertain row is deliberately treated as active until an operator
|
|
reconciles it.
|
|
"""
|
|
registry = self.registry
|
|
if registry is None:
|
|
# Pure control-plane callers intentionally have no host job store;
|
|
# they cannot race a libvirt worker and keep the historical unit
|
|
# harness behavior.
|
|
return None
|
|
components = getattr(registry, "components", {}).get(host_id)
|
|
store = getattr(components, "store", None)
|
|
if store is None or not callable(getattr(store, "list", None)):
|
|
return "vpc_lifecycle_state_unavailable"
|
|
active_states = {"queued", "running", "pending", "uncertain"}
|
|
try:
|
|
same_uncertain_snapshots: set[tuple[str, str]] = set()
|
|
same_uncertain_rescue = False
|
|
for kind in ("reinstall_job", "rescue_job", "snapshot_job"):
|
|
rows = store.list(kind)
|
|
if not isinstance(rows, Mapping):
|
|
return "vpc_lifecycle_state_unavailable"
|
|
for raw in rows.values():
|
|
try:
|
|
job = json.loads(raw)
|
|
except (TypeError, ValueError, UnicodeError):
|
|
return "vpc_lifecycle_state_unavailable"
|
|
if not isinstance(job, Mapping):
|
|
return "vpc_lifecycle_state_unavailable"
|
|
same_uncertain_operation = (
|
|
job.get("status") == "uncertain"
|
|
and operation_id is not None
|
|
and job.get("operation_id") == operation_id
|
|
and job.get("vm_id") == instance_uuid
|
|
and (
|
|
(kind == "snapshot_job"
|
|
and isinstance(job.get("snapshot_key"), str))
|
|
or (kind == "reinstall_job"
|
|
and _reinstall_job_committed(job))
|
|
)
|
|
)
|
|
# A rescue job's uncertain row is not evidence by itself.
|
|
# Defer this one same-operation case until the vm_rescue
|
|
# state below proves that the rescue XML reached its
|
|
# active/staged terminal state.
|
|
if (kind == "rescue_job"
|
|
and job.get("status") == "uncertain"
|
|
and operation_id is not None
|
|
and job.get("operation_id") == operation_id
|
|
and job.get("vm_id") == instance_uuid):
|
|
same_uncertain_rescue = True
|
|
continue
|
|
# An uncertain reinstall is exempt only after its
|
|
# durable post-commit marker proves the replacement
|
|
# domain exists. Rescue jobs have no generic exemption;
|
|
# their vm_rescue active/staged row below is the only
|
|
# same-operation terminal proof.
|
|
if (kind == "snapshot_job" and same_uncertain_operation
|
|
and job.get("vm_id") == instance_uuid
|
|
and isinstance(job.get("snapshot_key"), str)):
|
|
# The catalog row has no operation_id of its own; the
|
|
# durable snapshot job is the identity that proves a
|
|
# lifecycle worker is reconciling this exact row.
|
|
same_uncertain_snapshots.add(
|
|
(instance_uuid, job["snapshot_key"])
|
|
)
|
|
if (job.get("vm_id") == instance_uuid
|
|
and job.get("status") in active_states
|
|
and not same_uncertain_operation):
|
|
return "vpc_vm_lifecycle_conflict"
|
|
|
|
# Rescue mode has a durable state row even after its worker is no
|
|
# longer queued. Restoring the original XML while VPC apply is in
|
|
# flight would otherwise silently remove private0 again.
|
|
rescue_rows = store.list("vm_rescue")
|
|
if not isinstance(rescue_rows, Mapping):
|
|
return "vpc_lifecycle_state_unavailable"
|
|
same_terminal_rescue = False
|
|
for key, raw in rescue_rows.items():
|
|
try:
|
|
state = json.loads(raw)
|
|
except (TypeError, ValueError, UnicodeError):
|
|
return "vpc_lifecycle_state_unavailable"
|
|
if not isinstance(state, Mapping):
|
|
return "vpc_lifecycle_state_unavailable"
|
|
key_matches = isinstance(key, str) and key.endswith(
|
|
"." + instance_uuid)
|
|
if state.get("vm_id") == instance_uuid or key_matches:
|
|
state_status = state.get("status")
|
|
# Enter/force rescue finishes the libvirt XML mutation
|
|
# before its lifecycle worker asks for the private
|
|
# readback. Keep the durable active/staged marker as
|
|
# protection for every other operation, but let the
|
|
# authenticated rescue operation reconcile the terminal
|
|
# state it just created. Queued/running/exiting/failed
|
|
# rows remain blocking even when their operation id
|
|
# matches: those states have not proved that the XML
|
|
# mutation is complete (or need manual recovery).
|
|
same_terminal = (
|
|
state_status in {"active", "staged"}
|
|
and operation_id is not None
|
|
and state.get("operation_id") == operation_id
|
|
)
|
|
if same_terminal:
|
|
same_terminal_rescue = True
|
|
elif state_status not in {"completed", "deleted"}:
|
|
return "vpc_vm_lifecycle_conflict"
|
|
if same_uncertain_rescue and not same_terminal_rescue:
|
|
return "vpc_vm_lifecycle_conflict"
|
|
|
|
# A transitional snapshot catalog row without its job row is a
|
|
# durable indication that the previous process died at a physical
|
|
# boundary. Do not overwrite its domain XML until reconciliation.
|
|
snapshot_rows = store.list("vm_snapshot")
|
|
if not isinstance(snapshot_rows, Mapping):
|
|
return "vpc_lifecycle_state_unavailable"
|
|
for _key, raw in snapshot_rows.items():
|
|
try:
|
|
snapshot = json.loads(raw)
|
|
except (TypeError, ValueError, UnicodeError):
|
|
return "vpc_lifecycle_state_unavailable"
|
|
if (isinstance(snapshot, Mapping)
|
|
and snapshot.get("vm_id") == instance_uuid
|
|
and snapshot.get("state") in {
|
|
"creating", "restoring", "deleting", "uncertain",
|
|
}
|
|
and (snapshot.get("vm_id"),
|
|
snapshot.get("snapshot_key"))
|
|
not in same_uncertain_snapshots):
|
|
return "vpc_vm_lifecycle_conflict"
|
|
except Exception:
|
|
# A database/host-store read failure must not be interpreted as an
|
|
# empty job set, because that could race a destructive worker.
|
|
return "vpc_lifecycle_state_unavailable"
|
|
return None
|
|
|
|
def _mark_lifecycle_conflicts(
|
|
self, plan: VpcPlan, previous_plan: VpcPlan | None,
|
|
record: dict[str, Any], operation_id: str | None = None,
|
|
) -> bool:
|
|
identities: dict[tuple[str, str], None] = {}
|
|
for candidate in (plan.members
|
|
+ (previous_plan.members if previous_plan else ())):
|
|
identities[(candidate.host_id, candidate.instance_uuid)] = None
|
|
for host in record.get("hosts", []):
|
|
if not isinstance(host, Mapping):
|
|
continue
|
|
host_id = host.get("host_id")
|
|
for row in host.get("retiring_members", []):
|
|
if (isinstance(host_id, str) and isinstance(row, Mapping)
|
|
and isinstance(row.get("instance_uuid"), str)):
|
|
identities[(host_id, row["instance_uuid"])] = None
|
|
conflict = False
|
|
reasons_by_host: dict[str, str] = {}
|
|
for (host_id, instance_uuid) in identities:
|
|
reason = self._lifecycle_conflict(
|
|
host_id, instance_uuid, operation_id)
|
|
if reason is not None:
|
|
conflict = True
|
|
reasons_by_host[host_id] = reason
|
|
if not conflict:
|
|
return False
|
|
for host in record.get("hosts", []):
|
|
if not isinstance(host, dict):
|
|
continue
|
|
host_id = host.get("host_id")
|
|
reason = reasons_by_host.get(host_id)
|
|
if reason is not None:
|
|
host.update({
|
|
"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"last_error_code": reason,
|
|
})
|
|
return True
|
|
|
|
async def _reconcile_dataplane(
|
|
self, plan: VpcPlan, record: dict[str, Any], gate: Mapping[str, Any],
|
|
previous_plan: VpcPlan | None = None,
|
|
private_link_states: Mapping[str, str] | None = None,
|
|
extra_identities: Sequence[tuple[str, str]] | None = None,
|
|
operation_id: str | None = None,
|
|
) -> None:
|
|
# VPC lock is acquired by each public caller before this method. The
|
|
# VM locks are acquired in stable order here, so a VPC apply cannot
|
|
# race a normal lifecycle mutation or a targeted private reconcile.
|
|
# Derive every durable tombstone identity as well as any explicit
|
|
# target supplied by a lifecycle caller. A host pass removes all
|
|
# retiring members in one transaction; locking only the requested VM
|
|
# would leave its sibling tombstones exposed to delete/reinstall.
|
|
derived: list[tuple[str, str]] = []
|
|
for host in record.get("hosts", []):
|
|
if not isinstance(host, Mapping):
|
|
continue
|
|
host_id = host.get("host_id")
|
|
if not isinstance(host_id, str):
|
|
continue
|
|
for row in host.get("retiring_members", []):
|
|
if (isinstance(row, Mapping)
|
|
and isinstance(row.get("instance_uuid"), str)):
|
|
derived.append((host_id, row["instance_uuid"]))
|
|
if extra_identities is None:
|
|
extra_identities = derived
|
|
else:
|
|
extra_identities = tuple(extra_identities) + tuple(derived)
|
|
async with self._instance_locks(
|
|
plan, previous_plan, extra_identities):
|
|
kwargs: dict[str, Any] = {}
|
|
if private_link_states is not None:
|
|
kwargs["private_link_states"] = private_link_states
|
|
if operation_id is not None:
|
|
kwargs["operation_id"] = operation_id
|
|
await self._reconcile_dataplane_unlocked(
|
|
plan, record, gate, previous_plan, **kwargs)
|
|
|
|
async def _reconcile_dataplane_unlocked(
|
|
self, plan: VpcPlan, record: dict[str, Any], gate: Mapping[str, Any],
|
|
previous_plan: VpcPlan | None = None,
|
|
private_link_states: Mapping[str, str] | None = None,
|
|
operation_id: str | None = None,
|
|
) -> None:
|
|
"""Apply/read back every participating host; stay closed on failure."""
|
|
request_deadline = time.monotonic() + VPC_REQUEST_DEADLINE_SECONDS
|
|
retiring_ids = {
|
|
item.get("host_id") for item in record.get("hosts", [])
|
|
if isinstance(item, Mapping)
|
|
and item.get("retiring") is True
|
|
and isinstance(item.get("host_id"), str)
|
|
}
|
|
member_cleanup = any(
|
|
isinstance(item, Mapping)
|
|
and isinstance(item.get("retiring_members"), list)
|
|
and bool(item.get("retiring_members"))
|
|
for item in record.get("hosts", [])
|
|
)
|
|
# A Fabric gate is required for normal applies. Retiring hosts still
|
|
# need one final SSH pass even when the new VPC has no members and the
|
|
# Fabric coordinator has already marked the tenant gate inactive.
|
|
if self.registry is None or (
|
|
gate.get("active") is not True
|
|
and not retiring_ids and not member_cleanup):
|
|
self._finish(record, gate)
|
|
return
|
|
if self._mark_lifecycle_conflicts(
|
|
plan, previous_plan, record, operation_id):
|
|
self._finish(record, gate)
|
|
return
|
|
try:
|
|
self._validate_dataplane_resources(plan)
|
|
fabric_record = self._fabric_record(plan)
|
|
full_fabric_plan = self.fabric._plan_from_record(fabric_record)
|
|
fabric_plan_digest = full_fabric_plan.digest
|
|
raw_fabric_keys = fabric_record.get("public_keys")
|
|
if not isinstance(raw_fabric_keys, Mapping):
|
|
raise VpcError("Fabric 公钥记录不完整", 500,
|
|
"vpc_fabric_record_corrupt")
|
|
fabric_public_keys = {
|
|
node.host_id: raw_fabric_keys.get(node.host_id)
|
|
for node in full_fabric_plan.nodes
|
|
}
|
|
if any(not isinstance(key, str) for key in fabric_public_keys.values()):
|
|
raise VpcError("Fabric 公钥记录不完整", 500,
|
|
"vpc_fabric_record_corrupt")
|
|
fabric_plan = full_fabric_plan
|
|
# ``record`` is the execution closure for this pass. Root-delete
|
|
# retries deliberately project it down to hosts that do not yet
|
|
# have an exact durable absence proof; deriving this set from the
|
|
# immutable submitted plan would still reprobe an already removed
|
|
# host and could block every remaining cleanup when that host is
|
|
# now offline. Normal apply records already contain both desired
|
|
# and retiring hosts, so the same rule keeps their host closure
|
|
# exact as well.
|
|
wanted = {
|
|
item.get("host_id") for item in record.get("hosts", [])
|
|
if isinstance(item, Mapping)
|
|
and isinstance(item.get("host_id"), str)
|
|
}
|
|
fabric_plan = replace(
|
|
fabric_plan,
|
|
nodes=tuple(node for node in fabric_plan.nodes
|
|
if node.host_id in wanted),
|
|
)
|
|
components = await self.fabric._components(fabric_plan)
|
|
all_peers = self._fabric_peers_for_plan(plan, fabric_record)
|
|
except Exception as exc:
|
|
code = self._observation_error(exc)
|
|
for host in record.get("hosts", []):
|
|
if isinstance(host, dict):
|
|
host.update({"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"last_error_code": code})
|
|
self._finish(record, gate)
|
|
return
|
|
|
|
for host in record.get("hosts", []):
|
|
if not isinstance(host, dict):
|
|
continue
|
|
host_id = host.get("host_id")
|
|
if time.monotonic() >= request_deadline:
|
|
host.update({
|
|
"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"last_error_code":
|
|
"vpc_private_guest_probe_deadline_exceeded",
|
|
})
|
|
continue
|
|
if not isinstance(host_id, str) or host_id not in components:
|
|
host.update({"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"last_error_code": "vpc_host_unavailable"})
|
|
continue
|
|
component, profile = components[host_id]
|
|
try:
|
|
# Persist a fair retry order. If this host consumes the shared
|
|
# budget, never-probed/older hosts sort ahead of it on the
|
|
# next reconcile even when this runtime proof stayed pending.
|
|
host["probe_attempted_at_ns"] = time.time_ns()
|
|
observed = await asyncio.to_thread(
|
|
self._run_host_dataplane, plan, host_id, component,
|
|
profile, all_peers, previous_plan,
|
|
host.get("retiring_members", []), fabric_public_keys,
|
|
fabric_plan_digest, private_link_states,
|
|
request_deadline=request_deadline)
|
|
if time.monotonic() >= request_deadline:
|
|
member_nics = observed.get("member_nics")
|
|
if isinstance(member_nics, list):
|
|
for member_nic in member_nics:
|
|
if not isinstance(member_nic, dict):
|
|
continue
|
|
member_nic.update({
|
|
"guest_network_verified": False,
|
|
"guest_network_readback":
|
|
"deadline_exceeded",
|
|
"guest_reason_code":
|
|
"vpc_private_guest_probe_deadline_exceeded",
|
|
})
|
|
observed.update({
|
|
"state": "applying",
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"reason_code":
|
|
"vpc_private_guest_probe_deadline_exceeded",
|
|
})
|
|
host.update(observed)
|
|
host["observed_revision"] = (
|
|
plan.desired_revision
|
|
if observed.get("state") == "active" else 0)
|
|
host["observed_digest"] = (
|
|
plan.desired_digest
|
|
if observed.get("state") == "active" else None)
|
|
host["state"] = observed.get("state", "degraded")
|
|
host["last_error_code"] = observed.get("reason_code")
|
|
if host.get("retiring") is True:
|
|
host["removed"] = observed.get("removed") is True
|
|
if (host.get("retiring_members")
|
|
and observed.get("config_verified") is True
|
|
and observed.get("config_observed_revision")
|
|
== plan.desired_revision
|
|
and observed.get("config_observed_digest")
|
|
== plan.desired_digest):
|
|
# The host adapter has positively removed every member
|
|
# listed in this tombstone. Guest runtime readiness for a
|
|
# different member is deliberately irrelevant to this
|
|
# persistent removal proof.
|
|
host["retiring_members"] = []
|
|
except Exception as exc:
|
|
host.update({"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"last_error_code": self._observation_error(exc)})
|
|
self._finish(record, gate)
|
|
|
|
@classmethod
|
|
def _desired_record(cls, plan: VpcPlan, gate: Mapping[str, Any], *, state: str,
|
|
observed_revision: int = 0,
|
|
observed_digest: str | None = None,
|
|
previous_hosts: Mapping[str, Mapping[str, Any]] | None = None,
|
|
previous_plan: VpcPlan | None = None,
|
|
updated_at: int | None = None) -> dict[str, Any]:
|
|
now = int(time.time()) if updated_at is None else updated_at
|
|
previous_hosts = previous_hosts or {}
|
|
host_records = []
|
|
current_members_by_host: dict[str, set[str]] = {}
|
|
for member in plan.members:
|
|
current_members_by_host.setdefault(member.host_id, set()).add(
|
|
member.member_id)
|
|
for host in plan.hosts:
|
|
old = previous_hosts.get(host.host_id)
|
|
old_revision = old.get("observed_revision", 0) if isinstance(old, Mapping) else 0
|
|
old_digest = old.get("observed_digest") if isinstance(old, Mapping) else None
|
|
old_readback = old.get("readback", {}) if isinstance(old, Mapping) else {}
|
|
if (isinstance(old_revision, bool) or not isinstance(old_revision, int)
|
|
or old_revision < 0 or old_revision > plan.desired_revision):
|
|
old_revision = 0
|
|
old_digest = None
|
|
if old_digest is not None:
|
|
try:
|
|
old_digest = _digest(old_digest, "observed_digest")
|
|
except VpcError:
|
|
old_digest = None
|
|
if isinstance(old_readback, str) and len(old_readback) <= 128:
|
|
safe_readback: Any = old_readback
|
|
elif isinstance(old_readback, Mapping):
|
|
safe_readback = {
|
|
str(key): value for key, value in old_readback.items()
|
|
if isinstance(key, str) and len(key) <= 64
|
|
and _safe_scalar(value)
|
|
}
|
|
else:
|
|
safe_readback = {}
|
|
retiring_members: list[dict[str, Any]] = []
|
|
if isinstance(old, Mapping):
|
|
prior_retiring = old.get("retiring_members", [])
|
|
if isinstance(prior_retiring, list):
|
|
retiring_members.extend(
|
|
dict(item) for item in prior_retiring
|
|
if isinstance(item, Mapping))
|
|
if previous_plan is not None:
|
|
current_ids = current_members_by_host.get(host.host_id, set())
|
|
for member in previous_plan.members:
|
|
if (member.host_id == host.host_id
|
|
and member.member_id not in current_ids
|
|
and not any(
|
|
item.get("member_id") == member.member_id
|
|
for item in retiring_members)):
|
|
retiring_members.append({
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"host_id": member.host_id,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"subnet_id": member.subnet_id,
|
|
"private_mac": _effective_private_mac(
|
|
plan.vpc_id, member.member_id,
|
|
member.private_mac),
|
|
})
|
|
host_records.append({
|
|
"host_id": host.host_id, "member_ids": list(host.member_ids),
|
|
"desired_revision": plan.desired_revision,
|
|
"vpc_id": plan.vpc_id,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_revision": old_revision,
|
|
"observed_digest": old_digest,
|
|
"state": "applying" if gate.get("active") else "uncertain",
|
|
"transport_verified": bool(gate.get("active")),
|
|
# Fabric transport is necessary but never sufficient for a
|
|
# tenant VPC. Only the future host adapter may set this
|
|
# after private0/route/nft/member readback.
|
|
"private_dataplane_verified": False,
|
|
"config_verified": False,
|
|
"config_observed_revision": 0,
|
|
"config_observed_digest": None,
|
|
"removed": bool(old.get("removed", False)) if isinstance(old, Mapping) else False,
|
|
"bridge": old.get("bridge") if isinstance(old, Mapping) else None,
|
|
"bridge_owner": old.get("bridge_owner") if isinstance(old, Mapping) else None,
|
|
"nft_table": old.get("nft_table") if isinstance(old, Mapping) else None,
|
|
"anti_spoof": bool(old.get("anti_spoof")) if isinstance(old, Mapping) else False,
|
|
"rp_filter": old.get("rp_filter") if isinstance(old, Mapping) else None,
|
|
"proxy_arp": old.get("proxy_arp") if isinstance(old, Mapping) else None,
|
|
"proxy_arp_pvlan": old.get("proxy_arp_pvlan") if isinstance(old, Mapping) else None,
|
|
"proxy_ips": old.get("proxy_ips", []) if isinstance(old, Mapping) else [],
|
|
"acl_revision": old.get("acl_revision") if isinstance(old, Mapping) else None,
|
|
"rollback": old.get("rollback", "none") if isinstance(old, Mapping) else "none",
|
|
"member_nics": old.get("member_nics", []) if isinstance(old, Mapping) else [],
|
|
"routes": old.get("routes", []) if isinstance(old, Mapping) else [],
|
|
"wg_allowed_ips": old.get("wg_allowed_ips", []) if isinstance(old, Mapping) else [],
|
|
"readback": safe_readback,
|
|
"retiring_members": retiring_members,
|
|
})
|
|
# Keep host tombstones in the durable engine record until the host-side
|
|
# removal script returns its typed absence proof. They are excluded
|
|
# from the new VPC plan by _plan_from_record(retiring=true), but retain
|
|
# the old instance identity so a restart can still remove private0.
|
|
current_host_ids = {host.host_id for host in plan.hosts}
|
|
retiring_hosts: dict[str, list[dict[str, Any]]] = {}
|
|
if previous_plan is not None:
|
|
for old_host in previous_plan.hosts:
|
|
if old_host.host_id in current_host_ids:
|
|
continue
|
|
old_members = [
|
|
{"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"host_id": member.host_id,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"subnet_id": member.subnet_id,
|
|
"private_mac": _effective_private_mac(
|
|
plan.vpc_id, member.member_id, member.private_mac)}
|
|
for member in previous_plan.members
|
|
if member.host_id == old_host.host_id
|
|
]
|
|
old = previous_hosts.get(old_host.host_id)
|
|
if (isinstance(old, Mapping)
|
|
and not old_members
|
|
and cls._confirmed_removal(
|
|
old, previous_plan.desired_revision,
|
|
previous_plan.desired_digest)):
|
|
# PHP historically kept a deleting deployment in the
|
|
# desired host list until this exact empty-host proof was
|
|
# durable. Once the next revision omits that host, the
|
|
# proof is already terminal: recreating a retiring row
|
|
# would recontact a resource that has been positively
|
|
# removed and would expose it as an unexpected host again.
|
|
continue
|
|
retiring_hosts[old_host.host_id] = old_members
|
|
# ``_plan_from_record`` deliberately excludes retiring rows from
|
|
# the typed live plan. Carry any still-unconfirmed rows directly
|
|
# from the durable host closure so a second newer revision cannot
|
|
# silently discard the tombstone (and make _save_children publish
|
|
# a false removal marker). Exact absence proofs remain terminal
|
|
# and are pruned instead of being reprobed.
|
|
for old_host_id, old in previous_hosts.items():
|
|
if (old_host_id in current_host_ids
|
|
or old_host_id in retiring_hosts
|
|
or old.get("retiring") is not True):
|
|
continue
|
|
if cls._confirmed_removal(
|
|
old, previous_plan.desired_revision,
|
|
previous_plan.desired_digest):
|
|
continue
|
|
retiring_hosts[old_host_id] = cls._durable_retiring_members(
|
|
old, vpc_id=plan.vpc_id, host_id=old_host_id)
|
|
for old_host_id, old_members in retiring_hosts.items():
|
|
host_records.append({
|
|
"host_id": old_host_id,
|
|
"member_ids": [item["member_id"] for item in old_members],
|
|
"vpc_id": plan.vpc_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"state": "deleting",
|
|
"transport_verified": bool(gate.get("active")),
|
|
"private_dataplane_verified": False,
|
|
"config_verified": False,
|
|
"config_observed_revision": 0,
|
|
"config_observed_digest": None,
|
|
"retiring": True,
|
|
"retiring_members": old_members,
|
|
"bridge": None, "bridge_owner": None, "nft_table": None,
|
|
"anti_spoof": False, "rp_filter": None,
|
|
"proxy_arp": None, "proxy_arp_pvlan": None,
|
|
"proxy_ips": [], "acl_revision": None,
|
|
"rollback": "none", "member_nics": [], "routes": [],
|
|
"wg_allowed_ips": [], "readback": {},
|
|
})
|
|
return {
|
|
"schema_version": VPC_SCHEMA_VERSION,
|
|
"evidence_version": VPC_EVIDENCE_VERSION,
|
|
"vpc_id": plan.vpc_id, "region": plan.region,
|
|
"fabric_id": plan.fabric_id,
|
|
"fabric_revision": plan.fabric_revision,
|
|
"cidr": plan.cidr,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_revision": observed_revision,
|
|
"observed_digest": observed_digest,
|
|
"operation_id": plan.operation_id,
|
|
"state": state,
|
|
"next_stage_allowed": False,
|
|
"private_dataplane_verified": False,
|
|
"fabric_gate": dict(gate),
|
|
"hosts": host_records,
|
|
"subnets": [item.__dict__ for item in plan.subnets],
|
|
"members": [
|
|
{"member_id": item.member_id,
|
|
"instance_uuid": item.instance_uuid,
|
|
"host_id": item.host_id,
|
|
"private_ip_cidr": f"{item.private_ip}/32",
|
|
"subnet_id": item.subnet_id,
|
|
**({"private_mac": item.private_mac}
|
|
if item.private_mac else {})}
|
|
for item in plan.members
|
|
],
|
|
"routes": [item.__dict__ for item in plan.routes],
|
|
"acls": [item.__dict__ for item in plan.acls],
|
|
"updated_at": now,
|
|
}
|
|
|
|
@staticmethod
|
|
def _public(value: Mapping[str, Any]) -> dict[str, Any]:
|
|
result: dict[str, Any] = {}
|
|
for key in (
|
|
"vpc_id", "region", "fabric_id", "fabric_revision", "cidr",
|
|
"evidence_version",
|
|
"desired_revision", "desired_digest", "observed_revision",
|
|
"observed_digest", "state", "next_stage_allowed",
|
|
"private_dataplane_verified", "operation_id", "updated_at"):
|
|
if key in value:
|
|
result[key] = value[key]
|
|
gate = value.get("fabric_gate")
|
|
if isinstance(gate, Mapping):
|
|
result["fabric_gate"] = {
|
|
key: gate[key] for key in (
|
|
"active", "available", "fabric_id", "fabric_revision",
|
|
"state", "next_stage_allowed", "error_code")
|
|
if key in gate
|
|
}
|
|
for collection, fields in (
|
|
("subnets", ("subnet_id", "cidr")),
|
|
("members", ("member_id", "instance_uuid", "host_id",
|
|
"private_ip_cidr", "subnet_id", "private_mac")),
|
|
("routes", ("route_id", "destination_cidr",
|
|
"next_hop_member_id", "next_hop_host_id", "metric")),
|
|
("acls", ("acl_id", "direction", "action", "protocol",
|
|
"source_cidr", "destination_cidr", "port_from",
|
|
"port_to", "priority")),
|
|
):
|
|
rows = value.get(collection)
|
|
if isinstance(rows, list):
|
|
result[collection] = [
|
|
{key: row[key] for key in fields if key in row}
|
|
for row in rows if isinstance(row, Mapping)
|
|
][:VPC_MAX_READBACK]
|
|
hosts = value.get("hosts")
|
|
if isinstance(hosts, list):
|
|
result["hosts"] = []
|
|
for raw in hosts[:VPC_MAX_HOSTS]:
|
|
if not isinstance(raw, Mapping):
|
|
continue
|
|
item = {key: raw[key] for key in (
|
|
"host_id", "member_ids", "vpc_id", "desired_revision",
|
|
"desired_digest", "bridge", "bridge_owner", "nft_table",
|
|
"observed_revision", "observed_digest", "state",
|
|
"transport_verified", "private_dataplane_verified",
|
|
"config_verified", "config_observed_revision",
|
|
"config_observed_digest",
|
|
"last_error_code", "removed", "retiring", "anti_spoof", "rp_filter", "acl_revision", "rollback",
|
|
"proxy_arp", "proxy_arp_pvlan", "proxy_ips", "member_nics", "routes", "wg_allowed_ips") if key in raw}
|
|
readback = raw.get("readback")
|
|
if isinstance(readback, Mapping):
|
|
item["readback"] = {
|
|
str(key): val for key, val in readback.items()
|
|
if isinstance(key, str) and len(key) <= 64
|
|
and _safe_scalar(val)
|
|
}
|
|
elif isinstance(readback, str) and len(readback) <= 128:
|
|
item["readback"] = readback
|
|
result["hosts"].append(item)
|
|
# PHP's typed VPC service consumes an explicit observed projection and
|
|
# keys host evidence by stable host ID. Keep this projection bounded
|
|
# and scalar-only; the human-facing ``hosts`` list above remains the
|
|
# convenient status representation.
|
|
observed_hosts: dict[str, dict[str, Any]] = {}
|
|
if isinstance(hosts, list):
|
|
for raw in hosts[:VPC_MAX_HOSTS]:
|
|
if not isinstance(raw, Mapping) or not isinstance(raw.get("host_id"), str):
|
|
continue
|
|
host_id = raw["host_id"]
|
|
item: dict[str, Any] = {}
|
|
for key in ("state", "observed_revision", "observed_digest",
|
|
"transport_verified", "private_dataplane_verified",
|
|
"config_verified", "config_observed_revision",
|
|
"config_observed_digest",
|
|
"last_error_code", "vpc_id", "host_id",
|
|
"desired_revision", "desired_digest", "bridge",
|
|
"bridge_owner", "nft_table", "anti_spoof",
|
|
"rp_filter", "proxy_arp", "proxy_arp_pvlan", "proxy_ips", "acl_revision", "rollback", "removed", "retiring", "member_nics",
|
|
"routes", "wg_allowed_ips", "readback"):
|
|
if key in raw:
|
|
value_item = raw[key]
|
|
if key in {"member_nics", "routes", "wg_allowed_ips", "proxy_ips"}:
|
|
if isinstance(value_item, (list, tuple)):
|
|
if key == "proxy_ips":
|
|
item[key] = [str(ip)[:64] for ip in value_item[:4096]
|
|
if isinstance(ip, str)]
|
|
else:
|
|
item[key] = [dict(row) for row in value_item[:4096]
|
|
if isinstance(row, Mapping)]
|
|
elif key == "readback":
|
|
if isinstance(value_item, str) and len(value_item) <= 128:
|
|
item[key] = value_item
|
|
elif isinstance(value_item, Mapping):
|
|
item[key] = {str(k): v for k, v in value_item.items()
|
|
if isinstance(k, str) and _safe_scalar(v)}
|
|
elif _safe_scalar(value_item):
|
|
item[key] = value_item
|
|
item["next_stage_allowed"] = (
|
|
raw.get("state") == "active"
|
|
and raw.get("transport_verified") is True
|
|
and raw.get("private_dataplane_verified") is True
|
|
and raw.get("observed_revision") == value.get("desired_revision")
|
|
and raw.get("observed_digest") == value.get("desired_digest")
|
|
)
|
|
observed_hosts[host_id] = item
|
|
result["observed"] = {
|
|
"evidence_version": value.get("evidence_version", 0),
|
|
"revision": value.get("observed_revision", 0),
|
|
"digest": value.get("observed_digest"),
|
|
"state": value.get("state"),
|
|
"next_stage_allowed": value.get("next_stage_allowed") is True,
|
|
"private_dataplane_verified": (
|
|
value.get("private_dataplane_verified") is True
|
|
),
|
|
"hosts": observed_hosts,
|
|
# PHP consumes this bounded observed projection when deciding
|
|
# whether the control-plane result is deliberately waiting for
|
|
# the later private data-plane phase. Keep the gate here as well
|
|
# as at the top level so callers cannot accidentally discard the
|
|
# reason while selecting ``result['observed']``.
|
|
"fabric_gate": dict(result.get("fabric_gate", {})),
|
|
}
|
|
return result
|
|
|
|
def list_status(self) -> list[dict[str, Any]]:
|
|
result: list[dict[str, Any]] = []
|
|
for _key, raw in self.store.list("vpc_desired").items():
|
|
try:
|
|
value = json.loads(raw)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if (isinstance(value, Mapping)
|
|
and value.get("vpc_id") == _key):
|
|
result.append(self._public(value))
|
|
return sorted(result, key=lambda item: item.get("vpc_id", ""))
|
|
|
|
def get_status(self, vpc_id: str) -> dict[str, Any]:
|
|
_identifier(vpc_id, "VPC")
|
|
value = self._load("vpc_desired", vpc_id)
|
|
if value is None:
|
|
raise VpcError("VPC 不存在", 404, "vpc_not_found")
|
|
return self._public(value)
|
|
|
|
def _save_children(self, record: Mapping[str, Any]) -> None:
|
|
vpc_id = record.get("vpc_id")
|
|
if not isinstance(vpc_id, str):
|
|
return
|
|
for kind, collection, identifier in (
|
|
("vpc_subnet", "subnets", "subnet_id"),
|
|
("vpc_member", "members", "member_id"),
|
|
("vpc_route", "routes", "route_id"),
|
|
("vpc_acl", "acls", "acl_id")):
|
|
rows = record.get(collection)
|
|
if not isinstance(rows, list):
|
|
continue
|
|
live_keys: set[str] = set()
|
|
for row in rows:
|
|
if not isinstance(row, Mapping) or not isinstance(row.get(identifier), str):
|
|
continue
|
|
child_id = row[identifier]
|
|
key = self._key(vpc_id, child_id)
|
|
live_keys.add(key)
|
|
child = dict(row)
|
|
child.update({"vpc_id": vpc_id,
|
|
"desired_revision": record.get("desired_revision"),
|
|
"desired_digest": record.get("desired_digest"),
|
|
"observed_revision": record.get("observed_revision", 0),
|
|
"observed_digest": record.get("observed_digest"),
|
|
"state": record.get("state")})
|
|
self._save(kind, key, child)
|
|
for key in self.store.list(kind):
|
|
if key.startswith(vpc_id + ".") and key not in live_keys:
|
|
# Keep a tombstone until a later observed revision proves
|
|
# that the data-plane removed the child. Deleting the
|
|
# row here would make a partial member/route/ACL removal
|
|
# impossible to reconcile after a crash.
|
|
raw_old = self.store.get(kind, key)
|
|
try:
|
|
old = json.loads(raw_old) if raw_old is not None else {}
|
|
except (TypeError, ValueError):
|
|
old = {"record_corrupt": True}
|
|
if not isinstance(old, dict):
|
|
old = {"record_corrupt": True}
|
|
old.update({
|
|
"vpc_id": vpc_id,
|
|
"state": "removed",
|
|
"removed": True,
|
|
"removed_at": int(time.time()),
|
|
"desired_revision": record.get("desired_revision"),
|
|
"desired_digest": record.get("desired_digest"),
|
|
})
|
|
self._save(kind, key, old)
|
|
hosts = record.get("hosts")
|
|
if isinstance(hosts, list):
|
|
live = set()
|
|
for row in hosts:
|
|
if isinstance(row, Mapping) and isinstance(row.get("host_id"), str):
|
|
key = self._key(vpc_id, row["host_id"])
|
|
live.add(key)
|
|
child = dict(row)
|
|
child.update({"vpc_id": vpc_id,
|
|
"desired_revision": record.get("desired_revision"),
|
|
"desired_digest": record.get("desired_digest")})
|
|
self._save("vpc_host", key, child)
|
|
for key in self.store.list("vpc_host"):
|
|
if key.startswith(vpc_id + ".") and key not in live:
|
|
raw_old = self.store.get("vpc_host", key)
|
|
try:
|
|
old = json.loads(raw_old) if raw_old is not None else {}
|
|
except (TypeError, ValueError):
|
|
old = {"record_corrupt": True}
|
|
if not isinstance(old, dict):
|
|
old = {"record_corrupt": True}
|
|
if (old.get("vpc_id") == vpc_id
|
|
and isinstance(old.get("host_id"), str)
|
|
and key == self._key(vpc_id, old["host_id"])
|
|
and self._terminal_removal(old)):
|
|
# Keep the exact revision/digest/readback envelope as
|
|
# a monotonic historical proof. A later live row for
|
|
# the same host overwrites this child normally.
|
|
old.setdefault("removed_at", int(time.time()))
|
|
self._save("vpc_host", key, old)
|
|
continue
|
|
old.update({
|
|
"vpc_id": vpc_id,
|
|
"state": "removed",
|
|
"removed": True,
|
|
"removed_at": int(time.time()),
|
|
"desired_revision": record.get("desired_revision"),
|
|
"desired_digest": record.get("desired_digest"),
|
|
})
|
|
self._save("vpc_host", key, old)
|
|
|
|
@classmethod
|
|
def _finish(cls, record: dict[str, Any], gate: Mapping[str, Any]) -> None:
|
|
record["evidence_version"] = VPC_EVIDENCE_VERSION
|
|
record["fabric_gate"] = dict(gate)
|
|
# ``recover`` marks a durable record as needing one fresh
|
|
# control-plane pass. Once the Fabric gate is available again, reset
|
|
# only the host control-plane state to applying; never infer active or
|
|
# data-plane readiness from stale pre-restart evidence.
|
|
if gate.get("active") is True and record.pop("recovery_pending", False) is True:
|
|
recovery_revision = record.get("desired_revision")
|
|
recovery_digest = record.get("desired_digest")
|
|
for item in record.get("hosts", []):
|
|
if isinstance(item, dict):
|
|
if (isinstance(recovery_revision, int)
|
|
and not isinstance(recovery_revision, bool)
|
|
and isinstance(recovery_digest, str)
|
|
and cls._confirmed_removal(
|
|
item, recovery_revision, recovery_digest)):
|
|
# An exact absence proof is terminal and monotonic. A
|
|
# restart must invalidate live readiness, but must not
|
|
# turn an already removed host back into a probe target.
|
|
continue
|
|
item["state"] = "applying"
|
|
item["transport_verified"] = True
|
|
item["private_dataplane_verified"] = False
|
|
item["config_verified"] = False
|
|
item["config_observed_revision"] = 0
|
|
item["config_observed_digest"] = None
|
|
hosts = record.get("hosts")
|
|
desired_revision = record.get("desired_revision")
|
|
desired_digest = record.get("desired_digest")
|
|
all_observed = bool(hosts) and all(
|
|
isinstance(item, Mapping)
|
|
and item.get("state") == "active"
|
|
and item.get("observed_revision") == desired_revision
|
|
and item.get("observed_digest") == desired_digest
|
|
and item.get("transport_verified") is True
|
|
and item.get("private_dataplane_verified") is True
|
|
for item in hosts)
|
|
retiring_hosts = [
|
|
item for item in hosts if isinstance(item, Mapping)
|
|
and item.get("retiring") is True
|
|
] if isinstance(hosts, list) else []
|
|
live_hosts = [
|
|
item for item in hosts if not (isinstance(item, Mapping)
|
|
and item.get("retiring") is True)
|
|
] if isinstance(hosts, list) else []
|
|
all_removed = bool(retiring_hosts) and all(
|
|
item.get("state") == "active"
|
|
and item.get("removed") is True
|
|
and item.get("observed_revision") == desired_revision
|
|
and item.get("observed_digest") == desired_digest
|
|
and item.get("transport_verified") is True
|
|
and item.get("private_dataplane_verified") is True
|
|
for item in retiring_hosts)
|
|
fabric_active = gate.get("active") is True
|
|
if all_removed and not live_hosts:
|
|
# A VPC with no remaining members is a terminal removal proof;
|
|
# the Fabric gate may correctly be inactive because there is no
|
|
# tenant transport left to activate.
|
|
record["state"] = "active"
|
|
record["next_stage_allowed"] = True
|
|
record["private_dataplane_verified"] = True
|
|
record["observed_revision"] = desired_revision
|
|
record["observed_digest"] = desired_digest
|
|
elif fabric_active and all_observed:
|
|
record["state"] = "active"
|
|
record["next_stage_allowed"] = True
|
|
record["private_dataplane_verified"] = True
|
|
record["observed_revision"] = desired_revision
|
|
record["observed_digest"] = desired_digest
|
|
elif not fabric_active:
|
|
record["state"] = "degraded" if gate.get("available") else "uncertain"
|
|
record["next_stage_allowed"] = False
|
|
record["private_dataplane_verified"] = False
|
|
for item in hosts if isinstance(hosts, list) else []:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
if (isinstance(desired_revision, int)
|
|
and isinstance(desired_digest, str)
|
|
and cls._confirmed_removal(
|
|
item, desired_revision, desired_digest)):
|
|
continue
|
|
# Fabric reachability is a prerequisite for both shared host
|
|
# configuration and member runtime usability. Never let a
|
|
# prior active observation survive an inactive current gate.
|
|
item["transport_verified"] = False
|
|
item["private_dataplane_verified"] = False
|
|
item["config_verified"] = False
|
|
item["observed_revision"] = 0
|
|
item["observed_digest"] = None
|
|
item["config_observed_revision"] = 0
|
|
item["config_observed_digest"] = None
|
|
elif hosts and any(item.get("state") == "uncertain" for item in hosts if isinstance(item, Mapping)):
|
|
record["state"] = "uncertain"
|
|
record["next_stage_allowed"] = False
|
|
record["private_dataplane_verified"] = False
|
|
elif hosts and any(item.get("state") == "degraded" for item in hosts if isinstance(item, Mapping)):
|
|
record["state"] = "degraded"
|
|
record["next_stage_allowed"] = False
|
|
record["private_dataplane_verified"] = False
|
|
else:
|
|
record["state"] = "applying"
|
|
record["next_stage_allowed"] = False
|
|
record["private_dataplane_verified"] = False
|
|
record["updated_at"] = int(time.time())
|
|
|
|
async def _reconcile_pending_hosts(
|
|
self, plan: VpcPlan, record: dict[str, Any], gate: Mapping[str, Any],
|
|
previous_plan: VpcPlan | None,
|
|
private_link_states: Mapping[str, str] | None = None,
|
|
extra_identities: Sequence[tuple[str, str]] | None = None,
|
|
operation_id: str | None = None,
|
|
) -> None:
|
|
"""Retry only hosts without an exact durable runtime/removal proof."""
|
|
self._finish(record, gate)
|
|
durable_hosts = self._host_closure(record)
|
|
member_host_ids = {member.host_id for member in plan.members}
|
|
targeted_host_ids = {
|
|
member.host_id for member in plan.members
|
|
if (private_link_states is not None
|
|
and member.instance_uuid in private_link_states)
|
|
}
|
|
if extra_identities is not None:
|
|
targeted_host_ids.update(
|
|
host_id for host_id, _instance_uuid in extra_identities)
|
|
pending_host_ids = {
|
|
host_id for host_id, host in durable_hosts.items()
|
|
if ((host_id in member_host_ids
|
|
and (host_id in targeted_host_ids
|
|
or not self._confirmed_active_host(host, plan)))
|
|
or (host_id not in member_host_ids
|
|
and not self._confirmed_removal(
|
|
host, plan.desired_revision, plan.desired_digest)))
|
|
}
|
|
if not pending_host_ids:
|
|
def attempted_at(host: Mapping[str, Any]) -> int:
|
|
value = host.get("probe_attempted_at_ns", 0)
|
|
if (isinstance(value, bool)
|
|
or not isinstance(value, int) or value < 0):
|
|
return 0
|
|
return value
|
|
|
|
# Exact guest runtime evidence is not timeless: QGA, its route,
|
|
# or the guest address may disappear after the durable host row
|
|
# became active. Pending and lifecycle-targeted work remains the
|
|
# priority; only an otherwise idle periodic pass spends its
|
|
# bounded budget revalidating one oldest active member host.
|
|
oldest_active = min(
|
|
(
|
|
(attempted_at(host), host_id)
|
|
for host_id, host in durable_hosts.items()
|
|
if (host_id in member_host_ids
|
|
and self._confirmed_active_host(host, plan))
|
|
),
|
|
default=None,
|
|
)
|
|
if oldest_active is None:
|
|
return
|
|
pending_host_ids.add(oldest_active[1])
|
|
execution_record = dict(record)
|
|
execution_hosts = [
|
|
dict(host) for host in record.get("hosts", [])
|
|
if (isinstance(host, Mapping)
|
|
and host.get("host_id") in pending_host_ids)
|
|
]
|
|
def attempt_order(host: Mapping[str, Any]) -> tuple[int, int, str]:
|
|
attempted_at = host.get("probe_attempted_at_ns", 0)
|
|
if (isinstance(attempted_at, bool)
|
|
or not isinstance(attempted_at, int)
|
|
or attempted_at < 0):
|
|
attempted_at = 0
|
|
host_id = str(host.get("host_id", ""))
|
|
# A lifecycle request (suspend/resume/delete) must not sit behind
|
|
# unrelated periodic guest probes. Fair rotation applies within
|
|
# the targeted and ordinary groups independently.
|
|
return (0 if host_id in targeted_host_ids else 1,
|
|
attempted_at, host_id)
|
|
|
|
execution_hosts.sort(key=attempt_order)
|
|
execution_record["hosts"] = execution_hosts
|
|
reconcile_kwargs: dict[str, Any] = {}
|
|
if private_link_states is not None:
|
|
reconcile_kwargs["private_link_states"] = private_link_states
|
|
if extra_identities is not None:
|
|
reconcile_kwargs["extra_identities"] = extra_identities
|
|
if operation_id is not None:
|
|
reconcile_kwargs["operation_id"] = operation_id
|
|
await self._reconcile_dataplane(
|
|
plan, execution_record, gate, previous_plan, **reconcile_kwargs)
|
|
observations = {
|
|
host.get("host_id"): dict(host)
|
|
for host in execution_record.get("hosts", [])
|
|
if (isinstance(host, Mapping)
|
|
and isinstance(host.get("host_id"), str))
|
|
}
|
|
record["hosts"] = [
|
|
dict(observations[host["host_id"]])
|
|
if (isinstance(host, Mapping)
|
|
and host.get("host_id") in observations)
|
|
else host
|
|
for host in record.get("hosts", [])
|
|
]
|
|
self._finish(record, gate)
|
|
|
|
async def delete(self, vpc_id: str, value: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""Remove every host resource and persist an idempotent root receipt.
|
|
|
|
The receipt is written before the desired/child records are purged.
|
|
Consequently a lost HTTP response can be retried with the exact same
|
|
operation/revision/digest without turning store absence into guessed
|
|
success. A signed no-host target is the sole exception: when neither
|
|
a desired record nor a receipt exists, PHP's independently locked
|
|
deployment ledger supplies the other half of that absence proof.
|
|
"""
|
|
vpc_id = _identifier(vpc_id, "VPC")
|
|
plan, retiring_members = parse_vpc_delete_plan(value)
|
|
if plan.vpc_id != vpc_id:
|
|
raise VpcError("VPC 删除路径与目标不一致", 409,
|
|
"vpc_delete_target_mismatch")
|
|
self._delete_plan(plan)
|
|
|
|
async def delete_locked() -> dict[str, Any]:
|
|
receipt = self._load_delete_receipt(vpc_id)
|
|
if receipt is not None:
|
|
if (receipt["operation_id"] != plan.operation_id
|
|
or receipt["desired_revision"] != plan.desired_revision
|
|
or receipt["desired_digest"] != plan.desired_digest):
|
|
raise VpcError("VPC 已由另一删除操作完成", 409,
|
|
"vpc_delete_receipt_conflict")
|
|
self._purge_deleted_vpc(vpc_id)
|
|
return self._public_delete_receipt(receipt)
|
|
|
|
previous = self._load("vpc_desired", vpc_id)
|
|
if previous is None and not plan.host_ids:
|
|
receipt = {
|
|
"schema_version": VPC_SCHEMA_VERSION,
|
|
"evidence_version": VPC_EVIDENCE_VERSION,
|
|
"vpc_id": vpc_id,
|
|
"operation_id": plan.operation_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"state": "deleted",
|
|
"deleted": True,
|
|
"readback": "confirmed",
|
|
"removed_host_ids": [],
|
|
"completed_at": int(time.time()),
|
|
}
|
|
self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, receipt)
|
|
self._purge_deleted_vpc(vpc_id)
|
|
return self._public_delete_receipt(receipt)
|
|
|
|
# A host-bearing delete without a prior desired record creates a
|
|
# temporary durable VPC catalog reservation before contacting any
|
|
# host. It must obey the same global Fabric endpoint exclusion as
|
|
# an ordinary apply; otherwise it could race an endpoint migration
|
|
# and publish a conflicting tenant CIDR under the delete marker.
|
|
if previous is None:
|
|
self._validate_global_fabric_endpoints(plan)
|
|
|
|
marker = (self._delete_marker(previous)
|
|
if previous is not None else None)
|
|
transition_previous: VpcPlan | None = None
|
|
if marker is None:
|
|
if previous is None:
|
|
previous_plan = None
|
|
old_hosts: dict[str, Mapping[str, Any]] = {}
|
|
previous_observed_revision = 0
|
|
previous_observed_digest = None
|
|
else:
|
|
previous_plan = _plan_from_record(previous)
|
|
old_revision = previous.get("desired_revision")
|
|
old_digest = previous.get("desired_digest")
|
|
if (old_revision != previous_plan.desired_revision
|
|
or old_digest != previous_plan.desired_digest):
|
|
raise VpcError("VPC 持久化目标摘要不一致", 500,
|
|
"vpc_record_corrupt")
|
|
if (
|
|
plan.region,
|
|
plan.fabric_id,
|
|
plan.fabric_revision,
|
|
plan.cidr,
|
|
) != (
|
|
previous_plan.region,
|
|
previous_plan.fabric_id,
|
|
previous_plan.fabric_revision,
|
|
previous_plan.cidr,
|
|
):
|
|
raise VpcError(
|
|
"VPC 删除目标不能改变持久化目录身份",
|
|
409, "vpc_delete_identity_mismatch")
|
|
if plan.desired_revision <= previous_plan.desired_revision:
|
|
raise VpcError("VPC 删除版本必须高于当前目标", 409,
|
|
"vpc_revision_conflict")
|
|
if plan.operation_id == previous_plan.operation_id:
|
|
raise VpcError("VPC 删除必须使用新的操作编号", 409,
|
|
"vpc_delete_receipt_conflict")
|
|
old_hosts = self._host_closure(previous)
|
|
previous_observed_revision = previous.get(
|
|
"observed_revision", 0)
|
|
previous_observed_digest = previous.get("observed_digest")
|
|
old_host_ids = set(old_hosts)
|
|
requested_host_ids = set(plan.host_ids)
|
|
historical_preverified: set[str] = set()
|
|
for host_id in requested_host_ids - old_host_ids:
|
|
raw_child = self.store.get(
|
|
"vpc_host", self._key(vpc_id, host_id))
|
|
try:
|
|
child = (json.loads(raw_child)
|
|
if raw_child is not None else None)
|
|
except (TypeError, ValueError):
|
|
child = None
|
|
proof_revision = (child.get("desired_revision")
|
|
if isinstance(child, Mapping) else None)
|
|
if (isinstance(child, Mapping)
|
|
and child.get("vpc_id") == vpc_id
|
|
and child.get("host_id") == host_id
|
|
and isinstance(proof_revision, int)
|
|
and not isinstance(proof_revision, bool)
|
|
and proof_revision < plan.desired_revision
|
|
and self._terminal_removal(child)):
|
|
historical_preverified.add(host_id)
|
|
member_hosts = {
|
|
member.host_id for member in (previous_plan.members
|
|
if previous_plan else ())
|
|
}
|
|
manifest_hosts = {
|
|
member.host_id for member in retiring_members
|
|
}
|
|
preverified = set()
|
|
if previous_plan is not None:
|
|
preverified = {
|
|
host_id for host_id, host in old_hosts.items()
|
|
if host_id not in member_hosts
|
|
and host_id not in manifest_hosts
|
|
and self._confirmed_removal(
|
|
host, previous_plan.desired_revision,
|
|
previous_plan.desired_digest)
|
|
}
|
|
# A host pruned from a later normal target is absent from the
|
|
# current root closure. Its exact child proof remains valid
|
|
# unless a new live row overwrote it, and is strong enough to
|
|
# cover a historical member in PHP's final delete manifest.
|
|
preverified.update(historical_preverified)
|
|
missing = old_host_ids - requested_host_ids
|
|
if not missing.issubset(preverified):
|
|
raise VpcError("VPC 删除遗漏尚未清理的宿主机", 409,
|
|
"vpc_delete_host_set_mismatch")
|
|
required_host_ids = requested_host_ids - preverified
|
|
|
|
retiring_by_id = {
|
|
member.member_id: member for member in retiring_members
|
|
}
|
|
for old_member in (previous_plan.members
|
|
if previous_plan else ()):
|
|
signed = retiring_by_id.get(old_member.member_id)
|
|
if (signed is None
|
|
or signed.instance_uuid != old_member.instance_uuid
|
|
or signed.host_id != old_member.host_id
|
|
or signed.private_ip != old_member.private_ip
|
|
or (old_member.private_mac is not None
|
|
and signed.private_mac
|
|
!= old_member.private_mac)):
|
|
raise VpcError(
|
|
"VPC 删除成员清单未覆盖旧持久化成员", 409,
|
|
"vpc_delete_manifest_mismatch")
|
|
|
|
# Already-proven, omitted hosts must not be contacted again.
|
|
# Excluding them from the transition plan prevents the generic
|
|
# desired-record builder from recreating a retiring host row.
|
|
if previous_plan is not None:
|
|
transition_previous = replace(
|
|
previous_plan,
|
|
hosts=tuple(host for host in previous_plan.hosts
|
|
if host.host_id not in missing),
|
|
)
|
|
previous_hosts = {
|
|
host_id: host for host_id, host in old_hosts.items()
|
|
if host_id in requested_host_ids
|
|
}
|
|
gate = self._fabric_gate(plan)
|
|
record = self._desired_record(
|
|
plan, gate, state="deleting",
|
|
observed_revision=previous_observed_revision,
|
|
observed_digest=previous_observed_digest,
|
|
previous_hosts=previous_hosts,
|
|
previous_plan=transition_previous,
|
|
)
|
|
retiring_rows_by_host: dict[str, list[dict[str, str]]] = {}
|
|
for row in _canonical_delete_members(retiring_members):
|
|
retiring_rows_by_host.setdefault(
|
|
row["host_id"], []).append(row)
|
|
for host in record.get("hosts", []):
|
|
if (isinstance(host, dict)
|
|
and isinstance(host.get("host_id"), str)):
|
|
host["retiring_members"] = list(
|
|
retiring_rows_by_host.get(host["host_id"], []))
|
|
marker = {
|
|
"operation_id": plan.operation_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"all_host_ids": sorted(old_host_ids | requested_host_ids),
|
|
"required_host_ids": sorted(required_host_ids),
|
|
"preverified_host_ids": sorted(preverified),
|
|
"retiring_members": _canonical_delete_members(
|
|
retiring_members),
|
|
}
|
|
record["delete_target"] = marker
|
|
record["state"] = "deleting"
|
|
record["next_stage_allowed"] = False
|
|
record["private_dataplane_verified"] = False
|
|
record["updated_at"] = int(time.time())
|
|
# Persist the target before any SSH/libvirt mutation. A crash
|
|
# can then resume the exact same operation and host closure.
|
|
self._save("vpc_desired", vpc_id, record)
|
|
self._save_children(record)
|
|
else:
|
|
if (marker["operation_id"] != plan.operation_id
|
|
or marker["desired_revision"] != plan.desired_revision
|
|
or marker["desired_digest"] != plan.desired_digest):
|
|
raise VpcError("VPC 删除目标已由另一操作占用", 409,
|
|
"vpc_delete_receipt_conflict")
|
|
durable_plan = _plan_from_record(previous)
|
|
if (durable_plan.operation_id != plan.operation_id
|
|
or durable_plan.desired_revision != plan.desired_revision
|
|
or durable_plan.desired_digest != plan.desired_digest
|
|
or durable_plan.canonical() != plan.canonical()):
|
|
raise VpcError("VPC 删除持久化目标不一致", 500,
|
|
"vpc_record_corrupt")
|
|
if (not set(marker["required_host_ids"]).issubset(plan.host_ids)
|
|
or not plan.host_ids.issubset(
|
|
set(marker["all_host_ids"]))):
|
|
raise VpcError("VPC 删除持久化宿主机闭集不一致", 500,
|
|
"vpc_record_corrupt")
|
|
if marker["retiring_members"] != _canonical_delete_members(
|
|
retiring_members):
|
|
raise VpcError("VPC 删除成员清单与持久化目标不一致", 500,
|
|
"vpc_record_corrupt")
|
|
record = dict(previous)
|
|
transition_previous = durable_plan
|
|
|
|
required_host_ids = set(marker["required_host_ids"])
|
|
observed_hosts = self._host_closure(record)
|
|
already_confirmed = {
|
|
host_id for host_id in required_host_ids
|
|
if (host_id in observed_hosts
|
|
and self._confirmed_removal(
|
|
observed_hosts[host_id], plan.desired_revision,
|
|
plan.desired_digest))
|
|
}
|
|
pending_host_ids = required_host_ids - already_confirmed
|
|
if pending_host_ids:
|
|
gate = self._fabric_gate(plan)
|
|
# The submitted plan may still contain a deployment whose
|
|
# prior remove response was lost by PHP. Its exact durable
|
|
# absence proof classifies it as preverified; never contact it
|
|
# again. Reconcile a transient projection containing only
|
|
# genuinely pending host rows, then merge those observations
|
|
# back into the durable submitted plan.
|
|
execution_record = dict(record)
|
|
execution_hosts: list[dict[str, Any]] = []
|
|
retiring_rows_by_host: dict[str, list[dict[str, str]]] = {}
|
|
for row in marker["retiring_members"]:
|
|
retiring_rows_by_host.setdefault(
|
|
row["host_id"], []).append(dict(row))
|
|
for host in record.get("hosts", []):
|
|
if (not isinstance(host, Mapping)
|
|
or host.get("host_id") not in pending_host_ids):
|
|
continue
|
|
pending = dict(host)
|
|
# Persisted delete targets retain the submitted canonical
|
|
# host list, while the transient retiring marker forces
|
|
# the removal lane even after recover() cleared verified
|
|
# state and retiring_members had already been consumed.
|
|
pending["retiring"] = True
|
|
pending["retiring_members"] = list(
|
|
retiring_rows_by_host.get(pending["host_id"], []))
|
|
execution_hosts.append(pending)
|
|
execution_record["hosts"] = execution_hosts
|
|
await self._reconcile_dataplane(
|
|
plan, execution_record, gate, transition_previous,
|
|
operation_id=plan.operation_id)
|
|
observations = {
|
|
host.get("host_id"): {
|
|
key: value for key, value in host.items()
|
|
if key != "retiring"
|
|
}
|
|
for host in execution_record.get("hosts", [])
|
|
if (isinstance(host, Mapping)
|
|
and isinstance(host.get("host_id"), str))
|
|
}
|
|
record["hosts"] = [
|
|
dict(observations[host["host_id"]])
|
|
if (isinstance(host, Mapping)
|
|
and host.get("host_id") in observations)
|
|
else host
|
|
for host in record.get("hosts", [])
|
|
]
|
|
# Preserve partial/uncertain evidence before deciding whether
|
|
# a receipt may be issued.
|
|
self._save("vpc_desired", vpc_id, record)
|
|
self._save_children(record)
|
|
observed_hosts = self._host_closure(record)
|
|
if not required_host_ids.issubset(observed_hosts) or not all(
|
|
self._confirmed_removal(
|
|
observed_hosts[host_id], plan.desired_revision,
|
|
plan.desired_digest)
|
|
for host_id in required_host_ids
|
|
):
|
|
record["state"] = "uncertain"
|
|
record["next_stage_allowed"] = False
|
|
record["private_dataplane_verified"] = False
|
|
record["updated_at"] = int(time.time())
|
|
self._save("vpc_desired", vpc_id, record)
|
|
deterministic = [
|
|
(host_id, observed_hosts[host_id].get("last_error_code"))
|
|
for host_id in sorted(required_host_ids)
|
|
if (host_id in observed_hosts
|
|
and observed_hosts[host_id].get("last_error_code")
|
|
in VPC_DELETE_DETERMINISTIC_HOST_ERRORS)
|
|
]
|
|
if deterministic:
|
|
raise VpcError(
|
|
"VPC 宿主机存在需要用户处理的确定性清理阻断",
|
|
409, str(deterministic[0][1]))
|
|
raise VpcError("VPC 宿主机清理尚未全部确认", 503,
|
|
"vpc_delete_incomplete")
|
|
|
|
receipt = {
|
|
"schema_version": VPC_SCHEMA_VERSION,
|
|
"evidence_version": VPC_EVIDENCE_VERSION,
|
|
"vpc_id": vpc_id,
|
|
"operation_id": plan.operation_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"state": "deleted",
|
|
"deleted": True,
|
|
"readback": "confirmed",
|
|
"removed_host_ids": list(marker["all_host_ids"]),
|
|
"completed_at": int(time.time()),
|
|
}
|
|
# This ordering is the idempotency boundary. Never erase the
|
|
# evidence needed by a retry before its exact receipt is durable.
|
|
self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, receipt)
|
|
self._purge_deleted_vpc(vpc_id)
|
|
return self._public_delete_receipt(receipt)
|
|
|
|
# Root delete mutates the same CIDR/endpoint catalog as apply. Keep
|
|
# the global order per-VPC -> catalog, and do not release either lock
|
|
# while a cancelled coroutine still has a blocking host mutation or
|
|
# durable receipt commit in flight.
|
|
async with self._lock(vpc_id), self._catalog_mutation_lock():
|
|
return await _await_mutation_completion(delete_locked())
|
|
|
|
async def apply(self, value: Mapping[str, Any]) -> dict[str, Any]:
|
|
plan = parse_vpc_plan(value)
|
|
async with self._lock(plan.vpc_id), self._catalog_mutation_lock():
|
|
return await _await_mutation_completion(
|
|
self._apply_locked(plan))
|
|
|
|
async def _apply_locked(self, plan: VpcPlan) -> dict[str, Any]:
|
|
if self._load_delete_receipt(plan.vpc_id) is not None:
|
|
raise VpcError("VPC 已删除,不能重新应用旧标识", 409,
|
|
"vpc_deleted")
|
|
self._validate_global_fabric_endpoints(plan)
|
|
previous = self._load("vpc_desired", plan.vpc_id)
|
|
previous_plan: VpcPlan | None = None
|
|
old_revision = plan.desired_revision
|
|
if previous is not None:
|
|
if self._delete_marker(previous) is not None:
|
|
raise VpcError("VPC 正在删除,不能应用普通目标", 409,
|
|
"vpc_deleting")
|
|
# Validate the durable prior target before accepting a newer
|
|
# revision. Otherwise a malformed record could be silently
|
|
# overwritten by a tenant retry, erasing the evidence needed
|
|
# to reconcile an interrupted operation.
|
|
previous_plan = _plan_from_record(previous)
|
|
old_revision = previous.get("desired_revision")
|
|
old_digest = previous.get("desired_digest")
|
|
if (old_revision != previous_plan.desired_revision
|
|
or old_digest != previous_plan.desired_digest):
|
|
raise VpcError("VPC 持久化目标摘要不一致", 500,
|
|
"vpc_record_corrupt")
|
|
if isinstance(old_revision, int) and plan.desired_revision < old_revision:
|
|
raise VpcError("VPC desired_revision 不能回退", 409,
|
|
"vpc_revision_conflict")
|
|
if plan.desired_revision == old_revision:
|
|
if old_digest != plan.desired_digest:
|
|
raise VpcError("相同 desired_revision 的 VPC 计划已存在", 409,
|
|
"vpc_revision_conflict")
|
|
# Preserve exact host absence proofs monotonically. A
|
|
# partial pass may have removed host A while host B was
|
|
# unavailable; replay must retry only B instead of making
|
|
# the already removed host reachable again as a condition
|
|
# of convergence.
|
|
gate = self._fabric_gate(plan)
|
|
self._require_fabric_catalog_admission(gate)
|
|
record = dict(previous)
|
|
await self._reconcile_pending_hosts(
|
|
plan, record, gate, previous_plan)
|
|
self._save("vpc_desired", plan.vpc_id, record)
|
|
self._save_children(record)
|
|
return self._public(record)
|
|
gate = self._fabric_gate(plan)
|
|
self._require_fabric_catalog_admission(gate)
|
|
state = "applying" if gate.get("active") else (
|
|
"degraded" if gate.get("available") else "uncertain")
|
|
previous_hosts = {
|
|
item.get("host_id"): item
|
|
for item in (previous or {}).get("hosts", [])
|
|
if isinstance(item, Mapping) and isinstance(item.get("host_id"), str)
|
|
}
|
|
previous_observed_revision = (previous or {}).get("observed_revision", 0)
|
|
previous_observed_digest = (previous or {}).get("observed_digest")
|
|
if (isinstance(previous_observed_revision, bool)
|
|
or not isinstance(previous_observed_revision, int)
|
|
or previous_observed_revision < 0
|
|
or previous_observed_revision > old_revision
|
|
or previous_observed_revision > plan.desired_revision):
|
|
raise VpcError("VPC 持久化观测版本无效", 500,
|
|
"vpc_record_corrupt")
|
|
if previous_observed_digest is not None:
|
|
try:
|
|
previous_observed_digest = _digest(
|
|
previous_observed_digest, "observed_digest")
|
|
except VpcError as exc:
|
|
raise VpcError("VPC 持久化观测摘要无效", 500,
|
|
"vpc_record_corrupt") from exc
|
|
for old_host in previous_hosts.values():
|
|
old_host_revision = old_host.get("observed_revision", 0)
|
|
if (isinstance(old_host_revision, bool)
|
|
or not isinstance(old_host_revision, int)
|
|
or old_host_revision < 0
|
|
or old_host_revision > old_revision):
|
|
raise VpcError("VPC 宿主观测版本无效", 500,
|
|
"vpc_record_corrupt")
|
|
old_host_digest = old_host.get("observed_digest")
|
|
if old_host_digest is not None:
|
|
try:
|
|
_digest(old_host_digest, "observed_digest")
|
|
except VpcError as exc:
|
|
raise VpcError("VPC 宿主观测摘要无效", 500,
|
|
"vpc_record_corrupt") from exc
|
|
record = self._desired_record(
|
|
plan, gate, state=state,
|
|
observed_revision=previous_observed_revision,
|
|
observed_digest=previous_observed_digest,
|
|
previous_hosts=previous_hosts,
|
|
previous_plan=previous_plan,
|
|
)
|
|
self._finish(record, gate)
|
|
await self._reconcile_dataplane(
|
|
plan, record, gate, previous_plan)
|
|
self._save("vpc_desired", plan.vpc_id, record)
|
|
self._save_children(record)
|
|
return self._public(record)
|
|
|
|
async def observe(
|
|
self, vpc_id: str, observations: Mapping[str, Mapping[str, Any]],
|
|
) -> dict[str, Any]:
|
|
"""Record typed future data-plane readback for one VPC.
|
|
|
|
This method is intentionally not wired to the public HTTP body. A
|
|
later private reconciler may call it after reading host state. It
|
|
accepts only revision/digest/state/readback scalars and never shell or
|
|
raw nft/libvirt text.
|
|
"""
|
|
_identifier(vpc_id, "VPC")
|
|
if not isinstance(observations, Mapping):
|
|
raise VpcError("VPC observed 状态必须是对象")
|
|
async def observe_locked() -> dict[str, Any]:
|
|
record = self._load("vpc_desired", vpc_id)
|
|
if record is None:
|
|
raise VpcError("VPC 不存在", 404, "vpc_not_found")
|
|
plan = _plan_from_record(record)
|
|
expected_hosts = {item.host_id for item in plan.hosts}
|
|
if set(observations) - expected_hosts:
|
|
raise VpcError("VPC observed 包含未知宿主机", 422,
|
|
"vpc_observed_host_invalid")
|
|
for host in record.get("hosts", []):
|
|
if not isinstance(host, dict):
|
|
continue
|
|
if (host.get("host_id") not in {
|
|
member.host_id for member in plan.members}
|
|
and self._confirmed_removal(
|
|
host, plan.desired_revision,
|
|
plan.desired_digest)):
|
|
# A missing or stale external observation cannot revoke a
|
|
# durable exact absence proof for the same immutable plan.
|
|
continue
|
|
raw = observations.get(host.get("host_id"))
|
|
if raw is None:
|
|
host["state"] = "uncertain"
|
|
host["transport_verified"] = False
|
|
host["private_dataplane_verified"] = False
|
|
continue
|
|
if not isinstance(raw, Mapping):
|
|
raise VpcError("VPC observed 宿主记录无效")
|
|
observed_revision = raw.get("observed_revision")
|
|
observed_digest = raw.get("observed_digest")
|
|
state = raw.get("state")
|
|
transport = raw.get("transport_verified")
|
|
private_dataplane = raw.get("private_dataplane_verified", False)
|
|
if (isinstance(observed_revision, bool)
|
|
or not isinstance(observed_revision, int)
|
|
or observed_revision < 0):
|
|
raise VpcError("VPC observed_revision 无效")
|
|
if observed_revision > plan.desired_revision:
|
|
raise VpcError("VPC observed_revision 不能超过 desired_revision",
|
|
409, "vpc_revision_conflict")
|
|
if observed_digest is not None:
|
|
observed_digest = _digest(observed_digest, "observed_digest")
|
|
if state not in VPC_STATES:
|
|
raise VpcError("VPC observed state 无效")
|
|
if not isinstance(transport, bool):
|
|
raise VpcError("VPC observed transport_verified 无效")
|
|
if not isinstance(private_dataplane, bool):
|
|
raise VpcError("VPC observed private_dataplane_verified 无效")
|
|
if state == "active" and (transport is not True
|
|
or observed_digest is None):
|
|
raise VpcError(
|
|
"VPC active 观测缺少传输或摘要证据", 422,
|
|
"vpc_observation_invalid")
|
|
# The current Fabric adapter reports transport/readback before
|
|
# the tenant private data-plane adapter exists. Preserve that
|
|
# observation as an applying gate instead of rejecting the
|
|
# whole reconciliation request: transport is useful progress,
|
|
# but it must never open the VPC or be reported as active.
|
|
observation_error = raw.get("error_code")
|
|
if state == "active" and private_dataplane is not True:
|
|
state = "applying"
|
|
observation_error = "vpc_private_dataplane_unverified"
|
|
readback = raw.get("readback", {})
|
|
if isinstance(readback, str) and len(readback) <= 128:
|
|
safe_readback: Any = readback
|
|
elif isinstance(readback, Mapping) and len(readback) <= 64:
|
|
safe_readback = {
|
|
str(key): value for key, value in readback.items()
|
|
if isinstance(key, str) and len(key) <= 64
|
|
and _safe_scalar(value)
|
|
}
|
|
else:
|
|
raise VpcError("VPC readback 无效")
|
|
typed_fields = (
|
|
"vpc_id", "host_id", "desired_revision", "desired_digest",
|
|
"config_verified", "config_observed_revision",
|
|
"config_observed_digest",
|
|
"bridge", "bridge_owner", "nft_table", "anti_spoof",
|
|
"rp_filter", "proxy_arp", "proxy_arp_pvlan", "proxy_ips", "acl_revision", "rollback", "removed", "member_nics", "routes",
|
|
"wg_allowed_ips",
|
|
)
|
|
typed = {}
|
|
for field in typed_fields:
|
|
if field not in raw:
|
|
continue
|
|
item_value = raw[field]
|
|
if field in {"member_nics", "routes", "wg_allowed_ips", "proxy_ips"}:
|
|
if isinstance(item_value, (list, tuple)) and len(item_value) <= VPC_MAX_READBACK:
|
|
if field == "proxy_ips":
|
|
typed[field] = [str(ip)[:64] for ip in item_value
|
|
if isinstance(ip, str)]
|
|
else:
|
|
typed[field] = [dict(row) for row in item_value
|
|
if isinstance(row, Mapping)]
|
|
elif field == "readback":
|
|
continue
|
|
elif _safe_scalar(item_value):
|
|
typed[field] = item_value
|
|
host.update({"state": state,
|
|
"observed_revision": observed_revision,
|
|
"observed_digest": observed_digest,
|
|
"transport_verified": transport,
|
|
"private_dataplane_verified": private_dataplane,
|
|
"readback": safe_readback,
|
|
"last_error_code": observation_error
|
|
if isinstance(observation_error, str)
|
|
and len(observation_error) <= 64 else None})
|
|
host.update(typed)
|
|
gate = self._fabric_gate(plan)
|
|
self._finish(record, gate)
|
|
self._save("vpc_desired", vpc_id, record)
|
|
self._save_children(record)
|
|
return self._public(record)
|
|
|
|
# External observations may publish an active root from existing host
|
|
# evidence. Treat that publication as a catalog mutation too: pin the
|
|
# Fabric generation and every affected VM identity until the refreshed
|
|
# gate, root, and child projections are committed together.
|
|
async with self._lock(vpc_id), self._catalog_mutation_lock():
|
|
lock_record = self._load("vpc_desired", vpc_id)
|
|
if lock_record is None:
|
|
raise VpcError("VPC 不存在", 404, "vpc_not_found")
|
|
lock_plan = _plan_from_record(lock_record)
|
|
extra_identities = [
|
|
(host["host_id"], row["instance_uuid"])
|
|
for host in lock_record.get("hosts", [])
|
|
if (isinstance(host, Mapping)
|
|
and isinstance(host.get("host_id"), str))
|
|
for row in host.get("retiring_members", [])
|
|
if (isinstance(row, Mapping)
|
|
and isinstance(row.get("instance_uuid"), str))
|
|
]
|
|
async with self._instance_locks(
|
|
lock_plan, extra_identities=extra_identities):
|
|
return await _await_mutation_completion(observe_locked())
|
|
|
|
@staticmethod
|
|
def _instance_private_state(
|
|
instance_uuid: str, *, required: bool, vpc_id: str | None,
|
|
member_id: str | None, desired_state: str, private_ip_cidr: str | None,
|
|
private_mac: str | None, desired_revision: int, desired_digest: str | None,
|
|
host: Mapping[str, Any] | None, member_nic: Mapping[str, Any] | None,
|
|
reason_code: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Project bounded host/VPC evidence for one lifecycle gate."""
|
|
observed_revision = (
|
|
host.get("observed_revision", 0) if isinstance(host, Mapping) else 0)
|
|
observed_digest = (
|
|
host.get("observed_digest") if isinstance(host, Mapping) else None)
|
|
config_observed_revision = (
|
|
host.get("config_observed_revision", 0)
|
|
if isinstance(host, Mapping) else 0)
|
|
config_observed_digest = (
|
|
host.get("config_observed_digest")
|
|
if isinstance(host, Mapping) else None)
|
|
host_runtime_ok = bool(
|
|
isinstance(host, Mapping)
|
|
and host.get("state") == "active"
|
|
and host.get("private_dataplane_verified") is True
|
|
and observed_revision == desired_revision
|
|
and observed_digest == desired_digest
|
|
)
|
|
host_config_ok = bool(
|
|
isinstance(host, Mapping)
|
|
and host.get("transport_verified") is True
|
|
and host.get("config_verified") is True
|
|
and config_observed_revision == desired_revision
|
|
and config_observed_digest == desired_digest
|
|
)
|
|
# Host configuration is shared, while guest readiness is per member.
|
|
# A stopped or unobservable sibling must not erase a healthy member's
|
|
# independently confirmed runtime state.
|
|
host_ok = host_runtime_ok or host_config_ok
|
|
desired_mac: str | None = None
|
|
if (required and desired_state != "removed"
|
|
and isinstance(vpc_id, str) and isinstance(member_id, str)):
|
|
try:
|
|
desired_mac = _effective_private_mac(
|
|
vpc_id, member_id, private_mac)
|
|
except (TypeError, ValueError, VpcError):
|
|
# Keep the projection bounded and fail closed if a corrupt
|
|
# durable row contains an invalid MAC. The caller will
|
|
# expose an uncertain observation instead of publishing a
|
|
# guessed address.
|
|
desired_mac = None
|
|
observed_mac: str | None = None
|
|
guest_mac: str | None = None
|
|
guest_ip: str | None = None
|
|
if isinstance(member_nic, Mapping):
|
|
try:
|
|
observed_mac = _mac(member_nic.get("private_mac"))
|
|
except VpcError:
|
|
observed_mac = None
|
|
try:
|
|
guest_mac = _mac(member_nic.get("guest_private_mac"))
|
|
except VpcError:
|
|
guest_mac = None
|
|
if isinstance(member_nic.get("guest_private_ip_cidr"), str):
|
|
guest_ip = member_nic.get("guest_private_ip_cidr")
|
|
active_nic_ok = bool(
|
|
isinstance(member_nic, Mapping)
|
|
and member_nic.get("xml_verified") is True
|
|
and member_nic.get("seed_verified") is True
|
|
and member_nic.get("restart_required") is False
|
|
and member_nic.get("domain_running") is True
|
|
and member_nic.get("guest_network_verified") is True
|
|
and member_nic.get("guest_link_state") == "up"
|
|
and desired_mac is not None
|
|
and observed_mac == desired_mac
|
|
and guest_mac == desired_mac
|
|
and guest_ip == private_ip_cidr
|
|
)
|
|
link_state = (
|
|
member_nic.get("link_state") if isinstance(member_nic, Mapping)
|
|
else "absent")
|
|
if link_state not in {"up", "down"}:
|
|
link_state = "absent"
|
|
suspended_nic_ok = bool(
|
|
isinstance(member_nic, Mapping)
|
|
and member_nic.get("xml_verified") is True
|
|
and member_nic.get("config_verified") is True
|
|
and member_nic.get("seed_verified") is True
|
|
and member_nic.get("restart_required") is False
|
|
and member_nic.get("domain_running") is True
|
|
and link_state == "down"
|
|
and desired_mac is not None
|
|
and observed_mac == desired_mac
|
|
)
|
|
stopped_nic_ok = bool(
|
|
isinstance(member_nic, Mapping)
|
|
and member_nic.get("xml_verified") is True
|
|
and member_nic.get("config_verified") is True
|
|
and member_nic.get("seed_verified") is True
|
|
and member_nic.get("restart_required") is False
|
|
and member_nic.get("domain_running") is False
|
|
and member_nic.get("private_ip_cidr") == private_ip_cidr
|
|
and desired_mac is not None
|
|
and observed_mac == desired_mac
|
|
)
|
|
if not required:
|
|
state = "disabled" if not isinstance(member_nic, Mapping) else "uncertain"
|
|
return {
|
|
"version": 1, "required": False, "vm_uuid": instance_uuid,
|
|
"vpc_id": None, "member_id": None, "interface": None,
|
|
"state": state, "desired_state": "disabled",
|
|
"desired_revision": 0, "observed_revision": 0,
|
|
"desired_digest": None, "observed_digest": None,
|
|
"readback": "confirmed" if state == "disabled" else "mismatch",
|
|
"link_state": "absent", "private_ip_cidr": None,
|
|
"private_mac": None,
|
|
"reason_code": None if state == "disabled" else reason_code
|
|
or "vpc_private_observation_mismatch",
|
|
"observed_at": int(time.time()),
|
|
"xml_verified": False, "config_verified": False,
|
|
"seed_verified": False, "domain_running": False,
|
|
"restart_required": False,
|
|
"guest_network_verified": False,
|
|
"guest_network_readback": "missing",
|
|
"guest_link_state": None,
|
|
}
|
|
removed = desired_state == "removed"
|
|
if removed:
|
|
shape_ok = not isinstance(member_nic, Mapping)
|
|
state = "removed" if host_ok and shape_ok else "uncertain"
|
|
interface = None
|
|
link = "absent"
|
|
ip = None
|
|
mac = None
|
|
readback = "confirmed" if state == "removed" else "missing"
|
|
else:
|
|
# An intentional live link-down is an isolation state, not a
|
|
# usable-network claim. Exact persistent/live XML plus the host
|
|
# config proof is sufficient to confirm suspension; requiring a
|
|
# guest link-up probe here would make network_suspend impossible.
|
|
shape_ok = (stopped_nic_ok if stopped_nic_ok
|
|
else suspended_nic_ok if link_state == "down"
|
|
else active_nic_ok)
|
|
state = (
|
|
"stopped" if stopped_nic_ok
|
|
else "suspended" if link_state == "down" else "active"
|
|
) if host_ok and shape_ok else "uncertain"
|
|
interface = "private0" if shape_ok else None
|
|
link = link_state
|
|
ip = (private_ip_cidr
|
|
if stopped_nic_ok or suspended_nic_ok else guest_ip) \
|
|
if shape_ok else None
|
|
# Return the value actually present in the bounded host
|
|
# observation. Equality with the derived/configured target above
|
|
# is part of ``shape_ok``; this prevents a non-empty but stale
|
|
# desired MAC from masking host XML drift.
|
|
mac = (observed_mac
|
|
if stopped_nic_ok or suspended_nic_ok else guest_mac) \
|
|
if shape_ok else None
|
|
readback = "confirmed" if host_ok and shape_ok else "missing"
|
|
return {
|
|
"version": 1, "required": True, "vm_uuid": instance_uuid,
|
|
"vpc_id": vpc_id, "member_id": member_id,
|
|
"interface": interface, "state": state,
|
|
"desired_state": desired_state,
|
|
"desired_revision": desired_revision,
|
|
"observed_revision": (
|
|
observed_revision if host_runtime_ok
|
|
else config_observed_revision if host_config_ok else 0),
|
|
"desired_digest": desired_digest,
|
|
"observed_digest": (
|
|
observed_digest if host_runtime_ok
|
|
else config_observed_digest if host_config_ok else None),
|
|
"readback": readback, "link_state": link,
|
|
"private_ip_cidr": ip, "private_mac": mac,
|
|
"reason_code": (
|
|
"vpc_private_guest_stopped"
|
|
if state == "stopped" else
|
|
None if readback == "confirmed" else reason_code
|
|
or ("vpc_private_teardown_uncertain"
|
|
if removed else "vpc_private_observation_mismatch")),
|
|
"observed_at": int(time.time()),
|
|
"xml_verified": (
|
|
member_nic.get("xml_verified") is True
|
|
if isinstance(member_nic, Mapping) else False),
|
|
"config_verified": (
|
|
member_nic.get("config_verified") is True
|
|
if isinstance(member_nic, Mapping) else False),
|
|
"seed_verified": (
|
|
member_nic.get("seed_verified") is True
|
|
if isinstance(member_nic, Mapping) else False),
|
|
"domain_running": (
|
|
member_nic.get("domain_running") is True
|
|
if isinstance(member_nic, Mapping) else False),
|
|
"restart_required": (
|
|
member_nic.get("restart_required") is True
|
|
if isinstance(member_nic, Mapping) else False),
|
|
"guest_network_verified": (
|
|
member_nic.get("guest_network_verified") is True
|
|
if isinstance(member_nic, Mapping) else False),
|
|
"guest_network_readback": (
|
|
str(member_nic.get("guest_network_readback"))[:32]
|
|
if isinstance(member_nic, Mapping)
|
|
and isinstance(member_nic.get("guest_network_readback"), str)
|
|
else "missing"),
|
|
"guest_link_state": (
|
|
member_nic.get("guest_link_state")
|
|
if isinstance(member_nic, Mapping)
|
|
and member_nic.get("guest_link_state") in {"up", "down"}
|
|
else None),
|
|
}
|
|
|
|
async def reconcile_instance(
|
|
self, instance_uuid: str, *, host_id: str | None = None,
|
|
link_state: str = "preserve", expected: Mapping[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Converge and prove one VM's private dataplane.
|
|
|
|
Lifecycle workers call this idempotent lane after a VM mutation. It
|
|
reuses the same VPC desired record and host adapter as the periodic
|
|
reconciler, so the returned evidence covers the private bridge, exact
|
|
routes, WireGuard AllowedIPs, nft ACL/anti-spoof policy and the
|
|
persistent/live libvirt NIC XML together. XML alone is never marked
|
|
confirmed.
|
|
"""
|
|
instance_uuid = _uuid(instance_uuid)
|
|
if link_state not in {"preserve", "up", "down", "absent"}:
|
|
raise VpcError("私网生命周期链路状态无效", 422,
|
|
"vpc_private_link_state_invalid")
|
|
expected = expected if isinstance(expected, Mapping) else {}
|
|
expected_required = expected.get("required")
|
|
expected_vpc = expected.get("vpc_id")
|
|
expected_member = expected.get("member_id")
|
|
expected_state = expected.get("desired_state")
|
|
lifecycle_kind = expected.get("lifecycle_kind")
|
|
expected_revision = expected.get("desired_revision")
|
|
expected_digest = expected.get("desired_digest")
|
|
expected_operation_id = expected.get("operation_id")
|
|
if lifecycle_kind is None:
|
|
lifecycle_kind = "legacy"
|
|
elif (not isinstance(lifecycle_kind, str)
|
|
or _PRIVATE_LIFECYCLE_KIND.fullmatch(lifecycle_kind) is None):
|
|
raise VpcError("私网生命周期类型无效", 422,
|
|
"vpc_private_lifecycle_kind_invalid")
|
|
lifecycle_link_invalid = (
|
|
(lifecycle_kind in {"delete", "expiry_delete"}
|
|
and link_state != "absent")
|
|
or (lifecycle_kind == "network_suspend"
|
|
and link_state not in {"down", "preserve"})
|
|
or (lifecycle_kind == "network_resume"
|
|
and link_state not in {"up", "preserve"})
|
|
or (lifecycle_kind not in {
|
|
"delete", "expiry_delete", "network_suspend",
|
|
"network_resume"}
|
|
and link_state != "preserve")
|
|
)
|
|
if lifecycle_kind != "legacy" and lifecycle_link_invalid:
|
|
raise VpcError("私网生命周期策略与链路目标不一致", 422,
|
|
"vpc_private_lifecycle_policy_invalid")
|
|
if expected_vpc is not None and not isinstance(expected_vpc, str):
|
|
raise VpcError("私网 VPC 标识无效", 422)
|
|
if expected_member is not None and not isinstance(expected_member, str):
|
|
raise VpcError("私网成员标识无效", 422)
|
|
if expected_state is not None and expected_state not in {"active", "removed"}:
|
|
raise VpcError("私网目标状态无效", 422)
|
|
if expected_revision is not None and (
|
|
isinstance(expected_revision, bool)
|
|
or not isinstance(expected_revision, int)
|
|
or expected_revision < 0):
|
|
raise VpcError("私网目标 revision 无效", 422)
|
|
if expected_digest is not None and (
|
|
not isinstance(expected_digest, str)
|
|
or _DIGEST.fullmatch(expected_digest) is None):
|
|
raise VpcError("私网目标摘要无效", 422)
|
|
if expected_operation_id is not None and (
|
|
not isinstance(expected_operation_id, str)
|
|
or _OPERATION_ID.fullmatch(expected_operation_id) is None):
|
|
raise VpcError("私网操作编号无效", 422)
|
|
# Validate the immutable MAC expectation at the engine boundary too;
|
|
# PHP validation is not a substitute for a host-side authorization
|
|
# check. A missing value is allowed for old ledgers and is filled by
|
|
# the same deterministic derivation used by the dataplane parser.
|
|
expected_private_mac = _mac(expected.get("private_mac"))
|
|
|
|
candidates: list[tuple[str, dict[str, Any], VpcPlan, VpcMember, bool]] = []
|
|
retiring: list[tuple[str, dict[str, Any], VpcPlan, Mapping[str, Any]]] = []
|
|
corrupt = False
|
|
for key, raw in self.store.list("vpc_desired").items():
|
|
try:
|
|
record = json.loads(raw)
|
|
plan = _plan_from_record(record)
|
|
except (TypeError, ValueError, VpcError):
|
|
corrupt = True
|
|
continue
|
|
for member in plan.members:
|
|
if member.instance_uuid == instance_uuid:
|
|
candidates.append((key, record, plan, member, False))
|
|
for host in record.get("hosts", []):
|
|
if not isinstance(host, Mapping):
|
|
continue
|
|
rows = host.get("retiring_members", [])
|
|
if not isinstance(rows, list):
|
|
continue
|
|
for row in rows:
|
|
if (isinstance(row, Mapping)
|
|
and row.get("instance_uuid") == instance_uuid):
|
|
retiring.append((key, record, plan, row))
|
|
if len(candidates) + len(retiring) > 1:
|
|
return self._instance_private_state(
|
|
instance_uuid, required=True, vpc_id=expected_vpc,
|
|
member_id=expected_member, desired_state=expected_state or "active",
|
|
private_ip_cidr=expected.get("private_ip_cidr"),
|
|
private_mac=expected_private_mac,
|
|
desired_revision=int(expected_revision or 0),
|
|
desired_digest=expected_digest, host=None, member_nic=None,
|
|
reason_code="vpc_private_expectation_ambiguous")
|
|
if not candidates and not retiring:
|
|
# A malformed VPC ledger must not be hidden as a public-only VM
|
|
# when the worker supplied an attached expectation.
|
|
if expected_required is True or corrupt:
|
|
return self._instance_private_state(
|
|
instance_uuid, required=True, vpc_id=expected_vpc,
|
|
member_id=expected_member, desired_state=expected_state or "active",
|
|
private_ip_cidr=expected.get("private_ip_cidr"),
|
|
private_mac=expected_private_mac,
|
|
desired_revision=int(expected_revision or 0),
|
|
desired_digest=expected_digest, host=None, member_nic=None,
|
|
reason_code=("vpc_private_record_corrupt" if corrupt
|
|
else "vpc_private_expectation_unavailable"))
|
|
return self._instance_private_state(
|
|
instance_uuid, required=False, vpc_id=None, member_id=None,
|
|
desired_state="disabled", private_ip_cidr=None, private_mac=None,
|
|
desired_revision=0, desired_digest=None, host=None, member_nic=None)
|
|
|
|
if candidates:
|
|
key, _old_record, _old_plan, _member, _ = candidates[0]
|
|
desired_state = "active"
|
|
else:
|
|
key, _old_record, _old_plan, _row = retiring[0]
|
|
desired_state = "removed"
|
|
async def reconcile_instance_locked() -> dict[str, Any]:
|
|
record = self._load("vpc_desired", key)
|
|
if record is None:
|
|
return self._instance_private_state(
|
|
instance_uuid, required=True, vpc_id=expected_vpc,
|
|
member_id=expected_member, desired_state=desired_state,
|
|
private_ip_cidr=expected.get("private_ip_cidr"),
|
|
private_mac=expected_private_mac,
|
|
desired_revision=int(expected_revision or 0),
|
|
desired_digest=expected_digest, host=None, member_nic=None,
|
|
reason_code="vpc_private_expectation_unavailable")
|
|
plan = _plan_from_record(record)
|
|
member = next((item for item in plan.members
|
|
if item.instance_uuid == instance_uuid), None)
|
|
target_row: Mapping[str, Any] | None = None
|
|
target_host_id = host_id
|
|
for host in record.get("hosts", []):
|
|
if not isinstance(host, Mapping):
|
|
continue
|
|
if member is not None and host.get("host_id") == member.host_id:
|
|
target_host_id = member.host_id
|
|
for row in host.get("retiring_members", []):
|
|
if (isinstance(row, Mapping)
|
|
and row.get("instance_uuid") == instance_uuid):
|
|
target_row = row
|
|
target_host_id = str(host.get("host_id"))
|
|
if member is None and target_row is None:
|
|
return self._instance_private_state(
|
|
instance_uuid, required=True, vpc_id=expected_vpc,
|
|
member_id=expected_member, desired_state="removed",
|
|
private_ip_cidr=expected.get("private_ip_cidr"),
|
|
private_mac=expected_private_mac,
|
|
desired_revision=int(expected_revision or 0),
|
|
desired_digest=expected_digest, host=None, member_nic=None,
|
|
reason_code="vpc_private_expectation_stale")
|
|
target_vpc_id = plan.vpc_id
|
|
target_member_id = (member.member_id if member is not None
|
|
else target_row.get("member_id") if target_row else None)
|
|
target_ip = (f"{member.private_ip}/32" if member is not None
|
|
else target_row.get("private_ip_cidr") if target_row else None)
|
|
try:
|
|
target_mac = (_effective_private_mac(
|
|
plan.vpc_id, target_member_id, member.private_mac)
|
|
if member is not None else _effective_private_mac(
|
|
plan.vpc_id, target_member_id,
|
|
target_row.get("private_mac") if target_row else None))
|
|
except (TypeError, ValueError, VpcError):
|
|
return self._instance_private_state(
|
|
instance_uuid, required=True, vpc_id=target_vpc_id,
|
|
member_id=target_member_id, desired_state=(
|
|
"active" if member is not None else "removed"),
|
|
private_ip_cidr=target_ip, private_mac=None,
|
|
desired_revision=plan.desired_revision,
|
|
desired_digest=plan.desired_digest, host=None, member_nic=None,
|
|
reason_code="vpc_private_mac_invalid")
|
|
immutable_mismatch = (
|
|
(expected_vpc is not None and expected_vpc != target_vpc_id)
|
|
or (expected_member is not None and expected_member != target_member_id)
|
|
or (expected_revision is not None
|
|
and expected_revision != plan.desired_revision)
|
|
or (expected_digest is not None
|
|
and expected_digest != plan.desired_digest)
|
|
or (expected.get("private_ip_cidr") is not None
|
|
and expected.get("private_ip_cidr") != target_ip)
|
|
or (expected_private_mac is not None
|
|
and expected_private_mac != target_mac)
|
|
)
|
|
if immutable_mismatch:
|
|
return self._instance_private_state(
|
|
instance_uuid, required=True, vpc_id=target_vpc_id,
|
|
member_id=target_member_id,
|
|
desired_state="active" if member is not None else "removed",
|
|
private_ip_cidr=target_ip, private_mac=target_mac,
|
|
desired_revision=plan.desired_revision,
|
|
desired_digest=plan.desired_digest, host=None, member_nic=None,
|
|
reason_code="vpc_private_expectation_mismatch")
|
|
if host_id is not None and target_host_id != host_id:
|
|
return self._instance_private_state(
|
|
instance_uuid, required=True,
|
|
vpc_id=member and plan.vpc_id or expected_vpc,
|
|
member_id=member and member.member_id or expected_member,
|
|
desired_state="active" if member else "removed",
|
|
private_ip_cidr=(f"{member.private_ip}/32" if member
|
|
else target_row.get("private_ip_cidr")),
|
|
private_mac=target_mac,
|
|
desired_revision=plan.desired_revision,
|
|
desired_digest=plan.desired_digest, host=None, member_nic=None,
|
|
reason_code="vpc_private_host_mismatch")
|
|
if member is not None:
|
|
if link_state == "absent" or expected_state == "removed":
|
|
# The control plane has not yet produced a removal target;
|
|
# never reinterpret a delete request as a harmless link
|
|
# preservation while the active member is still desired.
|
|
return self._instance_private_state(
|
|
instance_uuid, required=True, vpc_id=plan.vpc_id,
|
|
member_id=member.member_id, desired_state="removed",
|
|
private_ip_cidr=f"{member.private_ip}/32",
|
|
private_mac=target_mac,
|
|
desired_revision=plan.desired_revision,
|
|
desired_digest=plan.desired_digest, host=None,
|
|
member_nic=None, reason_code="vpc_private_teardown_pending")
|
|
if (expected_vpc is not None and expected_vpc != plan.vpc_id
|
|
or expected_member is not None
|
|
and expected_member != member.member_id):
|
|
return self._instance_private_state(
|
|
instance_uuid, required=True, vpc_id=plan.vpc_id,
|
|
member_id=member.member_id, desired_state="active",
|
|
private_ip_cidr=f"{member.private_ip}/32",
|
|
private_mac=target_mac,
|
|
desired_revision=plan.desired_revision,
|
|
desired_digest=plan.desired_digest, host=None,
|
|
member_nic=None, reason_code="vpc_private_expectation_mismatch")
|
|
# Keep the target UUID in the map even for preserve. The
|
|
# host lane uses membership to prioritize this lifecycle VM
|
|
# ahead of a large periodic member batch without changing its
|
|
# link policy.
|
|
link_map = {instance_uuid: link_state}
|
|
gate = self._fabric_gate(plan)
|
|
reconcile_kwargs: dict[str, Any] = {
|
|
"private_link_states": link_map,
|
|
}
|
|
if expected_operation_id is not None:
|
|
reconcile_kwargs["operation_id"] = expected_operation_id
|
|
await self._reconcile_pending_hosts(
|
|
plan, record, gate, plan, **reconcile_kwargs)
|
|
self._save("vpc_desired", key, record)
|
|
self._save_children(record)
|
|
desired = member
|
|
else:
|
|
# A removed member is represented by the host's durable
|
|
# retiring_members tombstone. The same host adapter pass
|
|
# removes private0 and proves the bridge/ACL state afterwards.
|
|
gate = self._fabric_gate(plan)
|
|
reconcile_kwargs: dict[str, Any] = {}
|
|
if expected_operation_id is not None:
|
|
reconcile_kwargs["operation_id"] = expected_operation_id
|
|
await self._reconcile_pending_hosts(
|
|
plan, record, gate, plan, **reconcile_kwargs)
|
|
self._save("vpc_desired", key, record)
|
|
self._save_children(record)
|
|
desired = None
|
|
host_row = next((item for item in record.get("hosts", [])
|
|
if isinstance(item, Mapping)
|
|
and item.get("host_id") == target_host_id), None)
|
|
nic = None
|
|
if isinstance(host_row, Mapping):
|
|
for item in host_row.get("member_nics", []):
|
|
if (isinstance(item, Mapping)
|
|
and item.get("instance_uuid") == instance_uuid):
|
|
nic = item
|
|
break
|
|
result = self._instance_private_state(
|
|
instance_uuid, required=True, vpc_id=plan.vpc_id,
|
|
member_id=(desired.member_id if desired is not None
|
|
else target_row.get("member_id") if target_row else expected_member),
|
|
desired_state="active" if desired is not None else "removed",
|
|
private_ip_cidr=(f"{desired.private_ip}/32" if desired is not None
|
|
else target_row.get("private_ip_cidr") if target_row else expected.get("private_ip_cidr")),
|
|
private_mac=target_mac,
|
|
desired_revision=plan.desired_revision,
|
|
desired_digest=plan.desired_digest,
|
|
host=host_row, member_nic=nic,
|
|
reason_code=(host_row.get("last_error_code")
|
|
if isinstance(host_row, Mapping) else None),
|
|
)
|
|
if result.get("readback") == "confirmed":
|
|
required_link: str | None = None
|
|
if link_state == "down":
|
|
required_link = "down"
|
|
elif link_state == "up":
|
|
required_link = "up"
|
|
elif link_state == "absent":
|
|
required_link = "absent"
|
|
elif lifecycle_kind.endswith("_policy_active"):
|
|
required_link = "up"
|
|
elif lifecycle_kind.endswith("_policy_blocked"):
|
|
required_link = "down"
|
|
policy_unready = lifecycle_kind.endswith("_policy_unready")
|
|
if (policy_unready
|
|
or required_link is not None
|
|
and result.get("link_state") != required_link):
|
|
result.update({
|
|
"state": "uncertain", "readback": "mismatch",
|
|
"interface": None, "private_ip_cidr": None,
|
|
"private_mac": None,
|
|
"reason_code":
|
|
"vpc_private_lifecycle_policy_mismatch",
|
|
"observed_at": int(time.time()),
|
|
})
|
|
return result
|
|
|
|
# Revalidate the optimistically discovered VPC under the canonical
|
|
# lock order. _reconcile_pending_hosts acquires the complete sorted
|
|
# active/retiring VM lock set before any libvirt or host mutation.
|
|
async with self._lock(key), self._catalog_mutation_lock():
|
|
return await _await_mutation_completion(
|
|
reconcile_instance_locked())
|
|
|
|
async def reconcile(self, vpc_id: str) -> dict[str, Any]:
|
|
_identifier(vpc_id, "VPC")
|
|
# Do not run a durable delete target through the ordinary apply
|
|
# reconciler: only ``delete`` can issue the exact positive receipt and
|
|
# purge the target after every required host reports absence. The
|
|
# first read is only an optimization; the row is rechecked under the
|
|
# VPC lock below to close the apply/delete race.
|
|
record = self._load("vpc_desired", vpc_id)
|
|
if record is not None and self._delete_marker(record) is not None:
|
|
return await self.delete(
|
|
vpc_id, self._delete_payload_from_record(record))
|
|
async def reconcile_locked(
|
|
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
record = self._load("vpc_desired", vpc_id)
|
|
if record is None:
|
|
raise VpcError("VPC 不存在", 404, "vpc_not_found")
|
|
if self._delete_marker(record) is not None:
|
|
return None, self._delete_payload_from_record(record)
|
|
plan = _plan_from_record(record)
|
|
if record.get("desired_digest") != plan.desired_digest:
|
|
raise VpcError("VPC 计划摘要与持久化记录不一致", 500,
|
|
"vpc_record_corrupt")
|
|
gate = self._fabric_gate(plan)
|
|
# Reconcile is the durable retry path after a host response
|
|
# was lost or the engine restarted. Refreshing only the
|
|
# Fabric gate would leave private0/routes/nft in an unknown
|
|
# state forever.
|
|
await self._reconcile_pending_hosts(
|
|
plan, record, gate, plan)
|
|
self._save("vpc_desired", vpc_id, record)
|
|
self._save_children(record)
|
|
return self._public(record), None
|
|
|
|
async with self._lock(vpc_id), self._catalog_mutation_lock():
|
|
result, delete_payload = await _await_mutation_completion(
|
|
reconcile_locked())
|
|
# The locked recheck may observe a delete target that was installed
|
|
# after the optimistic read. Release the non-reentrant lock before
|
|
# entering ``delete``, which acquires the same per-VPC and catalog
|
|
# locks itself.
|
|
if delete_payload is not None:
|
|
return await self.delete(vpc_id, delete_payload)
|
|
assert result is not None
|
|
return result
|
|
|
|
def recover(self) -> None:
|
|
"""Fail closed after an engine restart until readback is supplied."""
|
|
# Successful roots have no desired row left, but old deployments may
|
|
# still contain the pre-compaction receipt shape. Validate and reduce
|
|
# those tombstones before normal recovery; corrupt identities remain
|
|
# fail-closed for operator inspection.
|
|
for key in tuple(self.store.list(VPC_DELETE_RECEIPT_KIND)):
|
|
try:
|
|
self._load_delete_receipt(key)
|
|
except VpcError:
|
|
continue
|
|
for key, raw in self.store.list("vpc_desired").items():
|
|
try:
|
|
value = json.loads(raw)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if not isinstance(value, dict):
|
|
continue
|
|
if value.get("vpc_id") != key:
|
|
# Do not move or rewrite children under a different VPC key
|
|
# when a durable row has been tampered with.
|
|
continue
|
|
value["state"] = "uncertain"
|
|
value["next_stage_allowed"] = False
|
|
value["private_dataplane_verified"] = False
|
|
# A previous Fabric gate may have been active before the process
|
|
# stopped. Preserve its revision/evidence for audit, but never
|
|
# expose that stale gate as usable after restart.
|
|
gate = value.get("fabric_gate")
|
|
if isinstance(gate, dict):
|
|
gate["active"] = False
|
|
gate["next_stage_allowed"] = False
|
|
gate["error_code"] = "vpc_recovery_pending"
|
|
value["recovery_pending"] = True
|
|
# Preserve the last durable observed evidence for audit and for a
|
|
# future read-only reconciler. The gate is still closed until a
|
|
# fresh observation proves that this revision remains converged.
|
|
marker = value.get("delete_target")
|
|
marker_revision = (marker.get("desired_revision")
|
|
if isinstance(marker, Mapping) else None)
|
|
marker_digest = (marker.get("desired_digest")
|
|
if isinstance(marker, Mapping) else None)
|
|
record_revision = value.get("desired_revision")
|
|
record_digest = value.get("desired_digest")
|
|
for host in value.get("hosts", []):
|
|
if isinstance(host, dict):
|
|
# An exact absence proof is terminal evidence, not a live
|
|
# readiness claim. Preserve both root-delete receipts and
|
|
# normal apply empty-host removals across restart so retry
|
|
# contacts only hosts whose removal is still pending.
|
|
delete_confirmed = (
|
|
isinstance(marker_revision, int)
|
|
and not isinstance(marker_revision, bool)
|
|
and isinstance(marker_digest, str)
|
|
and self._confirmed_removal(
|
|
host, marker_revision, marker_digest)
|
|
)
|
|
apply_confirmed = (
|
|
isinstance(record_revision, int)
|
|
and not isinstance(record_revision, bool)
|
|
and isinstance(record_digest, str)
|
|
and self._confirmed_removal(
|
|
host, record_revision, record_digest)
|
|
)
|
|
if delete_confirmed or apply_confirmed:
|
|
continue
|
|
host.update({"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False})
|
|
value["updated_at"] = int(time.time())
|
|
self._save("vpc_desired", key, value)
|
|
self._save_children(value)
|
|
|
|
async def reconcile_all(self) -> None:
|
|
for item in await __import__("asyncio").to_thread(self.list_status):
|
|
vpc_id = item.get("vpc_id")
|
|
if not isinstance(vpc_id, str):
|
|
continue
|
|
try:
|
|
await self.reconcile(vpc_id)
|
|
except Exception:
|
|
continue
|
|
|
|
async def reconcile_loop(self, interval: float = 30.0) -> None:
|
|
import asyncio
|
|
while True:
|
|
await self.reconcile_all()
|
|
await asyncio.sleep(interval)
|