fix(vpc): add explicit inconsistent-state repair

go
codex 3 weeks ago
parent b699e8301d
commit 8e3a6dc607

@ -108,6 +108,12 @@ func trustedSupersededOperation(ctx context.Context, operationID string) (operat
return prior, ok && prior.OperationID == operationID
}
func isVPCRepairTarget(target string) bool {
return strings.HasSuffix(target, "/repair") &&
(strings.Contains(target, "/v1/vpc/networks/") ||
strings.Contains(target, "/vpc-port/"))
}
type errorBody struct {
Code string `json:"code"`
Message string `json:"message"`
@ -259,7 +265,8 @@ func (s *Server) route(w http.ResponseWriter, r *http.Request, body []byte, acto
}
func (s *Server) routeVPC(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
if len(segments) != 5 || segments[2] != "networks" || !catalogIDPattern.MatchString(segments[3]) ||
repairPath := len(segments) == 6 && segments[5] == "repair"
if (len(segments) != 5 && !repairPath) || segments[2] != "networks" || !catalogIDPattern.MatchString(segments[3]) ||
(segments[4] != "apply" && segments[4] != "remove") {
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
return
@ -294,15 +301,28 @@ func (s *Server) routeVPC(w http.ResponseWriter, r *http.Request, segments []str
return
}
request = normalized
if request.VPCRepair != repairPath ||
(request.VPCRepair != (request.SupersedesOperationID != "")) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation")
return
}
action := "apply-vpc-network:" + vpcKey
expected := &operation.ExpectedState{
Kind: operation.ExpectedVPCNetwork, VPCKey: vpcKey,
VPCRevision: request.Network.DesiredRevision,
VPCDigest: request.DesiredDigest, VPCPresent: true,
}
admission := hypervisor.VPCNetworkAdmission(vpcKey)
supersedes, ok := s.authorizeVPCRepair(
w, r, operationID, request.SupersedesOperationID,
"vpc:"+vpcKey, action, expected, admission, repairPath,
)
if !ok {
return
}
s.runMutationExpected(w, r, body, actor, operationID, "vpc:"+vpcKey, action, expected, true, http.StatusOK,
hypervisor.VPCNetworkAdmission(vpcKey),
func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyVPCNetwork(ctx, request) })
admission,
func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyVPCNetwork(ctx, request) }, supersedes...)
case "remove":
request, ok := decodeRequired[vpcrouting.RemoveRequest](w, body)
if !ok {
@ -322,18 +342,92 @@ func (s *Server) routeVPC(w http.ResponseWriter, r *http.Request, segments []str
writePreDispatchMappedError(w, hypervisor.NewError(hypervisor.ErrInvalid, publicErrorCode(err, "invalid_vpc_remove"), publicErrorMessage(err, "VPC removal request is invalid"), err))
return
}
if request.VPCRepair != repairPath ||
(request.VPCRepair != (request.SupersedesOperationID != "")) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation")
return
}
action := "remove-vpc-network:" + vpcKey
expected := &operation.ExpectedState{
Kind: operation.ExpectedVPCNetwork, VPCKey: vpcKey,
VPCRevision: request.ExpectedRevision,
VPCDigest: request.ExpectedDigest, VPCPresent: false,
}
admission := hypervisor.VPCNetworkAdmission(vpcKey)
supersedes, ok := s.authorizeVPCRepair(
w, r, operationID, request.SupersedesOperationID,
"vpc:"+vpcKey, action, expected, admission, repairPath,
)
if !ok {
return
}
s.runMutationExpected(w, r, body, actor, operationID, "vpc:"+vpcKey, action, expected, true, http.StatusOK,
hypervisor.VPCNetworkAdmission(vpcKey),
func(ctx context.Context) (any, error) { return s.Hypervisor.RemoveVPCNetwork(ctx, request) })
admission,
func(ctx context.Context) (any, error) { return s.Hypervisor.RemoveVPCNetwork(ctx, request) }, supersedes...)
}
}
func (s *Server) authorizeVPCRepair(w http.ResponseWriter, r *http.Request,
operationID, predecessorID, resourceID, action string,
expected *operation.ExpectedState, admission hypervisor.MutationAdmission,
repair bool) ([]string, bool) {
if !repair {
return nil, true
}
if !operationIDPattern.MatchString(predecessorID) ||
operationID == predecessorID {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair_predecessor", "VPC repair requires a different valid predecessor operation")
return nil, false
}
trustedPrior, trusted := trustedSupersededOperation(
r.Context(), predecessorID,
)
existing, existingErr := s.Journal.Get(operationID)
if existingErr == nil {
if existing.Action != action || existing.ResourceID != resourceID ||
existing.SupersedesOperationID != predecessorID {
writePreDispatchError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request")
return nil, false
}
} else if !errors.Is(existingErr, operation.ErrNotFound) {
writeMappedError(w, existingErr, nil)
return nil, false
} else {
prior := trustedPrior
var err error
if !trusted {
prior, err = s.Journal.Get(predecessorID)
}
spec := operation.Spec{
OperationID: operationID, ResourceID: resourceID,
Action: action, ExpectedState: expected,
LockKeys: append([]string(nil), admission.LockKeys...),
SupersedesOperationID: predecessorID,
}
if err != nil || !operation.ValidVPCRepairSupersession(spec, prior) {
writePreDispatchError(w, http.StatusConflict, "vpc_repair_not_authorized", "VPC repair no longer matches one exact fenced predecessor")
return nil, false
}
if !trusted {
_, incident, incidentErr := s.Journal.CompletionDeliveryIncident(
predecessorID,
)
if incidentErr != nil {
writeMappedError(w, incidentErr, nil)
return nil, false
}
if incident {
writePreDispatchError(w, http.StatusConflict, "vpc_repair_not_authorized", "VPC repair is blocked by a completion-delivery incident")
return nil, false
}
}
}
if trusted {
return nil, true
}
return []string{predecessorID}, true
}
func (s *Server) routeStorage(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) {
if len(segments) != 3 || segments[2] != "configuration" {
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
@ -1010,7 +1104,9 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str
return
}
case "vpc-port":
if len(segments) == 4 && (r.Method == http.MethodPost || r.Method == http.MethodDelete) {
repairPath := len(segments) == 5 && segments[4] == "repair"
if (len(segments) == 4 || repairPath) &&
(r.Method == http.MethodPost || r.Method == http.MethodDelete) {
if !requireManage(w, actor) {
return
}
@ -1029,6 +1125,11 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str
writePreDispatchMappedError(w, err)
return
}
if request.VPCRepair != repairPath ||
(request.VPCRepair != (request.SupersedesOperationID != "")) {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation")
return
}
bridge, _ := vpcrouting.BridgeName(request.VPCKey)
present := r.Method == http.MethodPost
action := "apply-vpc-port"
@ -1043,15 +1144,23 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str
VPCBridge: bridge, VPCRevision: request.HostRevision,
VPCDigest: request.HostDigest,
}
admission := hypervisor.VPCPortAdmission(uuid, request)
supersedes, ok := s.authorizeVPCRepair(
w, r, operationID, request.SupersedesOperationID,
uuid, action, expected, admission, repairPath,
)
if !ok {
return
}
s.runMutationExpected(w, r, body, actor, operationID, uuid,
action, expected, true, http.StatusOK,
hypervisor.VPCPortAdmission(uuid, request),
admission,
func(ctx context.Context) (any, error) {
if present {
return s.Hypervisor.ApplyVPCPort(ctx, uuid, request)
}
return s.Hypervisor.RemoveVPCPort(ctx, uuid, request)
})
}, supersedes...)
return
}
case "service":
@ -2134,6 +2243,9 @@ func (s *Server) runIsolatedMutation(
return
}
request.SupersededOperation = &prior
if isVPCRepairTarget(request.RequestTarget) {
request.SupersededExpectedState = prior.ExpectedState
}
}
dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout)
record, replayed, done, err := s.Journal.ExecuteBackground(

@ -41,6 +41,21 @@ type testHypervisor struct {
inspectionErr error
}
type apiFailure struct {
code string
effect bool
safe bool
fence bool
}
func (e apiFailure) Error() string { return e.code }
func (e apiFailure) PublicCode() string { return e.code }
func (e apiFailure) PublicMessage() string { return e.code }
func (e apiFailure) FailureSemantics() (bool, bool) {
return e.effect, e.safe
}
func (e apiFailure) RequiresExecutionFence() bool { return e.fence }
func (h *testHypervisor) Ping(context.Context) error { return nil }
func (h *testHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) {
@ -283,6 +298,93 @@ func TestVPCNetworkMutationUsesCanonicalDigestAndDurableExpectedState(t *testing
}
}
func TestVPCNetworkRepairCarriesExactSupersededGeneration(t *testing.T) {
api := newTestAPI(t)
const (
vpcKey = "0123456789abcdef0123456789abcdef"
priorID = "operation-vpc-repair-prior"
repairID = "operation-vpc-repair-next"
)
network := vpcrouting.Network{
VPCKey: vpcKey, DesiredRevision: 9,
CIDR: "10.89.0.0/24", Gateway: "10.89.0.1",
Transport: vpcrouting.Transport{
AddressCIDR: "172.16.50.11/24", Interface: "br-private", MTU: 1500,
},
LocalPorts: []vpcrouting.LocalPort{{IP: "10.89.0.2", MAC: "02:00:00:00:00:12"}},
}
digest, err := vpcrouting.Digest(network)
if err != nil {
t.Fatal(err)
}
prior := operation.Spec{
OperationID: priorID, ResourceID: "vpc:" + vpcKey,
Action: "apply-vpc-network:" + vpcKey,
Fingerprint: operation.Fingerprint(
http.MethodPost, "/v1/vpc/networks/"+vpcKey+"/apply", []byte(`{}`),
),
Destructive: true,
ExpectedState: &operation.ExpectedState{
Kind: operation.ExpectedVPCNetwork, TransitionID: priorID,
VPCKey: vpcKey, VPCRevision: 9,
VPCDigest: digest, VPCPresent: true,
},
LockKeys: hypervisor.VPCNetworkAdmission(vpcKey).LockKeys,
}
if _, _, err := api.journal.Execute(
context.Background(), prior,
func(context.Context) (json.RawMessage, error) {
return nil, apiFailure{
code: "vpc_reconcile_unconfirmed", effect: true,
safe: false, fence: true,
}
},
); err == nil {
t.Fatal("VPC repair predecessor unexpectedly succeeded")
}
request := vpcrouting.ApplyRequest{
OperationID: repairID, DesiredDigest: digest, Network: network,
VPCRepair: true, SupersedesOperationID: priorID,
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
bridge, _ := vpcrouting.BridgeName(vpcKey)
result, _ := json.Marshal(vpcrouting.Result{
VPCKey: vpcKey, DesiredRevision: 9, DesiredDigest: digest,
Bridge: bridge, Applied: true,
})
executor := &fakeMutationExecutor{result: result}
if err := api.handler.SetMutationExecutor(executor); err != nil {
t.Fatal(err)
}
response := api.request(t, http.MethodPost,
"/v1/vpc/networks/"+vpcKey+"/apply/repair", body,
auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: repairID,
},
)
if response.Code != http.StatusAccepted && response.Code != http.StatusOK {
t.Fatalf("VPC repair status=%d body=%s", response.Code, response.Body.String())
}
record := waitForTerminal(t, api.journal, repairID)
if record.State != operation.StateSucceeded ||
record.SupersedesOperationID != priorID {
t.Fatalf("VPC repair record = %#v", record)
}
executor.mu.Lock()
defer executor.mu.Unlock()
if executor.calls != 1 || len(executor.requests) != 1 ||
executor.requests[0].SupersededOperation == nil ||
executor.requests[0].SupersededOperation.OperationID != priorID ||
executor.requests[0].SupersededExpectedState == nil ||
executor.requests[0].SupersededExpectedState.VPCDigest != digest {
t.Fatalf("VPC repair worker envelope = %#v", executor.requests)
}
}
func TestVPCPortMutationBindsGuestAndHostProjectionIdentity(t *testing.T) {
api := newTestAPI(t)
const (

@ -460,9 +460,11 @@ func runOperationWorker(configPath, journalPath, receiptPath string) error {
}).WithContext(ctx)
responseCapture := newBoundedWorkerResponse(worker.MaxWorkerResponse)
if request.SupersededOperation != nil {
prior := *request.SupersededOperation
prior.ExpectedState = request.SupersededExpectedState
handler.ServeTrustedWorkerRequest(
responseCapture, httpRequest, []byte(request.Body), request.Actor,
*request.SupersededOperation,
prior,
)
} else {
handler.ServeTrustedWorkerRequest(

@ -525,6 +525,56 @@ func mutateSeedVPCPort(contents []byte, port VPCPortMetadata,
return buildSeedISO(files)
}
// repairSeedVPCPort rewrites only Virt Cloud's reserved private0 entry. A
// different valid VPC identity proves that panel intent has moved on and is
// rejected; a missing or malformed reserved entry is the partial-state case
// this explicit repair operation exists to converge.
func repairSeedVPCPort(contents []byte, port VPCPortMetadata,
transitionID string, present bool) ([]byte, error) {
if !operationIdentifierPattern.MatchString(transitionID) {
return nil, NewError(ErrInvalid, "invalid_operation_id", "VPC port transition id is invalid", nil)
}
files, err := readSeedISO(contents)
if err != nil {
return nil, err
}
var network map[string]any
if err := json.Unmarshal(bytes.TrimSpace(files.NetworkConfig), &network); err != nil {
return nil, NewError(ErrConflict, "invalid_seed", "NoCloud network-config is invalid", err)
}
ethernets, err := managedSeedEthernets(network)
if err != nil {
return nil, err
}
if existing, exists := ethernets[seedVPCInterfaceKey]; exists {
if parsed, ok := parseSeedVPCInterface(existing); ok && parsed != (seedVPCNetwork{
MAC: port.MAC, PrivateIP: port.PrivateIP,
CIDR: port.CIDR, Gateway: port.Gateway,
}) {
return nil, NewError(ErrConflict, "vpc_repair_intent_changed", "NoCloud contains a different VPC identity", nil)
}
}
if present {
ethernets[seedVPCInterfaceKey] = seedVPCInterface(port)
} else {
delete(ethernets, seedVPCInterfaceKey)
}
files.NetworkConfig, err = marshalJSONLine(network)
if err != nil {
return nil, err
}
var metadata map[string]any
if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil {
return nil, NewError(ErrConflict, "invalid_seed", "NoCloud meta-data is invalid", err)
}
metadata[vpcPortOperationMarker] = transitionID
files.MetaData, err = marshalJSONLine(metadata)
if err != nil {
return nil, err
}
return buildSeedISO(files)
}
func seedVPCPort(contents []byte) (*seedVPCNetwork, error) {
files, err := readSeedISO(contents)
if err != nil {

@ -495,6 +495,52 @@ func TestMutateSeedVPCPortPreservesPublicNetworkAndBindsMarker(t *testing.T) {
}
}
func TestRepairSeedVPCPortConvergesOnlyTheSameManagedIdentity(t *testing.T) {
port := validVPCPortMetadata()
original := managedSeedForTest(t)
attached, err := repairSeedVPCPort(
original, port, "operation-vpc-repair-attach", true,
)
if err != nil {
t.Fatal(err)
}
repaired, err := repairSeedVPCPort(
attached, port, "operation-vpc-repair-rebind", true,
)
if err != nil {
t.Fatal(err)
}
private, err := seedVPCPort(repaired)
want := seedVPCNetwork{
MAC: port.MAC, PrivateIP: port.PrivateIP,
CIDR: port.CIDR, Gateway: port.Gateway,
}
if err != nil || private == nil || *private != want {
t.Fatalf("repaired private network = %#v err=%v", private, err)
}
changed := port
changed.PrivateIP = "10.60.0.10"
if _, err := repairSeedVPCPort(
repaired, changed, "operation-vpc-repair-changed", true,
); err == nil {
t.Fatal("repair adopted a different VPC identity")
}
detached, err := repairSeedVPCPort(
repaired, port, "operation-vpc-repair-detach", false,
)
if err != nil {
t.Fatal(err)
}
if detached, err = repairSeedVPCPort(
detached, port, "operation-vpc-repair-detach-again", false,
); err != nil {
t.Fatalf("repair could not converge an already absent seed port: %v", err)
}
if private, err := seedVPCPort(detached); err != nil || private != nil {
t.Fatalf("repair detach left private network = %#v err=%v", private, err)
}
}
func TestPreservedVPCCloudInitNetworkRejectsCrossLayerMismatch(t *testing.T) {
port := validVPCPortMetadata()
attached, err := mutateSeedVPCPort(

@ -94,20 +94,35 @@ func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string,
if err != nil {
return VPCPortResult{}, err
}
if !managedNetworkLayoutMatches(preflightDefinition, preflightMetadata) {
return VPCPortResult{}, NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its durable metadata", nil)
}
if present {
if preflightMetadata.VPCPort != nil {
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil)
if request.VPCRepair {
if err := validateRepairableVPCPortDomain(
preflightDefinition, preflightMetadata, port,
); err != nil {
return VPCPortResult{}, err
}
} else {
if !managedNetworkLayoutMatches(preflightDefinition, preflightMetadata) {
return VPCPortResult{}, NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its durable metadata", nil)
}
if present {
if preflightMetadata.VPCPort != nil {
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil)
}
} else if preflightMetadata.VPCPort == nil ||
!sameVPCPortIdentity(preflightMetadata.VPCPort.public(), port) {
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_identity_changed", "VM VPC port no longer matches the requested detach identity", nil)
}
} else if preflightMetadata.VPCPort == nil ||
!sameVPCPortIdentity(preflightMetadata.VPCPort.public(), port) {
return VPCPortResult{}, NewError(ErrConflict, "vpc_port_identity_changed", "VM VPC port no longer matches the requested detach identity", nil)
}
replacementSeed, err := mutateSeedVPCPort(
originalSeed, port, expected.TransitionID, present,
)
var replacementSeed []byte
if request.VPCRepair {
replacementSeed, err = repairSeedVPCPort(
originalSeed, port, expected.TransitionID, present,
)
} else {
replacementSeed, err = mutateSeedVPCPort(
originalSeed, port, expected.TransitionID, present,
)
}
if err != nil {
return VPCPortResult{}, err
}
@ -124,6 +139,15 @@ func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string,
}
updatedXML, err := mutateDomainDocument(originalXML,
func(definition *libvirtxml.Domain, metadata *domainMetadataXML) error {
if request.VPCRepair {
if err := repairVPCPortDomain(
definition, metadata, port, present,
); err != nil {
return err
}
metadata.VPCTransitionID = expected.TransitionID
return nil
}
if !managedNetworkLayoutMatches(definition, metadata) {
return NewError(ErrConflict, "network_layout_changed", "managed VM network layout changed while applying the VPC port", nil)
}
@ -131,28 +155,10 @@ func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string,
if metadata.VPCPort != nil {
return NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil)
}
linkDown := definitionNetworkSuspended(definition)
queues := uint(1)
if definition.VCPU != nil {
queues = min(definition.VCPU.Value, 16)
}
iface := libvirtxml.DomainInterface{
MAC: &libvirtxml.DomainInterfaceMAC{Address: port.MAC},
Model: &libvirtxml.DomainInterfaceModel{Type: "virtio"},
Driver: &libvirtxml.DomainInterfaceDriver{Queues: queues},
Source: &libvirtxml.DomainInterfaceSource{Bridge: &libvirtxml.DomainInterfaceSourceBridge{Bridge: port.Bridge}},
}
if metadata.BandwidthMbps > 0 {
rate := bandwidthRateKiBPerSecond(metadata.BandwidthMbps)
iface.Bandwidth = &libvirtxml.DomainInterfaceBandwidth{
Inbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
Outbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
}
}
if linkDown {
iface.Link = &libvirtxml.DomainInterfaceLink{State: "down"}
}
definition.Devices.Interfaces = append(definition.Devices.Interfaces, iface)
definition.Devices.Interfaces = append(
definition.Devices.Interfaces,
canonicalVPCInterface(definition, metadata, port),
)
stored := domainVPCPortFromPublic(port)
metadata.VPCPort = &stored
} else {
@ -194,6 +200,93 @@ func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string,
return result, nil
}
// validateRepairableVPCPortDomain proves that every non-public interface is
// either the exact panel-owned VPC NIC or is already bound to that identity by
// durable metadata. It never adopts or deletes an unrelated guest interface.
func validateRepairableVPCPortDomain(definition *libvirtxml.Domain,
metadata *domainMetadataXML, port VPCPortMetadata) error {
if definition == nil || definition.Devices == nil || metadata == nil {
return NewError(ErrConflict, "network_layout_changed", "managed VM network layout is incomplete", nil)
}
publicIndex, found := managedPublicInterfaceIndex(
definition, networkFilterName(definition.UUID), metadata.PublicIPCIDR,
)
if !found || len(definition.Devices.Interfaces) > 2 {
return NewError(ErrConflict, "network_layout_changed", "managed VM has an unrepairable network layout", nil)
}
metadataOwnsPort := metadata.VPCPort != nil
if metadataOwnsPort &&
!sameVPCPortIdentity(metadata.VPCPort.public(), port) {
return NewError(ErrConflict, "vpc_repair_intent_changed", "domain metadata contains a different VPC identity", nil)
}
if len(definition.Devices.Interfaces) == 2 {
privateIndex := 1 - publicIndex
iface := definition.Devices.Interfaces[privateIndex]
exactInterface := iface.MAC != nil &&
strings.EqualFold(iface.MAC.Address, port.MAC) &&
iface.Source != nil && iface.Source.Bridge != nil &&
iface.Source.Bridge.Bridge == port.Bridge &&
iface.Source.Network == nil && iface.FilterRef == nil
if !metadataOwnsPort && !exactInterface {
return NewError(ErrConflict, "vpc_repair_unowned_interface", "unrelated private interface cannot be changed by VPC repair", nil)
}
}
return nil
}
func repairVPCPortDomain(definition *libvirtxml.Domain,
metadata *domainMetadataXML, port VPCPortMetadata, present bool) error {
if err := validateRepairableVPCPortDomain(definition, metadata, port); err != nil {
return err
}
publicIndex, _ := managedPublicInterfaceIndex(
definition, networkFilterName(definition.UUID), metadata.PublicIPCIDR,
)
if len(definition.Devices.Interfaces) == 2 {
privateIndex := 1 - publicIndex
definition.Devices.Interfaces = append(
definition.Devices.Interfaces[:privateIndex],
definition.Devices.Interfaces[privateIndex+1:]...,
)
}
if present {
definition.Devices.Interfaces = append(
definition.Devices.Interfaces,
canonicalVPCInterface(definition, metadata, port),
)
stored := domainVPCPortFromPublic(port)
metadata.VPCPort = &stored
} else {
metadata.VPCPort = nil
}
return nil
}
func canonicalVPCInterface(definition *libvirtxml.Domain,
metadata *domainMetadataXML, port VPCPortMetadata) libvirtxml.DomainInterface {
queues := uint(1)
if definition.VCPU != nil {
queues = min(definition.VCPU.Value, 16)
}
iface := libvirtxml.DomainInterface{
MAC: &libvirtxml.DomainInterfaceMAC{Address: port.MAC},
Model: &libvirtxml.DomainInterfaceModel{Type: "virtio"},
Driver: &libvirtxml.DomainInterfaceDriver{Queues: queues},
Source: &libvirtxml.DomainInterfaceSource{Bridge: &libvirtxml.DomainInterfaceSourceBridge{Bridge: port.Bridge}},
}
if metadata.BandwidthMbps > 0 {
rate := bandwidthRateKiBPerSecond(metadata.BandwidthMbps)
iface.Bandwidth = &libvirtxml.DomainInterfaceBandwidth{
Inbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
Outbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate},
}
}
if definitionNetworkSuspended(definition) {
iface.Link = &libvirtxml.DomainInterfaceLink{State: "down"}
}
return iface
}
func (l *Libvirt) vpcPortIdentityConflict(ctx context.Context,
requestedUUID string, port VPCPortMetadata) (bool, error) {
domains, err := l.conn.ListAllDomains(0)

@ -224,16 +224,18 @@ type VPCPortMetadata struct {
}
type VPCPortRequest struct {
OperationID string `json:"operation_id,omitempty"`
PortKey string `json:"port_key"`
VPCKey string `json:"vpc_key"`
PrivateIP string `json:"private_ip"`
CIDR string `json:"cidr"`
Gateway string `json:"gateway"`
MAC string `json:"mac"`
HostRevision uint64 `json:"host_revision"`
HostDigest string `json:"host_digest"`
ConfirmRestart bool `json:"confirm_restart"`
OperationID string `json:"operation_id,omitempty"`
PortKey string `json:"port_key"`
VPCKey string `json:"vpc_key"`
PrivateIP string `json:"private_ip"`
CIDR string `json:"cidr"`
Gateway string `json:"gateway"`
MAC string `json:"mac"`
HostRevision uint64 `json:"host_revision"`
HostDigest string `json:"host_digest"`
ConfirmRestart bool `json:"confirm_restart"`
VPCRepair bool `json:"vpc_repair,omitempty"`
SupersedesOperationID string `json:"supersedes_operation_id,omitempty"`
}
type VPCPortResult struct {

@ -19,6 +19,11 @@ func NormalizeVPCPortRequest(request VPCPortRequest) (VPCPortRequest, error) {
!digestPattern.MatchString(request.HostDigest) || !request.ConfirmRestart {
return VPCPortRequest{}, NewError(ErrInvalid, "invalid_vpc_port_request", "VPC port identity, host projection, and restart confirmation are required", nil)
}
if request.VPCRepair != (request.SupersedesOperationID != "") ||
(request.VPCRepair && (!operationIdentifierPattern.MatchString(request.SupersedesOperationID) ||
request.SupersedesOperationID == request.OperationID)) {
return VPCPortRequest{}, NewError(ErrInvalid, "invalid_vpc_repair", "VPC repair requires one different valid predecessor operation", nil)
}
prefix, err := netip.ParsePrefix(request.CIDR)
if err != nil || !prefix.Addr().Is4() || !prefix.Addr().IsPrivate() ||
prefix != prefix.Masked() || prefix.Bits() < 16 || prefix.Bits() > 28 {

@ -45,6 +45,16 @@ func TestNormalizeVPCPortRequestRequiresCanonicalRoutedIdentity(t *testing.T) {
"multicast MAC": func(value *VPCPortRequest) { value.MAC = "03:00:00:00:00:09" },
"upper-case MAC": func(value *VPCPortRequest) { value.MAC = "02:00:00:00:00:AA" },
"invalid digest": func(value *VPCPortRequest) { value.HostDigest = "bad" },
"repair without predecessor": func(value *VPCPortRequest) {
value.VPCRepair = true
},
"predecessor without repair": func(value *VPCPortRequest) {
value.SupersedesOperationID = "operation-vpc-port-prior"
},
"repair supersedes itself": func(value *VPCPortRequest) {
value.VPCRepair = true
value.SupersedesOperationID = value.OperationID
},
} {
t.Run(name, func(t *testing.T) {
candidate := request
@ -54,6 +64,12 @@ func TestNormalizeVPCPortRequestRequiresCanonicalRoutedIdentity(t *testing.T) {
}
})
}
repair := request
repair.VPCRepair = true
repair.SupersedesOperationID = "operation-vpc-port-prior"
if _, err := NormalizeVPCPortRequest(repair); err != nil {
t.Fatalf("valid repair request was rejected: %v", err)
}
}
func TestValidateVPCPortMetadataRequiresDeterministicBridge(t *testing.T) {

@ -177,8 +177,7 @@ type Record struct {
LockKeys []string `json:"-"`
HeavyIO bool `json:"-"`
// SupersedesOperationID is valid only for an explicitly reviewed successor:
// force-delete-vm after force_required, or a plan-bound PeerMigration target
// cleanup after one exact failed receive operation.
// force-delete-vm, plan-bound cleanup, or an exact VPC repair generation.
SupersedesOperationID string `json:"-"`
// InspectionReceiptRecovered and InspectionEvidenceUnavailable are
// ephemeral read-path markers. They are never persisted or exposed by the
@ -2440,8 +2439,9 @@ func (j *Journal) updateRecordWhen(operationID string, eligible func(Record) boo
if err := syncResourceBlockers(blockers, &previous, &record); err != nil {
return err
}
if record.State == StateSucceeded && record.Action == "force-delete-vm" && record.SupersedesOperationID != "" {
if err := closeForceDeletePredecessor(bucket, blockers, record); err != nil {
if record.State == StateSucceeded && record.SupersedesOperationID != "" &&
(record.Action == "force-delete-vm" || isVPCAction(record.Action)) {
if err := closeSuccessfulSupersessionPredecessor(bucket, blockers, record); err != nil {
return err
}
}
@ -2466,11 +2466,10 @@ func (j *Journal) updateRecordWhen(operationID string, eligible func(Record) boo
return cloneRecord(result), err
}
// closeForceDeletePredecessor runs in the same Bolt transaction which marks
// the explicit force operation succeeded. It removes only the predecessor's
// blocker index entries; the failed operation and its completion receipt stay
// byte-for-byte immutable, with the successor retaining the audit link.
func closeForceDeletePredecessor(operations, blockers *bolt.Bucket, successor Record) error {
// closeSuccessfulSupersessionPredecessor runs in the same Bolt transaction
// which marks the reviewed successor successful. It removes only the prior
// blocker index entries; terminal evidence remains byte-for-byte immutable.
func closeSuccessfulSupersessionPredecessor(operations, blockers *bolt.Bucket, successor Record) error {
encoded := operations.Get([]byte(successor.SupersedesOperationID))
if encoded == nil {
return ErrInvalidSupersession
@ -2480,15 +2479,31 @@ func closeForceDeletePredecessor(operations, blockers *bolt.Bucket, successor Re
return err
}
lockKeys := effectiveLockKeys(successor.OperationID, successor.ResourceID, successor.Action, successor.LockKeys)
if !validForceDeleteSupersession(Spec{
spec := Spec{
OperationID: successor.OperationID, ResourceID: successor.ResourceID,
Action: successor.Action, SupersedesOperationID: successor.SupersedesOperationID,
}, lockKeys, predecessor) {
Action: successor.Action, ExpectedState: successor.ExpectedState,
SupersedesOperationID: successor.SupersedesOperationID,
}
if validForceDeleteSupersession(spec, lockKeys, predecessor) {
for _, lockKey := range effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys) {
if err := blockers.Delete(blockerEntryKey(lockKey, predecessor.OperationID)); err != nil {
return err
}
}
return nil
}
if !isVPCAction(successor.Action) {
return ErrInvalidSupersession
}
for _, lockKey := range effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys) {
if err := blockers.Delete(blockerEntryKey(lockKey, predecessor.OperationID)); err != nil {
return err
chain, err := vpcSupersessionChain(operations, spec, lockKeys)
if err != nil {
return err
}
for _, prior := range chain {
for _, lockKey := range effectiveLockKeys(prior.OperationID, prior.ResourceID, prior.Action, prior.LockKeys) {
if err := blockers.Delete(blockerEntryKey(lockKey, prior.OperationID)); err != nil {
return err
}
}
}
return nil
@ -2715,10 +2730,120 @@ func validBackupWorkspaceCleanupSupersession(spec Spec, lockKeys []string, prede
return lockKeysOverlap(lockKeys, predecessorKeys)
}
func isVPCAction(action string) bool {
return strings.HasPrefix(action, "apply-vpc-network:") ||
strings.HasPrefix(action, "remove-vpc-network:") ||
action == "apply-vpc-port" || action == "remove-vpc-port"
}
func sameVPCRepairIntent(current, predecessor *ExpectedState) bool {
if current == nil || predecessor == nil || current.Kind != predecessor.Kind ||
current.VPCKey != predecessor.VPCKey ||
current.VPCRevision != predecessor.VPCRevision ||
current.VPCDigest != predecessor.VPCDigest ||
current.VPCPresent != predecessor.VPCPresent {
return false
}
if current.Kind == ExpectedVPCNetwork {
return true
}
return current.Kind == ExpectedVPCPort &&
current.VPCPortKey == predecessor.VPCPortKey &&
current.VPCCIDR == predecessor.VPCCIDR &&
current.VPCGateway == predecessor.VPCGateway &&
current.VPCPrivateIP == predecessor.VPCPrivateIP &&
current.VPCMAC == predecessor.VPCMAC &&
current.VPCBridge == predecessor.VPCBridge
}
func validVPCRepairSupersession(spec Spec, lockKeys []string,
predecessor Record) bool {
if spec.SupersedesOperationID == "" ||
spec.SupersedesOperationID == spec.OperationID ||
predecessor.OperationID != spec.SupersedesOperationID ||
predecessor.ResourceID != spec.ResourceID ||
predecessor.Action != spec.Action || !isVPCAction(spec.Action) ||
predecessor.State != StateFailed || !predecessor.EffectMayHaveApplied ||
predecessor.SafeToResubmit || !predecessor.ExecutionFenced ||
predecessor.Tombstone ||
!sameVPCRepairIntent(spec.ExpectedState, predecessor.ExpectedState) {
return false
}
predecessorKeys := effectiveLockKeys(
predecessor.OperationID, predecessor.ResourceID,
predecessor.Action, predecessor.LockKeys,
)
return lockKeysOverlap(lockKeys, predecessorKeys)
}
// ValidVPCRepairSupersession exposes the same fail-closed predecessor check to
// the authenticated API before a repair is dispatched. Journal admission
// repeats it under the Bolt transaction, so this preflight is not authority.
func ValidVPCRepairSupersession(spec Spec, predecessor Record) bool {
lockKeys, err := normalizeLockKeys(spec.LockKeys)
return err == nil && validVPCRepairSupersession(
spec, lockKeys, predecessor,
)
}
func validExplicitSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
return validForceDeleteSupersession(spec, lockKeys, predecessor) ||
validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor) ||
validBackupWorkspaceCleanupSupersession(spec, lockKeys, predecessor)
validBackupWorkspaceCleanupSupersession(spec, lockKeys, predecessor) ||
validVPCRepairSupersession(spec, lockKeys, predecessor)
}
const maxVPCSupersessionDepth = 32
// vpcSupersessionChain validates every durable predecessor generation. A
// repair that itself lost its response may be followed by another exact
// repair, but it may never skip a changed intent or grow without a bound.
func vpcSupersessionChain(operations *bolt.Bucket, spec Spec,
lockKeys []string) ([]Record, error) {
if operations == nil || !isVPCAction(spec.Action) ||
spec.SupersedesOperationID == "" {
return nil, ErrInvalidSupersession
}
currentSpec := spec
currentKeys := lockKeys
seen := map[string]struct{}{spec.OperationID: {}}
chain := make([]Record, 0, 2)
for len(chain) < maxVPCSupersessionDepth {
predecessorID := currentSpec.SupersedesOperationID
if predecessorID == "" {
return chain, nil
}
if _, exists := seen[predecessorID]; exists {
return nil, ErrInvalidSupersession
}
seen[predecessorID] = struct{}{}
encoded := operations.Get([]byte(predecessorID))
if encoded == nil {
return nil, ErrInvalidSupersession
}
predecessor, err := decodeRecord(encoded)
if err != nil || !validVPCRepairSupersession(
currentSpec, currentKeys, predecessor,
) {
return nil, ErrInvalidSupersession
}
chain = append(chain, predecessor)
if predecessor.SupersedesOperationID == "" {
return chain, nil
}
currentSpec = Spec{
OperationID: predecessor.OperationID,
ResourceID: predecessor.ResourceID,
Action: predecessor.Action,
ExpectedState: predecessor.ExpectedState,
SupersedesOperationID: predecessor.SupersedesOperationID,
}
currentKeys = effectiveLockKeys(
predecessor.OperationID, predecessor.ResourceID,
predecessor.Action, predecessor.LockKeys,
)
}
return nil, ErrInvalidSupersession
}
// findResourceBlocker validates any explicit force-delete predecessor and
@ -2729,7 +2854,7 @@ func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, boo
if operations == nil || blockers == nil {
return Record{}, false, errors.New("operation resource blocker index is unavailable")
}
supersededOperationID := ""
supersededOperationIDs := map[string]struct{}{}
if spec.SupersedesOperationID != "" {
encoded := operations.Get([]byte(spec.SupersedesOperationID))
if encoded == nil {
@ -2739,14 +2864,27 @@ func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, boo
if err != nil || !validExplicitSupersession(spec, lockKeys, predecessor) {
return Record{}, false, ErrInvalidSupersession
}
supersededOperationID = predecessor.OperationID
if isVPCAction(spec.Action) {
chain, chainErr := vpcSupersessionChain(
operations, spec, lockKeys,
)
if chainErr != nil {
return Record{}, false, chainErr
}
for _, prior := range chain {
supersededOperationIDs[prior.OperationID] = struct{}{}
}
} else {
supersededOperationIDs[predecessor.OperationID] = struct{}{}
}
}
for _, lockKey := range lockKeys {
prefix := blockerEntryPrefix(lockKey)
cursor := blockers.Cursor()
for key, _ := cursor.Seek(prefix); key != nil && bytes.HasPrefix(key, prefix); key, _ = cursor.Next() {
operationID := string(key[len(prefix):])
if operationID == spec.OperationID || operationID == supersededOperationID {
_, explicitlySuperseded := supersededOperationIDs[operationID]
if operationID == spec.OperationID || explicitlySuperseded {
continue
}
encoded := operations.Get([]byte(operationID))
@ -3035,7 +3173,8 @@ func validateSpec(spec Spec) error {
spec.SupersedesOperationID == spec.OperationID ||
(spec.Action != "force-delete-vm" &&
!cleanupPeerTargetActionPattern.MatchString(spec.Action) &&
!cleanupBackupActionPattern.MatchString(spec.Action)) {
!cleanupBackupActionPattern.MatchString(spec.Action) &&
!isVPCAction(spec.Action)) {
return ErrInvalidSupersession
}
}

@ -1763,6 +1763,187 @@ func TestSuccessfulForceSuccessorOnlyRemovesPredecessorBlockerIndex(t *testing.T
}
}
func TestSuccessfulVPCRepairSupersedesOnlyExactFencedIntent(t *testing.T) {
j := openTestJournal(t)
const (
vpcKey = "0123456789abcdef0123456789abcdef"
digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
)
makeSpec := func(id string) Spec {
return Spec{
OperationID: id,
ResourceID: "vpc:" + vpcKey,
Action: "apply-vpc-network:" + vpcKey,
Fingerprint: Fingerprint("POST", "/v1/vpc/networks/"+vpcKey+"/apply/repair", []byte(id)),
Destructive: true,
ExpectedState: &ExpectedState{
Kind: ExpectedVPCNetwork, TransitionID: id,
VPCKey: vpcKey, VPCRevision: 7,
VPCDigest: digest, VPCPresent: true,
},
LockKeys: []string{"catalog:vpc-host/state", "catalog:vpc/" + vpcKey},
}
}
predecessor := makeSpec("operation-vpc-repair-predecessor")
if _, _, err := j.Execute(context.Background(), predecessor,
func(context.Context) (json.RawMessage, error) {
return nil, testFailure{
code: "vpc_reconcile_unconfirmed", message: "VPC state is inconsistent",
effect: true, safe: false, fence: true,
}
}); err == nil {
t.Fatal("VPC predecessor unexpectedly succeeded")
}
prior, err := j.Get(predecessor.OperationID)
if err != nil || !prior.ExecutionFenced {
t.Fatalf("VPC predecessor is not fenced: %#v err=%v", prior, err)
}
var before []byte
if err := j.db.View(func(tx *bolt.Tx) error {
before = append([]byte(nil), tx.Bucket(operationsBucket).Get(
[]byte(predecessor.OperationID),
)...)
return nil
}); err != nil {
t.Fatal(err)
}
successor := makeSpec("operation-vpc-repair-successor")
successor.SupersedesOperationID = predecessor.OperationID
if record, _, err := j.Execute(context.Background(), successor,
func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"applied":true}`), nil
}); err != nil || record.State != StateSucceeded {
t.Fatalf("VPC repair successor = %#v err=%v", record, err)
}
var after []byte
if err := j.db.View(func(tx *bolt.Tx) error {
after = append([]byte(nil), tx.Bucket(operationsBucket).Get(
[]byte(predecessor.OperationID),
)...)
return nil
}); err != nil {
t.Fatal(err)
}
if !bytes.Equal(before, after) {
t.Fatal("VPC repair rewrote predecessor terminal evidence")
}
followup := makeSpec("operation-vpc-after-repair")
followup.ExpectedState.VPCRevision = 8
followup.ExpectedState.VPCDigest = strings.Repeat("b", 64)
if _, _, err := j.Execute(context.Background(), followup,
func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"applied":true}`), nil
}); err != nil {
t.Fatalf("VPC predecessor blocker survived repair: %v", err)
}
invalid := makeSpec("operation-vpc-repair-wrong-intent")
invalid.ExpectedState.VPCDigest = strings.Repeat("c", 64)
invalid.SupersedesOperationID = predecessor.OperationID
if _, _, err := j.Execute(context.Background(), invalid,
func(context.Context) (json.RawMessage, error) { return nil, nil }); !errors.Is(err, ErrInvalidSupersession) {
t.Fatalf("VPC repair accepted changed intent: %v", err)
}
}
func TestSuccessfulVPCRepairClosesExactUnknownRepairChain(t *testing.T) {
j := openTestJournal(t)
const (
vpcKey = "1123456789abcdef0123456789abcdef"
digest = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"
)
makeSpec := func(id string) Spec {
return Spec{
OperationID: id,
ResourceID: "vpc:" + vpcKey,
Action: "apply-vpc-network:" + vpcKey,
Fingerprint: Fingerprint(
"POST", "/v1/vpc/networks/"+vpcKey+"/apply/repair",
[]byte(id),
),
Destructive: true,
ExpectedState: &ExpectedState{
Kind: ExpectedVPCNetwork, TransitionID: id,
VPCKey: vpcKey, VPCRevision: 9,
VPCDigest: digest, VPCPresent: true,
},
LockKeys: []string{
"catalog:vpc-host/state", "catalog:vpc/" + vpcKey,
},
}
}
failUnknown := func(spec Spec) {
if _, _, err := j.Execute(context.Background(), spec,
func(context.Context) (json.RawMessage, error) {
return nil, testFailure{
code: "vpc_reconcile_unconfirmed",
message: "repair result was not confirmed",
effect: true, safe: false, fence: true,
}
}); err == nil {
t.Fatalf("VPC generation %s unexpectedly succeeded", spec.OperationID)
}
}
original := makeSpec("operation-vpc-chain-original")
failUnknown(original)
firstRepair := makeSpec("operation-vpc-chain-repair-one")
firstRepair.SupersedesOperationID = original.OperationID
failUnknown(firstRepair)
var originalBefore, repairBefore []byte
if err := j.db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(operationsBucket)
originalBefore = append(
[]byte(nil), bucket.Get([]byte(original.OperationID))...,
)
repairBefore = append(
[]byte(nil), bucket.Get([]byte(firstRepair.OperationID))...,
)
return nil
}); err != nil {
t.Fatal(err)
}
secondRepair := makeSpec("operation-vpc-chain-repair-two")
secondRepair.SupersedesOperationID = firstRepair.OperationID
if record, _, err := j.Execute(
context.Background(), secondRepair,
func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"applied":true}`), nil
},
); err != nil || record.State != StateSucceeded {
t.Fatalf("second repair = %#v err=%v", record, err)
}
if err := j.db.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket(operationsBucket)
if !bytes.Equal(
originalBefore, bucket.Get([]byte(original.OperationID)),
) || !bytes.Equal(
repairBefore, bucket.Get([]byte(firstRepair.OperationID)),
) {
t.Fatal("successful repair rewrote predecessor evidence")
}
return nil
}); err != nil {
t.Fatal(err)
}
followup := makeSpec("operation-vpc-chain-followup")
followup.ExpectedState.VPCRevision = 10
followup.ExpectedState.VPCDigest = strings.Repeat("d", 64)
if _, _, err := j.Execute(
context.Background(), followup,
func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"applied":true}`), nil
},
); err != nil {
t.Fatalf("repair chain retained a stale blocker: %v", err)
}
}
func TestPeerMigrationTargetCleanupMayCrossOnlyItsExactReceiveFence(t *testing.T) {
j := openTestJournal(t)
migrationID := "0123456789abcdef0123456789abcdef"

@ -133,6 +133,11 @@ func (m *Manager) Apply(ctx context.Context, request ApplyRequest) (Result, erro
return loadErr
}
before := document.networkMap()
if normalized.VPCRepair {
return m.repairApplyLocked(
ctx, document, before, normalized, &result,
)
}
if existing, exists := document.network(normalized.Network.VPCKey); exists {
switch {
case existing.Network.DesiredRevision == normalized.Network.DesiredRevision && existing.Digest == normalized.DesiredDigest:
@ -202,6 +207,11 @@ func (m *Manager) Remove(ctx context.Context, request RemoveRequest) (Result, er
return loadErr
}
existing, exists := document.network(request.VPCKey)
if request.VPCRepair {
return m.repairRemoveLocked(
ctx, document, existing, exists, request, &result,
)
}
if !exists || existing.Network.DesiredRevision != request.ExpectedRevision || existing.Digest != request.ExpectedDigest {
return invalid("vpc_remove_precondition_failed", "VPC removal no longer matches the applied host state")
}
@ -230,6 +240,82 @@ func (m *Manager) Remove(ctx context.Context, request RemoveRequest) (Result, er
return result, err
}
// repairApplyLocked deliberately re-runs the complete host projection even
// when the durable document already matches. It is reachable only through an
// Agent-authorized successor bound to one exact fenced predecessor; ordinary
// retries never enter this path.
func (m *Manager) repairApplyLocked(ctx context.Context, document stateDocument,
before map[string]Network, request ApplyRequest, result *Result) error {
if existing, exists := document.network(request.Network.VPCKey); exists &&
(existing.Network.DesiredRevision != request.Network.DesiredRevision ||
existing.Digest != request.DesiredDigest) {
return invalid("vpc_repair_projection_changed", "VPC repair no longer matches the durable host projection")
}
after := cloneNetworkMap(before)
after[request.Network.VPCKey] = request.Network
if err := validateUniqueBridgeNames(after); err != nil {
return unavailable("vpc_bridge_collision", "VPC bridge identity could not be allocated", err)
}
if err := m.driver.Reconcile(ctx, before, after); err != nil {
return m.rollback(before, after, err)
}
document.putNetwork(storedNetwork{
Network: request.Network, Digest: request.DesiredDigest,
OperationID: request.OperationID,
})
if err := m.save(document); err != nil {
if operation.IsUncertain(err) {
return err
}
return m.rollback(before, after, err)
}
bridge, _ := BridgeName(request.Network.VPCKey)
*result = Result{
VPCKey: request.Network.VPCKey,
DesiredRevision: request.Network.DesiredRevision,
DesiredDigest: request.DesiredDigest,
Bridge: bridge, Applied: true,
}
return nil
}
// repairRemoveLocked converges the host to an absent projection even when a
// prior partial removal already deleted the durable entry. Conflicting newer
// projections remain fail-closed.
func (m *Manager) repairRemoveLocked(ctx context.Context, document stateDocument,
existing storedNetwork, exists bool, request RemoveRequest,
result *Result) error {
if exists && (existing.Network.DesiredRevision != request.ExpectedRevision ||
existing.Digest != request.ExpectedDigest) {
return invalid("vpc_repair_projection_changed", "VPC repair no longer matches the durable host projection")
}
before := document.networkMap()
after := cloneNetworkMap(before)
delete(after, request.VPCKey)
if err := m.driver.Reconcile(ctx, before, after); err != nil {
return m.rollback(before, after, err)
}
document.removeNetwork(removalMarker{
VPCKey: request.VPCKey,
PreviousRevision: request.ExpectedRevision,
PreviousDigest: request.ExpectedDigest,
OperationID: request.OperationID,
})
if err := m.save(document); err != nil {
if operation.IsUncertain(err) {
return err
}
return m.rollback(before, after, err)
}
*result = Result{
VPCKey: request.VPCKey,
DesiredRevision: request.ExpectedRevision,
DesiredDigest: request.ExpectedDigest,
Removed: true,
}
return nil
}
func (m *Manager) Inspect(operationID, vpcKey string, revision uint64, digest string, present bool) (Inspection, error) {
if m == nil || operationID == "" || !vpcKeyPattern.MatchString(vpcKey) || revision == 0 || !lowerHexDigest.MatchString(digest) {
return Inspection{}, nil

@ -58,16 +58,20 @@ type Network struct {
}
type ApplyRequest struct {
OperationID string `json:"operation_id,omitempty"`
DesiredDigest string `json:"desired_digest"`
Network Network `json:"network"`
OperationID string `json:"operation_id,omitempty"`
DesiredDigest string `json:"desired_digest"`
Network Network `json:"network"`
VPCRepair bool `json:"vpc_repair,omitempty"`
SupersedesOperationID string `json:"supersedes_operation_id,omitempty"`
}
type RemoveRequest struct {
OperationID string `json:"operation_id,omitempty"`
VPCKey string `json:"vpc_key"`
ExpectedRevision uint64 `json:"expected_revision"`
ExpectedDigest string `json:"expected_digest"`
OperationID string `json:"operation_id,omitempty"`
VPCKey string `json:"vpc_key"`
ExpectedRevision uint64 `json:"expected_revision"`
ExpectedDigest string `json:"expected_digest"`
VPCRepair bool `json:"vpc_repair,omitempty"`
SupersedesOperationID string `json:"supersedes_operation_id,omitempty"`
}
type Result struct {
@ -222,6 +226,11 @@ func ValidateApplyRequest(request ApplyRequest) (ApplyRequest, error) {
if !operationIDPattern.MatchString(request.OperationID) {
return ApplyRequest{}, invalid("invalid_operation_id", "operation_id must contain 8-80 URL-safe characters")
}
if request.VPCRepair != (request.SupersedesOperationID != "") ||
(request.VPCRepair && (!operationIDPattern.MatchString(request.SupersedesOperationID) ||
request.SupersedesOperationID == request.OperationID)) {
return ApplyRequest{}, invalid("invalid_vpc_repair", "VPC repair requires one different valid predecessor operation")
}
network, err := Normalize(request.Network)
if err != nil {
return ApplyRequest{}, err
@ -241,6 +250,11 @@ func ValidateRemoveRequest(request RemoveRequest) error {
if !operationIDPattern.MatchString(request.OperationID) || !vpcKeyPattern.MatchString(request.VPCKey) || request.ExpectedRevision == 0 || !lowerHexDigest.MatchString(request.ExpectedDigest) {
return invalid("invalid_vpc_remove", "VPC removal identity, revision, or digest is invalid")
}
if request.VPCRepair != (request.SupersedesOperationID != "") ||
(request.VPCRepair && (!operationIDPattern.MatchString(request.SupersedesOperationID) ||
request.SupersedesOperationID == request.OperationID)) {
return invalid("invalid_vpc_repair", "VPC repair requires one different valid predecessor operation")
}
return nil
}

@ -80,6 +80,35 @@ func TestNormalizeRejectsAmbiguousOrUnsafeRoutes(t *testing.T) {
}
}
func TestValidateRequestsRejectMalformedRepairIdentity(t *testing.T) {
apply := applyRequest(t, 1, "operation-vpc-repair-current")
apply.VPCRepair = true
if _, err := ValidateApplyRequest(apply); err == nil {
t.Fatal("repair apply without predecessor was accepted")
}
apply.SupersedesOperationID = apply.OperationID
if _, err := ValidateApplyRequest(apply); err == nil {
t.Fatal("repair apply superseding itself was accepted")
}
apply.SupersedesOperationID = "operation-vpc-repair-prior"
if _, err := ValidateApplyRequest(apply); err != nil {
t.Fatalf("valid repair apply was rejected: %v", err)
}
remove := RemoveRequest{
OperationID: "operation-vpc-repair-remove", VPCKey: testVPCKey,
ExpectedRevision: 1, ExpectedDigest: apply.DesiredDigest,
SupersedesOperationID: "operation-vpc-repair-prior",
}
if err := ValidateRemoveRequest(remove); err == nil {
t.Fatal("remove predecessor without repair marker was accepted")
}
remove.VPCRepair = true
if err := ValidateRemoveRequest(remove); err != nil {
t.Fatalf("valid repair remove was rejected: %v", err)
}
}
type fakeDriver struct {
calls [][2]map[string]Network
errors []error
@ -141,6 +170,47 @@ func TestManagerAppliesReplaysAndRemovesWholeState(t *testing.T) {
}
}
func TestManagerRepairForcesRuntimeConvergenceAndRebindsMarker(t *testing.T) {
driver := &fakeDriver{}
manager := testManager(t, driver)
original := applyRequest(t, 5, "operation-vpc-repair-original")
if _, err := manager.Apply(t.Context(), original); err != nil {
t.Fatal(err)
}
repair := original
repair.OperationID = "operation-vpc-repair-apply"
repair.VPCRepair = true
repair.SupersedesOperationID = original.OperationID
if result, err := manager.Apply(t.Context(), repair); err != nil ||
!result.Applied || len(driver.calls) != 2 ||
!reflect.DeepEqual(driver.calls[1][0], driver.calls[1][1]) {
t.Fatalf("repair apply=%#v calls=%#v err=%v", result, driver.calls, err)
}
inspection, err := manager.Inspect(
repair.OperationID, testVPCKey, 5, repair.DesiredDigest, true,
)
if err != nil || !inspection.Applied {
t.Fatalf("repair marker was not rebound: %#v err=%v", inspection, err)
}
remove := RemoveRequest{
OperationID: "operation-vpc-repair-remove-source",
VPCKey: testVPCKey, ExpectedRevision: 5,
ExpectedDigest: repair.DesiredDigest,
}
if _, err := manager.Remove(t.Context(), remove); err != nil {
t.Fatal(err)
}
repairRemove := remove
repairRemove.OperationID = "operation-vpc-repair-remove"
repairRemove.VPCRepair = true
repairRemove.SupersedesOperationID = remove.OperationID
if result, err := manager.Remove(t.Context(), repairRemove); err != nil ||
!result.Removed || len(driver.calls) != 4 ||
len(driver.calls[3][0]) != 0 || len(driver.calls[3][1]) != 0 {
t.Fatalf("repair remove=%#v calls=%#v err=%v", result, driver.calls, err)
}
}
func TestManagerRestoresOnlyDurablyCommittedProjection(t *testing.T) {
path := t.TempDir() + "/vpc-state.json"
applyDriver := &fakeDriver{}

@ -42,7 +42,8 @@ type Request struct {
// SupersededOperation is a non-secret immutable snapshot injected only by
// the parent Agent after it authorizes an exceptional successor. The child
// uses a separate Journal, so it cannot load the predecessor itself.
SupersededOperation *operation.Record `json:"superseded_operation,omitempty"`
SupersededOperation *operation.Record `json:"superseded_operation,omitempty"`
SupersededExpectedState *operation.ExpectedState `json:"superseded_expected_state,omitempty"`
}
// RequestIdentity is the non-secret, immutable subset persisted beside an
@ -126,14 +127,20 @@ func (r Request) Validate() error {
}
if r.SupersededOperation != nil {
prior := r.SupersededOperation
vpcRepair := strings.HasSuffix(r.RequestTarget, "/repair")
if prior.OperationID == "" || prior.OperationID == r.OperationID ||
!operation.ValidOperationIdentity(prior.Action, prior.ResourceID, prior.Fingerprint) ||
prior.State != operation.StateFailed || !prior.EffectMayHaveApplied ||
prior.SafeToResubmit || prior.Tombstone ||
!(strings.HasSuffix(r.RequestTarget, "/force-delete") ||
strings.HasSuffix(r.RequestTarget, "/peer-migration/cleanup-target")) {
strings.HasSuffix(r.RequestTarget, "/peer-migration/cleanup-target") ||
vpcRepair) ||
(vpcRepair != (r.SupersededExpectedState != nil)) ||
(vpcRepair && !prior.ExecutionFenced) {
return errors.New("worker superseded operation is invalid")
}
} else if r.SupersededExpectedState != nil {
return errors.New("worker superseded expected state has no operation")
}
if Fingerprint(r.Method, r.RequestTarget, []byte(r.Body)) != r.Fingerprint {
return errors.New("worker request fingerprint does not match target and body")

@ -8,7 +8,7 @@ use App\Services\Config;
/** Initializes only the clean-break PHP business and Agent log schemas. */
final class CloudSetup
{
private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v5-fences';
private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v6-repair';
private const MAIN_TABLES = [
'cloud_regions', 'cloud_network_lines', 'cloud_resource_pools',
'cloud_hosts', 'cloud_host_agents',
@ -323,6 +323,7 @@ final class CloudSetup
'cloud_job_backup', 'cloud_job_backup_restore',
'cloud_job_backup_recovery',
'cloud_job_vpc', 'cloud_job_vpc_port', 'cloud_job_vpc_batch',
'cloud_job_supersedes', 'cloud_job_vpc_repair_root',
],
'cloud_agent_completion_events' => [
'PRIMARY', 'uniq_cloud_completion_operation',

@ -88,4 +88,17 @@ final class CloudAdminOperationsController
202
)->withHeader('Cache-Control', 'private, no-store');
}
public function repairVpc($request, $response, $args)
{
return $response->withJson(
CloudAdminOperationRecoveryService::repairVpc(
$request->getAttribute('cloud_actor')
?? CloudAccess::actor(false),
(string) ($args['id'] ?? ''),
CloudInput::object($request)
),
202
)->withHeader('Cache-Control', 'private, no-store');
}
}

@ -14,6 +14,8 @@ final class CloudJob extends Model
'vpc_id' => 'int',
'vpc_port_id' => 'int',
'vpc_batch_id' => 'int',
'supersedes_job_id' => 'int',
'vpc_repair_root_job_id' => 'int',
];
/**

@ -532,6 +532,37 @@ final class CloudAdminOperationRecoveryService
];
}
/** Queue a new VPC convergence operation after one fresh Agent inspection. */
public static function repairVpc(
array $actor,
string $id,
array $input
): array {
CloudInput::only($input, ['acknowledge_repair']);
if (($input['acknowledge_repair'] ?? null) !== true) {
throw new CloudError('必须明确确认将创建新的 VPC 修复任务', 409);
}
if (!($actor['is_admin'] ?? false)
|| !($actor['is_super_admin'] ?? false)
|| !($actor['can_manage'] ?? false)) {
throw new CloudError('只有超级管理员可以修复 VPC 状态', 403);
}
CloudAccess::manage($actor);
if (preg_match('/^[1-9][0-9]{0,17}$/D', $id) !== 1) {
throw new CloudError('编号无效', 400);
}
$inspection = self::inspect($actor, $id);
$job = CloudVpcPortService::repairUnknownEffect(
$actor,
(int) $id,
$inspection
);
return [
'task_id' => (string) $job->id,
'status_url' => '/api/v1/tasks/' . (int) $job->id,
];
}
/** @return array{0:CloudJob,1:CloudAgentClient} */
private static function context(
array $actor,
@ -565,7 +596,9 @@ final class CloudAdminOperationRecoveryService
'effect_resolution', 'effect_resolution_reason', 'dispatched_at',
'finalization_incident', 'finalization_error_code', 'result_json',
'sealed_payload', 'instance_id', 'snapshot_id', 'backup_id',
'backup_restore_id', 'parent_job_id',
'backup_restore_id', 'backup_recovery_id', 'parent_job_id',
'supersedes_job_id', 'vpc_repair_root_job_id',
'vpc_id', 'vpc_port_id', 'vpc_batch_id',
'user_id', 'order_id',
])->where('id', (int) $id)->first();
if ($job === null) {

@ -325,7 +325,8 @@ final class CloudAdminOperationsService
foreach ([
'id', 'operation_id', 'request_idempotency_key',
'request_fingerprint', 'user_id', 'host_id', 'kind',
'parent_job_id', 'execution_mode', 'invoice_id', 'phase',
'parent_job_id', 'supersedes_job_id',
'vpc_repair_root_job_id', 'execution_mode', 'invoice_id', 'phase',
'order_id', 'instance_id', 'snapshot_id',
'outcome', 'status', 'attempts', 'dispatch_attempts', 'error_code',
'dispatched_at', 'dispatch_deadline_at', 'dispatch_started_at',
@ -440,6 +441,9 @@ final class CloudAdminOperationsService
[$canForceDelete, $forceDeleteReason] = self::forceDeleteable(
$job, $canManage
);
[$canRepairVpc, $repairVpcReason] = self::vpcRepairable(
$job, $canManage
);
$recovery = CloudJobService::recoveryPresentation($job, true);
$effectScope = $recovery === null
? CloudJobService::effectScope($job) : null;
@ -477,6 +481,10 @@ final class CloudAdminOperationsService
'resolve_effect_reason' => $resolveReason,
'can_force_delete' => $canForceDelete,
'force_delete_reason' => $forceDeleteReason,
'can_repair_vpc' => $canRepairVpc,
'repair_vpc_reason' => $repairVpcReason,
'supersedes_job_id' => $job->supersedes_job_id === null
? null : (string) $job->supersedes_job_id,
'dispatch_attempts' => $job->dispatch_attempts === null
? null : CloudDispatchLedger::attempts($job),
'dispatch_state' => CloudDispatchLedger::isInvalid($job)
@ -681,6 +689,35 @@ final class CloudAdminOperationsService
return [true, '将创建新的任务并再次确认数据丢失风险'];
}
/** @return array{0:bool,1:string} */
private static function vpcRepairable($job, bool $canManage): array
{
if (!$canManage) {
return [false, '只有超级管理员可以创建 VPC 修复任务'];
}
if (!in_array((string) $job->kind, [
'vpc_host_apply', 'vpc_host_remove',
'vpc_port_attach', 'vpc_port_detach',
], true)) {
return [false, '当前任务不是 VPC 操作'];
}
if (CloudDispatchLedger::isInvalid($job)
|| CloudDispatchLedger::hasRepairIncident($job)
|| (bool) ($job->finalization_incident ?? false)
|| (string) $job->execution_mode !== 'agent'
|| !CloudDispatchLedger::isDispatched($job)
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown') {
return [false, 'VPC 任务不处于可修复的影响未知状态'];
}
return [true,
'服务端会先读取宿主机新鲜证据,再创建具有新操作标识的精确修复任务'];
}
private static function timestamp($value): ?int
{
return $value !== null && (int) $value > 0 ? (int) $value : null;

@ -544,6 +544,16 @@ final class CloudJobService
'vpc_route_predecessors.kind',
$routeKinds
)
->where(static function ($outsideRepair): void {
$outsideRepair->whereNull(
'cloud_jobs.vpc_repair_root_job_id'
)->orWhereRaw(
'COALESCE('
. 'vpc_route_predecessors.vpc_repair_root_job_id, '
. 'vpc_route_predecessors.id) <> '
. 'cloud_jobs.vpc_repair_root_job_id'
);
})
->where(static function ($blocking): void {
$blocking->where(
'vpc_route_predecessors.phase',

@ -10,6 +10,8 @@ use App\Models\CloudJob;
use App\Models\CloudResourcePool;
use App\Models\CloudVpc;
use App\Models\CloudVpcHostState;
use App\Models\CloudVpcMembershipBatch;
use App\Models\CloudVpcMembershipBatchItem;
use App\Models\CloudVpcPort;
/**
@ -802,6 +804,40 @@ final class CloudVpcPortService
|| ($body['operation_id'] ?? null) !== (string) $job->operation_id) {
throw new CloudError('VPC 任务与资源账本不一致', 409);
}
$repair = $job->supersedes_job_id !== null;
$prior = null;
if ($repair) {
$prior = CloudJob::query()->where(
'id', (int) $job->supersedes_job_id
)->first();
$rootId = (int) ($job->vpc_repair_root_job_id ?? 0);
if ($prior === null || $rootId < 1
|| ($payload['vpc_repair'] ?? null) !== true
|| (int) ($payload['vpc_repair_root_job_id'] ?? 0)
!== $rootId
|| ($body['vpc_repair'] ?? null) !== true
|| ($body['supersedes_operation_id'] ?? null)
!== (string) $prior->operation_id
|| (int) $prior->instance_id !== (int) $job->instance_id
|| (int) $prior->vpc_id !== (int) $job->vpc_id
|| (int) $prior->vpc_port_id !== (int) $job->vpc_port_id
|| (int) $prior->host_id !== (int) $job->host_id
|| (string) $prior->kind !== $kind
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'failed'
|| !(bool) $prior->effect_may_have_applied
|| (bool) $prior->safe_to_resubmit
|| (string) $prior->effect_resolution !== 'unknown'
|| (bool) $prior->finalization_incident
|| (int) ($prior->vpc_repair_root_job_id ?? $prior->id)
!== $rootId) {
throw new CloudError('VPC 修复任务继任账本不一致', 409);
}
} elseif (($payload['vpc_repair'] ?? false) === true
|| array_key_exists('vpc_repair', $body)
|| array_key_exists('supersedes_operation_id', $body)) {
throw new CloudError('普通 VPC 任务包含修复字段', 409);
}
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
@ -830,12 +866,18 @@ final class CloudVpcPortService
}
$expected = self::portBody($job, $vpc, $port, $state);
}
if ($repair) {
$expected['vpc_repair'] = true;
$expected['supersedes_operation_id'] =
(string) $prior->operation_id;
}
$expectedTarget = in_array($kind,
['vpc_host_apply', 'vpc_host_remove'], true)
? '/v1/vpc/networks/' . rawurlencode((string) $vpc->vpc_key)
. ($kind === 'vpc_host_apply' ? '/apply' : '/remove')
. ($repair ? '/repair' : '')
: '/v1/vms/' . rawurlencode(strtolower((string) $instance->uuid))
. '/vpc-port';
. '/vpc-port' . ($repair ? '/repair' : '');
$expectedMethod = match ($kind) {
'vpc_host_apply', 'vpc_host_remove', 'vpc_port_attach' => 'POST',
'vpc_port_detach' => 'DELETE',
@ -882,6 +924,16 @@ final class CloudVpcPortService
if ($port === null || $vpc === null || $instance === null) {
throw new CloudError('VPC 任务资源账本已丢失', 409);
}
if ($job->supersedes_job_id !== null) {
if ($status === 'succeeded') {
self::settleRepairChainLocked($job);
} elseif ($status === 'failed') {
self::restoreRepairPredecessorLocked(
$job, $vpc, $port
);
return;
}
}
if ($job->vpc_batch_id !== null) {
CloudVpcMembershipBatchService::finishChildLocked(
$job, $status, $result, $operation, $vpc, $port, $instance
@ -904,6 +956,134 @@ final class CloudVpcPortService
);
}
/** Resolve only VPC safety metadata after the successor proves convergence. */
private static function settleRepairChainLocked(CloudJob $job): void
{
$rootId = (int) ($job->vpc_repair_root_job_id ?? 0);
$priorId = (int) ($job->supersedes_job_id ?? 0);
if ($rootId < 1 || $priorId < 1) {
throw new CloudError('VPC 修复继任链缺失', 409);
}
$seen = [(int) $job->id => true];
$now = time();
for ($depth = 0; $depth < 32; $depth++) {
if ($priorId < 1 || isset($seen[$priorId])) {
throw new CloudError('VPC 修复继任链包含循环', 409);
}
$seen[$priorId] = true;
$prior = CloudJob::query()->where('id', $priorId)
->lockForUpdate()->first();
if ($prior === null
|| (string) $prior->kind !== (string) $job->kind
|| (int) $prior->instance_id !== (int) $job->instance_id
|| (int) $prior->vpc_id !== (int) $job->vpc_id
|| (int) $prior->vpc_port_id !== (int) $job->vpc_port_id
|| (int) $prior->host_id !== (int) $job->host_id
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'failed'
|| !(bool) $prior->effect_may_have_applied
|| (bool) $prior->safe_to_resubmit
|| (string) $prior->effect_resolution !== 'unknown'
|| (bool) $prior->finalization_incident
|| (int) ($prior->vpc_repair_root_job_id ?? $prior->id)
!== $rootId) {
throw new CloudError('VPC 修复继任链已变化', 409);
}
$nextId = (int) ($prior->supersedes_job_id ?? 0);
$prior->effect_resolution = 'applied';
$prior->effect_resolved_at = $now;
$prior->effect_resolved_by = null;
$prior->effect_resolution_reason = substr(
'superseded_by_vpc_repair:' . (int) $job->id . ':'
. (string) $job->operation_id,
0,
500
);
$prior->available_at = 0;
$prior->read_at = null;
$prior->updated_at = $now;
$prior->save();
if ($nextId < 1) {
if ((int) $prior->id !== $rootId) {
throw new CloudError('VPC 修复根任务身份不一致', 409);
}
return;
}
$priorId = $nextId;
}
throw new CloudError('VPC 修复继任链超过安全上限', 409);
}
/** A proved no-effect repair restores the still-fenced predecessor tip. */
private static function restoreRepairPredecessorLocked(
CloudJob $job,
CloudVpc $vpc,
CloudVpcPort $port
): void {
$prior = CloudJob::query()->where(
'id', (int) $job->supersedes_job_id
)->lockForUpdate()->first();
if ($prior === null
|| (string) $prior->kind !== (string) $job->kind
|| (string) $prior->effect_resolution !== 'unknown'
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'failed') {
throw new CloudError('VPC 修复前序任务已变化', 409);
}
$now = time();
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $job->vpc_id)
->where('host_id', (int) $job->host_id)
->lockForUpdate()->first();
if ($state === null || !hash_equals(
(string) $state->last_operation_id,
(string) $job->operation_id
)) {
throw new CloudError('VPC 宿主机修复回退代次不一致', 409);
}
$state->last_operation_id = (string) $prior->operation_id;
$state->sync_state = 'unknown';
$state->last_error_code = 'vpc_result_uncertain';
$state->last_failure_at = $now;
$state->updated_at = $now;
$state->save();
} else {
if (!hash_equals(
(string) $port->last_operation_id,
(string) $job->operation_id
)) {
throw new CloudError('VPC 网卡修复回退代次不一致', 409);
}
$port->last_operation_id = (string) $prior->operation_id;
$port->last_error_code = 'vpc_result_uncertain';
$port->updated_at = $now;
$port->save();
}
if ($job->vpc_batch_id !== null
&& in_array((string) $job->kind,
['vpc_port_attach', 'vpc_port_detach'], true)) {
$item = CloudVpcMembershipBatchItem::query()
->where('batch_id', (int) $job->vpc_batch_id)
->where('instance_id', (int) $job->instance_id)
->where('vpc_port_id', (int) $job->vpc_port_id)
->lockForUpdate()->first();
if ($item === null
|| (int) $item->child_job_id !== (int) $job->id) {
throw new CloudError('VPC 批次修复回退代次不一致', 409);
}
$item->child_job_id = (int) $prior->id;
$item->state = 'unknown';
$item->last_error_code = 'vpc_result_uncertain';
$item->updated_at = $now;
$item->save();
}
$vpc->state = 'degraded';
$vpc->updated_at = $now;
$vpc->save();
}
public static function markJobUncertain(CloudJob $job): void
{
$kind = (string) $job->kind;
@ -946,6 +1126,296 @@ final class CloudVpcPortService
}
}
/**
* Queue one new, explicitly linked repair generation for an exact VPC
* operation whose Agent journal is reachable but still inconclusive.
* The failed operation is never replayed.
*/
public static function repairUnknownEffect(
array $actor,
int $jobId,
array $inspection
): CloudJob {
CloudAccess::manage($actor);
$reference = CloudJob::query()->where('id', $jobId)->first();
if ($reference === null) {
throw new CloudError('任务不存在', 404);
}
if (!in_array((string) $reference->kind, self::JOB_KINDS, true)
|| (int) ($reference->instance_id ?? 0) < 1
|| (int) ($reference->vpc_id ?? 0) < 1
|| (int) ($reference->vpc_port_id ?? 0) < 1
|| (int) ($reference->host_id ?? 0) < 1) {
throw new CloudError('该任务不是可修复的 VPC 操作', 409);
}
self::assertRepairInspection($reference, $inspection);
return (new CloudJob())->getConnection()->transaction(
function () use ($reference, $inspection): CloudJob {
$instance = CloudVpcLockOrder::instance(
(int) $reference->instance_id
);
$vpc = CloudVpcLockOrder::vpcById(
(int) $reference->vpc_id
);
$port = CloudVpcLockOrder::portById(
(int) $reference->vpc_port_id
);
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $reference->vpc_id)
->where('host_id', (int) $reference->host_id)
->lockForUpdate()->first();
if ($instance === null || $vpc === null || $port === null
|| $state === null
|| (int) $port->instance_id !== (int) $instance->id
|| (int) $port->vpc_id !== (int) $vpc->id
|| (int) $reference->host_id
!== (int) $state->host_id) {
throw new CloudError('VPC 修复资源账本不完整', 409);
}
$batch = null;
$item = null;
if ($reference->vpc_batch_id !== null) {
$batch = CloudVpcMembershipBatch::query()
->where('id', (int) $reference->vpc_batch_id)
->lockForUpdate()->first();
if ($batch === null
|| (string) $batch->state !== 'blocked'
|| (int) $batch->vpc_id !== (int) $vpc->id) {
throw new CloudError('VPC 批次不处于可修复状态', 409);
}
if (in_array((string) $reference->kind,
['vpc_port_attach', 'vpc_port_detach'], true)) {
$item = CloudVpcMembershipBatchItem::query()
->where('batch_id', (int) $batch->id)
->where('instance_id', (int) $instance->id)
->where('vpc_port_id', (int) $port->id)
->lockForUpdate()->first();
}
}
$prior = CloudJob::query()->where(
'id', (int) $reference->id
)->lockForUpdate()->first();
if ($prior === null) {
throw new CloudError('待修复任务已不存在', 409);
}
self::assertRepairCandidateLocked(
$prior, $instance, $vpc, $port, $state,
$batch, $item, $inspection
);
$rootId = (int) ($prior->vpc_repair_root_job_id
?? $prior->id);
$chain = CloudJob::query()
->where(static function ($query) use ($rootId): void {
$query->where('id', $rootId)
->orWhere('vpc_repair_root_job_id', $rootId);
})->orderBy('id')->lockForUpdate()->get();
$unknownGenerations = $chain->filter(
static fn (CloudJob $candidate): bool =>
(string) $candidate->effect_resolution === 'unknown'
);
$latestUnknown = $unknownGenerations->last();
if ($chain->isEmpty() || $unknownGenerations->count() > 32
|| $chain->contains(static fn (CloudJob $candidate): bool =>
(string) $candidate->phase !== 'terminal')
|| !$latestUnknown instanceof CloudJob
|| (int) $latestUnknown->id !== (int) $prior->id) {
throw new CloudError('VPC 修复链已有其他活动代次', 409);
}
foreach ($chain as $candidate) {
if ((int) $candidate->instance_id !== (int) $instance->id
|| (int) $candidate->vpc_id !== (int) $vpc->id
|| (int) $candidate->vpc_port_id !== (int) $port->id
|| (int) $candidate->host_id !== (int) $prior->host_id
|| (string) $candidate->kind
!== (string) $prior->kind) {
throw new CloudError('VPC 修复链资源身份不一致', 409);
}
}
$payload = CloudSecrets::decrypt(
(string) $prior->sealed_payload, 'job-payload'
);
self::assertJobRunnable($prior, $payload);
$job = self::baseJob(
(string) $prior->kind,
$vpc,
$port,
(int) $prior->host_id,
$prior->vpc_batch_id === null
? null : (int) $prior->vpc_batch_id
);
$job->supersedes_job_id = (int) $prior->id;
$job->vpc_repair_root_job_id = $rootId;
$extra = [];
foreach ([
'maintenance_migration_id', 'vpc_migration_direction',
'vpc_batch_id', 'vpc_batch_revision',
] as $key) {
if (array_key_exists($key, $payload)) {
$extra[$key] = $payload[$key];
}
}
if ((string) $prior->kind === 'vpc_host_apply'
&& is_array($payload['host_projection'] ?? null)
&& !array_is_list($payload['host_projection'])) {
$extra['host_projection'] = $payload['host_projection'];
}
if ($job->vpc_batch_id !== null
&& in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$revision = (int) ($extra['vpc_batch_revision']
?? $batch?->desired_revision ?? 0);
if ($revision < 1) {
throw new CloudError('VPC 批次修复缺少投影代次', 409);
}
$job->request_idempotency_key = substr(
self::batchProjectionRequestPrefix(
(int) $job->vpc_batch_id,
$revision
) . (string) $job->kind . '-'
. (int) $job->host_id . '-repair-'
. (string) $job->operation_id,
0,
128
);
}
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$body = (string) $job->kind === 'vpc_host_apply'
? self::hostApplyBody($job, $vpc, $state, $payload)
: self::hostRemoveBody($job, $vpc, $state);
$target = '/v1/vpc/networks/'
. rawurlencode((string) $vpc->vpc_key)
. ((string) $job->kind === 'vpc_host_apply'
? '/apply/repair' : '/remove/repair');
$method = 'POST';
} else {
if (!self::hostReady($state)) {
throw new CloudError(
'VPC 网卡修复所需宿主机路由未就绪', 409
);
}
$body = self::portBody($job, $vpc, $port, $state);
$target = '/v1/vms/' . rawurlencode(
strtolower((string) $instance->uuid)
) . '/vpc-port/repair';
$method = (string) $job->kind === 'vpc_port_attach'
? 'POST' : 'DELETE';
}
$body['vpc_repair'] = true;
$body['supersedes_operation_id'] =
(string) $prior->operation_id;
self::sealJob(
$job, $vpc, $port, $method, $target, $body,
$extra + [
'vpc_repair' => true,
'vpc_repair_root_job_id' => $rootId,
]
);
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$state->last_operation_id = (string) $job->operation_id;
$state->sync_state = 'pending';
$state->last_error_code = '';
$state->updated_at = time();
$state->save();
} else {
$port->last_operation_id = (string) $job->operation_id;
$port->last_error_code = '';
$port->updated_at = time();
$port->save();
if ($item !== null) {
$item->child_job_id = (int) $job->id;
$item->state = 'queued';
$item->last_error_code = '';
$item->updated_at = time();
$item->save();
}
}
return $job;
},
3
);
}
private static function assertRepairInspection(
CloudJob $job,
array $inspection
): void {
if (($inspection['operation_id'] ?? null)
!== (string) $job->operation_id
|| ($inspection['journal_status'] ?? null) !== 'failed'
|| ($inspection['outcome'] ?? null) !== 'inconclusive'
|| ($inspection['effect_resolution'] ?? null) !== 'inconclusive'
|| ($inspection['fence_state'] ?? null) !== 'active'
|| ($inspection['fence_resolution_required'] ?? null) !== true
|| array_key_exists('completion_delivery_state', $inspection)
|| !is_int($inspection['checked_at'] ?? null)
|| $inspection['checked_at'] < time() - 120
|| $inspection['checked_at'] > time() + 5) {
throw new CloudError(
'宿主机没有提供可执行 VPC 修复的新鲜不一致证据', 409
);
}
}
private static function assertRepairCandidateLocked(
CloudJob $job,
CloudInstance $instance,
CloudVpc $vpc,
CloudVpcPort $port,
CloudVpcHostState $state,
?CloudVpcMembershipBatch $batch,
?CloudVpcMembershipBatchItem $item,
array $inspection
): void {
self::assertRepairInspection($job, $inspection);
if (!in_array((string) $job->kind, self::JOB_KINDS, true)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (int) $job->dispatch_attempts !== 1
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown'
|| (bool) $job->finalization_incident
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->vpc_id !== (int) $vpc->id
|| (int) $job->vpc_port_id !== (int) $port->id
|| (int) $job->host_id !== (int) $state->host_id
|| (int) $job->user_id !== (int) $port->user_id) {
throw new CloudError('VPC 任务已不处于可修复状态', 409);
}
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
if (!hash_equals(
(string) $state->last_operation_id,
(string) $job->operation_id
) || (string) $state->sync_state !== 'unknown') {
throw new CloudError('VPC 宿主机修复代次已过期', 409);
}
} elseif (!hash_equals(
(string) $port->last_operation_id,
(string) $job->operation_id
) || !in_array((string) $port->state,
['preparing', 'detaching'], true)) {
throw new CloudError('VPC 网卡修复代次已过期', 409);
}
if ($job->vpc_batch_id !== null
&& ($batch === null || (string) $batch->state !== 'blocked'
|| (in_array((string) $job->kind,
['vpc_port_attach', 'vpc_port_detach'], true)
&& ($item === null
|| (int) $item->child_job_id !== (int) $job->id
|| (string) $item->state !== 'unknown')))) {
throw new CloudError('VPC 批次修复子任务已过期', 409);
}
}
private static function finishHostJobLocked(CloudJob $job,
string $status, $result, array $operation, CloudVpc $vpc,
CloudVpcPort $port, CloudInstance $instance): void

@ -142,6 +142,8 @@ $app->group('/api/v1', function () {
$this->post('/admin/operations/jobs/{id:[0-9]+}/resolve-effect',
'App\Controllers\CloudAdminOperationsController:resolveEffect');
$this->post('/admin/operations/jobs/{id:[0-9]+}/force-delete', 'App\Controllers\CloudAdminOperationsController:forceDelete');
$this->post('/admin/operations/jobs/{id:[0-9]+}/repair-vpc',
'App\Controllers\CloudAdminOperationsController:repairVpc');
$this->get('/admin/billing/invoices',
'App\Controllers\CloudAdminBillingController:invoices');
$this->get('/admin/billing/invoices/{id:[0-9]+}',

@ -24,6 +24,7 @@ const state = {
opener: null, scroll: null, actionRequest: null, forceDeleteAccepted: false,
forceDeleteBlocked: false, forceDeleteAcceptedIds: new Set(), forceDeleteBlockedIds: new Set(),
forceDeleteObservers: new Map(),
vpcRepairAccepted: false, vpcRepairAcceptedIds: new Set(),
inspection: null, effectResolved: false,
};
@ -394,6 +395,11 @@ function renderDetailActions() {
inspect.hidden = state.detailJob?.can_inspect !== true;
inspect.disabled = busy;
inspect.textContent = busy && state.actionRequest?.kind === 'inspect' ? '正在检查…' : '只读检查宿主机';
const repairVpc = byId('operation-detail-repair-vpc');
repairVpc.hidden = !canManage() || state.detailJob?.can_repair_vpc !== true;
repairVpc.disabled = busy || state.vpcRepairAccepted;
repairVpc.textContent = state.vpcRepairAccepted ? '修复任务已接收'
: busy && state.actionRequest?.kind === 'repair-vpc' ? '正在创建…' : '创建 VPC 修复任务';
const force = byId('operation-detail-force-delete');
force.hidden = !canManage() || state.detailJob?.can_force_delete !== true;
force.disabled = busy || state.forceDeleteAccepted || state.forceDeleteBlocked;
@ -583,6 +589,52 @@ async function forceDelete(id) {
}
}
async function repairVpc(id) {
if (!canManage() || state.detailId !== String(id)
|| state.detailJob?.can_repair_vpc !== true || state.actionRequest
|| state.vpcRepairAccepted) return;
if (!window.confirm('系统将先执行一次只读宿主机检查;只有原 VPC 操作仍处于影响未知且 Agent 可达时,才会创建新的修复任务。\n\n原操作不会重放,修复任务使用新的操作标识并保留继任关系。确认继续?')) return;
const controller = new AbortController();
controller.kind = 'repair-vpc'; state.actionRequest = controller;
renderDetailActions(); detailNotice('正在检查宿主机并创建精确 VPC 修复任务…');
try {
const payload = await acceptedTask(`/api/v1/admin/operations/jobs/${encodeURIComponent(id)}/repair-vpc`, controller, 45000, {
method: 'POST', body: JSON.stringify({ acknowledge_repair: true }),
});
state.vpcRepairAccepted = true;
state.vpcRepairAcceptedIds.add(String(id));
announceTaskAccepted(payload, { source: 'admin-vpc-repair' });
const taskId = String(payload.task_id || '').trim();
detailNotice(`VPC 修复任务已接收${taskId ? `(#${taskId})` : ''};原任务不会重放。`);
notice(`VPC 修复任务已接收${taskId ? `(#${taskId})` : ''}。`);
if (taskId) {
const handle = observeTask(payload, {
onTerminal: task => {
const succeeded = taskSucceededWithoutRecovery(task);
const message = succeeded
? `VPC 修复任务 #${task.id} 已成功收敛。`
: taskFailureMessage(task, `VPC 修复任务 #${task.id} 失败。`);
void (async () => {
await loadPage(listUrl(), 'none');
if (byId('operation-detail').open) await openDetail(task.id, null);
notice(message, !succeeded);
if (byId('operation-detail').open) detailNotice(message, !succeeded);
})();
},
});
void handle.done;
}
} catch (error) {
if (controller.signal.aborted && controller.signal.reason !== 'timeout') return;
detailNotice(`VPC 修复任务未创建:${error.message}`, true);
notice(`VPC 修复请求未确认:${error.message}`, true);
} finally {
if (state.actionRequest === controller) {
state.actionRequest = null; renderDetailActions();
}
}
}
function finishDetailClose(id) {
state.detailClosing = false;
document.documentElement.classList.remove('admin-instance-detail-open');
@ -602,6 +654,7 @@ function closeDetail() {
state.actionRequest?.abort('closed'); state.actionRequest = null;
state.detailId = null; state.detailJob = null; state.forceDeleteAccepted = false;
state.forceDeleteBlocked = false;
state.vpcRepairAccepted = false;
state.inspection = null; state.effectResolved = false;
state.detailClosing = true;
void window.VenixDrawerMotion.closeDrawerWithMotion(byId('operation-detail')).then(closed => {
@ -615,11 +668,13 @@ async function openDetail(id, opener) {
state.detailRequest?.abort('replaced');
state.detailId = id; state.detailJob = null; state.opener = opener; state.forceDeleteAccepted = state.forceDeleteAcceptedIds.has(String(id));
state.forceDeleteBlocked = state.forceDeleteBlockedIds.has(String(id));
state.vpcRepairAccepted = state.vpcRepairAcceptedIds.has(String(id));
state.inspection = null; state.effectResolved = false;
const controller = new AbortController(); state.detailRequest = controller;
byId('operation-detail-title').textContent = `任务 #${id}`;
byId('operation-detail-body').replaceChildren(); byId('operation-detail-links').replaceChildren();
byId('operation-detail-inspect').hidden = true;
byId('operation-detail-repair-vpc').hidden = true;
byId('operation-detail-force-delete').hidden = true;
byId('operation-effect-resolution').hidden = true;
byId('operation-effect-value').value = '';
@ -672,6 +727,9 @@ if (root) {
byId('operation-detail-force-delete').addEventListener('click', event => {
event.preventDefault(); if (state.detailId) void forceDelete(state.detailId);
});
byId('operation-detail-repair-vpc').addEventListener('click', event => {
event.preventDefault(); if (state.detailId) void repairVpc(state.detailId);
});
byId('operation-effect-submit').addEventListener('click', event => {
event.preventDefault(); if (state.detailId) void resolveEffect(state.detailId);
});

@ -39,7 +39,7 @@
<dialog data-admin-instance-drawer data-admin-operations-drawer id="operation-detail" aria-labelledby="operation-detail-title">
<header class="admin-instance-drawer-header"><div><p class="text-muted fs-13 mb-1">任务详情</p><h2 class="h4 fw-semibold mb-0" id="operation-detail-title">任务</h2></div><button class="admin-modal-close position-static flex-shrink-0" id="operation-detail-close" type="button" aria-label="关闭任务详情" autofocus><i class="mdi mdi-close" aria-hidden="true"></i></button></header>
<div class="admin-instance-drawer-body"><p class="alert alert-info" id="operation-detail-state" role="status" aria-live="polite">正在读取任务详情…</p><div id="operation-detail-body"></div><section class="alert alert-warning mt-4" id="operation-effect-resolution" hidden><h3 class="h6">人工收敛任务级资源安全栅栏</h3><p class="fs-13">仅在已取得可审计的真实效果证据后使用。原任务仍保持失败,不会重放 Agent 或支付请求;服务端只会执行该任务类型已实现的原子账本收尾。</p><label class="form-label" for="operation-effect-value">人工核实结果</label><select class="form-select" id="operation-effect-value"><option value="">请选择</option><option value="applied">操作效果已经发生</option><option value="not_applied">操作效果未发生</option></select><label class="form-label mt-3" for="operation-effect-reason">审计证据与说明(20–500 字符)</label><textarea class="form-control" id="operation-effect-reason" minlength="20" maxlength="500" rows="4"></textarea><button class="btn btn-danger btn-sm mt-3" id="operation-effect-submit" type="button">人工确认并收敛安全栅栏</button></section></div>
<footer class="admin-instance-drawer-footer"><span class="text-muted fs-12">失败任务不会自动重提。详情标为持续自动核对时,只需修复所示通信或数据异常;系统会继续收敛。</span><div class="d-flex flex-wrap gap-2"><button class="btn btn-primary btn-sm" id="operation-detail-inspect" type="button" hidden>只读检查宿主机</button><button class="btn btn-danger btn-sm" id="operation-detail-force-delete" type="button" hidden>确认强制删除</button><div class="d-flex flex-wrap gap-2" id="operation-detail-links"></div></div></footer>
<footer class="admin-instance-drawer-footer"><span class="text-muted fs-12">失败任务不会自动重提。详情标为持续自动核对时,只需修复所示通信或数据异常;系统会继续收敛。</span><div class="d-flex flex-wrap gap-2"><button class="btn btn-primary btn-sm" id="operation-detail-inspect" type="button" hidden>只读检查宿主机</button><button class="btn btn-warning btn-sm" id="operation-detail-repair-vpc" type="button" hidden>创建 VPC 修复任务</button><button class="btn btn-danger btn-sm" id="operation-detail-force-delete" type="button" hidden>确认强制删除</button><div class="d-flex flex-wrap gap-2" id="operation-detail-links"></div></div></footer>
</dialog>
<script id="admin-operations-bootstrap" type="application/json">{$operations_bootstrap_json|default:'null' nofilter}</script>
<script type="module" src="/cloud/admin-operations.js?v={$venix_asset_version}"></script>

@ -1,2 +1,2 @@
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
{assign var=venix_asset_version value='2026102906' scope='global'}
{assign var=venix_asset_version value='2026102907' scope='global'}

@ -1261,6 +1261,8 @@ CREATE TABLE cloud_jobs (
request_idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
request_fingerprint CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
parent_job_id BIGINT UNSIGNED DEFAULT NULL,
supersedes_job_id BIGINT UNSIGNED DEFAULT NULL,
vpc_repair_root_job_id BIGINT UNSIGNED DEFAULT NULL,
order_id BIGINT UNSIGNED DEFAULT NULL,
invoice_id BIGINT UNSIGNED DEFAULT NULL,
instance_id BIGINT UNSIGNED DEFAULT NULL,
@ -1397,7 +1399,9 @@ CREATE TABLE cloud_jobs (
KEY cloud_job_vpc_batch (vpc_batch_id, id),
KEY cloud_job_overview_state (kind, order_id, phase, outcome, terminal_at, created_at),
KEY cloud_job_owner (user_id, id),
KEY cloud_job_parent (parent_job_id, id)
KEY cloud_job_parent (parent_job_id, id),
KEY cloud_job_supersedes (supersedes_job_id, id),
KEY cloud_job_vpc_repair_root (vpc_repair_root_job_id, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- Agent terminal receipts are immutable transport events. Retransmission of

@ -158,6 +158,7 @@ $assert($xpath->query('//*[@id="operations-health-body"]//p')->length === 1
&& !str_contains($html, '数据库可读'), 'SSR must not pretend to have checked runtime health.');
$assert($xpath->query('//button[contains(@id,"retry") or contains(@id,"refund") or contains(@id,"stop-auto") or contains(@id,"abort-job") or contains(@id,"cleanup")]')->length === 0
&& $xpath->query('//button[@id="operation-detail-inspect" and @hidden]')->length === 1
&& $xpath->query('//button[@id="operation-detail-repair-vpc" and @hidden]')->length === 1
&& $xpath->query('//button[@id="operation-detail-force-delete" and @hidden]')->length === 1
&& $xpath->query('//*[@id="operation-effect-resolution" and @hidden]')->length === 1,
'The task list must expose no replay/stop mutation and keep audited fence resolution inside detail.');

@ -76,6 +76,8 @@ namespace {
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('supersedes_job_id')->nullable();
$table->integer('vpc_repair_root_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();

@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false
$assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema),
'cloud.sql retained an engine-prefixed identifier.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v5-fences'")
"SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v6-repair'")
&& !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"),
'Outbound Agent schema must reject the incompatible earlier Go database.');
$assert(str_contains($schema,
@ -64,6 +64,10 @@ $assert(str_contains($schema,
'CONSTRAINT chk_cloud_job_finalization_incident CHECK (')
&& str_contains($schema,
'KEY cloud_job_overview_state (kind, order_id, phase, outcome, terminal_at, created_at)')
&& str_contains($schema,
'KEY cloud_job_supersedes (supersedes_job_id, id)')
&& str_contains($schema,
'KEY cloud_job_vpc_repair_root (vpc_repair_root_job_id, id)')
&& str_contains($schema,
'UNIQUE KEY uniq_cloud_job_request (user_id, request_idempotency_key)')
&& str_contains($schema,

@ -25,6 +25,7 @@ $capsule->schema('default')->create('cloud_jobs', static function (
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('phase');
$table->integer('vpc_repair_root_job_id')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->boolean('finalization_incident')->default(0);
});
@ -35,6 +36,7 @@ $insert = static function (int $id, int $hostId, string $kind): void {
'host_id' => $hostId,
'kind' => $kind,
'phase' => 'accepted',
'vpc_repair_root_job_id' => null,
'effect_resolution' => 'not_applicable',
'finalization_incident' => 0,
]);
@ -82,6 +84,20 @@ Capsule::table('cloud_jobs')->where('id', 1)->update([
$assert($eligibleIds() === [2, 3, 4],
'Resolved route predecessor did not release exactly one host successor.');
Capsule::table('cloud_jobs')->insert([
['id' => 6, 'host_id' => 20, 'kind' => 'vpc_host_apply',
'phase' => 'terminal', 'effect_resolution' => 'unknown',
'finalization_incident' => 0, 'vpc_repair_root_job_id' => null],
['id' => 7, 'host_id' => 20, 'kind' => 'vpc_host_apply',
'phase' => 'terminal', 'effect_resolution' => 'unknown',
'finalization_incident' => 0, 'vpc_repair_root_job_id' => 6],
['id' => 8, 'host_id' => 20, 'kind' => 'vpc_host_apply',
'phase' => 'accepted', 'effect_resolution' => 'not_applicable',
'finalization_incident' => 0, 'vpc_repair_root_job_id' => 6],
]);
$assert(in_array(8, $eligibleIds(), true),
'An exact VPC repair successor could not cross its own predecessor chain.');
$worker = file_get_contents(
dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php'
);

@ -197,6 +197,8 @@ namespace {
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('supersedes_job_id')->nullable();
$table->integer('vpc_repair_root_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
@ -1211,6 +1213,135 @@ namespace {
!CloudJobService::blockingQueryForInstance($id)->exists()),
'Retried rollback left an instance operation fence behind.');
// A reachable but inconclusive VPC generation is repaired by a new
// operation. If the first repair also loses its result, a second exact
// repair may supersede the complete chain without rewriting history.
$repairInstance = $newInstance(40, 1);
$repairAttach = CloudVpcPortService::attach($repairInstance, [
'user_id' => 7,
'request_idempotency_key' => 'vpc-repair-fixture-attach-0001',
], [
'vpc_key' => (string) $vpc->vpc_key,
'confirm_restart' => true,
]);
$repairOriginal = CloudJob::where('kind', 'vpc_host_apply')
->where('instance_id', 40)->where('phase', 'accepted')->first();
$assert($repairAttach['job'] !== null && $repairOriginal !== null,
'VPC repair fixture did not create its host generation.');
$repairFinishedAt = time();
$repairOriginal->phase = 'terminal';
$repairOriginal->outcome = $repairOriginal->status = 'failed';
$repairOriginal->attempts = $repairOriginal->dispatch_attempts = 1;
$repairOriginal->dispatch_started_at =
$repairOriginal->dispatched_at = $repairFinishedAt - 2;
$repairOriginal->agent_accepted_at = $repairFinishedAt - 1;
$repairOriginal->receipt_deadline_at = $repairFinishedAt + 900;
$repairOriginal->terminal_at = $repairFinishedAt;
$repairOriginal->effect_may_have_applied = 1;
$repairOriginal->safe_to_resubmit = 0;
$repairOriginal->effect_resolution = 'unknown';
$repairOriginal->error_code = 'operation_effect_unknown';
$repairOriginal->updated_at = $repairFinishedAt;
$repairOriginal->save();
CloudVpcPortService::markJobUncertain($repairOriginal);
$repairInspection = static function (CloudJob $job): array {
return [
'operation_id' => (string) $job->operation_id,
'journal_status' => 'failed',
'checked_at' => time(),
'outcome' => 'inconclusive',
'effect_resolution' => 'inconclusive',
'fence_state' => 'active',
'fence_resolution_required' => true,
];
};
$repairActor = [
'user_id' => 1, 'is_admin' => true,
'is_super_admin' => true, 'can_manage' => true,
];
$repairOne = CloudVpcPortService::repairUnknownEffect(
$repairActor,
(int) $repairOriginal->id,
$repairInspection($repairOriginal)
);
$repairOnePayload = CloudSecrets::decrypt(
(string) $repairOne->sealed_payload, 'job-payload'
);
$repairState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', 1)->first();
$assert((int) $repairOne->supersedes_job_id
=== (int) $repairOriginal->id
&& (int) $repairOne->vpc_repair_root_job_id
=== (int) $repairOriginal->id
&& ($repairOnePayload['body']['vpc_repair'] ?? false) === true
&& ($repairOnePayload['body']['supersedes_operation_id'] ?? null)
=== (string) $repairOriginal->operation_id
&& str_ends_with(
(string) $repairOnePayload['request_target'], '/repair'
)
&& (string) $repairState->last_operation_id
=== (string) $repairOne->operation_id,
'First VPC repair was not bound to the exact predecessor generation.');
CloudVpcPortService::finishJobLocked(
$repairOne,
'uncertain',
null,
$repairOnePayload,
[
'operation_id' => (string) $repairOne->operation_id,
'error_code' => 'vpc_reconcile_unconfirmed',
]
);
$repairOne->phase = 'terminal';
$repairOne->outcome = $repairOne->status = 'failed';
$repairOne->attempts = $repairOne->dispatch_attempts = 1;
$repairOne->dispatch_started_at =
$repairOne->dispatched_at = time() - 2;
$repairOne->agent_accepted_at = time() - 1;
$repairOne->receipt_deadline_at = time() + 900;
$repairOne->terminal_at = time();
$repairOne->effect_may_have_applied = 1;
$repairOne->safe_to_resubmit = 0;
$repairOne->effect_resolution = 'unknown';
$repairOne->error_code = 'operation_effect_unknown';
$repairOne->updated_at = time();
$repairOne->save();
$repairTwo = CloudVpcPortService::repairUnknownEffect(
$repairActor,
(int) $repairOne->id,
$repairInspection($repairOne)
);
$repairTwoPayload = CloudSecrets::decrypt(
(string) $repairTwo->sealed_payload, 'job-payload'
);
$repairState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', 1)->first();
CloudVpcPortService::finishJobLocked($repairTwo, 'succeeded', [
'vpc_key' => (string) $vpc->vpc_key,
'desired_revision' => (int) $repairState->desired_revision,
'desired_digest' => (string) $repairState->desired_digest,
'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key),
'applied' => true,
], $repairTwoPayload, [
'operation_id' => (string) $repairTwo->operation_id,
]);
$terminalize($repairTwo);
$assert((int) $repairTwo->supersedes_job_id === (int) $repairOne->id
&& (int) $repairTwo->vpc_repair_root_job_id
=== (int) $repairOriginal->id
&& (string) $repairOriginal->fresh()->effect_resolution === 'applied'
&& str_starts_with(
(string) $repairOriginal->fresh()->effect_resolution_reason,
'superseded_by_vpc_repair:'
)
&& (string) $repairOne->fresh()->effect_resolution === 'applied'
&& (string) CloudVpcHostState::where('vpc_id', 1)
->where('host_id', 1)->value('sync_state') === 'ready'
&& CloudJob::where('kind', 'vpc_port_attach')
->where('instance_id', 40)->where('phase', 'accepted')->exists(),
'A successful chained VPC repair did not converge every fence and ledger.');
$worker = file_get_contents(
dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php'
);

@ -88,17 +88,26 @@ $jobs = $jobsStart === false || $jobsEnd === false ? ''
$assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL')
&& str_contains($jobs, 'vpc_port_id BIGINT UNSIGNED DEFAULT NULL')
&& str_contains($jobs, 'vpc_batch_id BIGINT UNSIGNED DEFAULT NULL')
&& str_contains($jobs, 'supersedes_job_id BIGINT UNSIGNED DEFAULT NULL')
&& str_contains($jobs,
'vpc_repair_root_job_id BIGINT UNSIGNED DEFAULT NULL')
&& str_contains($jobs, 'KEY cloud_job_vpc (vpc_id, id)')
&& str_contains($jobs, 'KEY cloud_job_vpc_port (vpc_port_id, id)')
&& str_contains($jobs, 'KEY cloud_job_vpc_batch (vpc_batch_id, id)'),
&& str_contains($jobs, 'KEY cloud_job_vpc_batch (vpc_batch_id, id)')
&& str_contains($jobs,
'KEY cloud_job_supersedes (supersedes_job_id, id)')
&& str_contains($jobs,
'KEY cloud_job_vpc_repair_root (vpc_repair_root_job_id, id)'),
'Cloud jobs cannot be correlated to VPC or VPC-port work.');
$assert(str_contains($setup,
"private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v5-fences'")
"private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v6-repair'")
&& str_contains($setup, "'name' => 'cloud_vpc_enabled'")
&& str_contains($setup, "'value' => 'false'")
&& str_contains($setup,
"'cloud_job_vpc', 'cloud_job_vpc_port', 'cloud_job_vpc_batch'")
"'cloud_job_vpc', 'cloud_job_vpc_port', 'cloud_job_vpc_batch',")
&& str_contains($setup,
"'cloud_job_supersedes', 'cloud_job_vpc_repair_root'")
&& str_contains($setup, "'chk_cloud_vpc_port_release'"),
'cloudInit does not publish and validate the disabled VPC schema.');
@ -130,7 +139,9 @@ foreach ($models as $class => $table) {
$jobModel = file_get_contents($root . '/app/Models/CloudJob.php');
$assert(str_contains($jobModel, "'vpc_id' => 'int'")
&& str_contains($jobModel, "'vpc_port_id' => 'int'")
&& str_contains($jobModel, "'vpc_batch_id' => 'int'"),
&& str_contains($jobModel, "'vpc_batch_id' => 'int'")
&& str_contains($jobModel, "'supersedes_job_id' => 'int'")
&& str_contains($jobModel, "'vpc_repair_root_job_id' => 'int'"),
'CloudJob does not cast VPC correlations.');
echo "VPC schema, model and feature-switch contracts passed.\n";

Loading…
Cancel
Save