diff --git a/agent/api/server.go b/agent/api/server.go index e8727686..6400b9ac 100644 --- a/agent/api/server.go +++ b/agent/api/server.go @@ -108,6 +108,12 @@ func trustedSupersededOperation(ctx context.Context, operationID string) (operat return prior, ok && prior.OperationID == operationID } +func isVPCRepairTarget(target string) bool { + return strings.HasSuffix(target, "/repair") && + (strings.Contains(target, "/v1/vpc/networks/") || + strings.Contains(target, "/vpc-port/")) +} + type errorBody struct { Code string `json:"code"` Message string `json:"message"` @@ -259,7 +265,8 @@ func (s *Server) route(w http.ResponseWriter, r *http.Request, body []byte, acto } func (s *Server) routeVPC(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { - if len(segments) != 5 || segments[2] != "networks" || !catalogIDPattern.MatchString(segments[3]) || + repairPath := len(segments) == 6 && segments[5] == "repair" + if (len(segments) != 5 && !repairPath) || segments[2] != "networks" || !catalogIDPattern.MatchString(segments[3]) || (segments[4] != "apply" && segments[4] != "remove") { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") return @@ -294,15 +301,28 @@ func (s *Server) routeVPC(w http.ResponseWriter, r *http.Request, segments []str return } request = normalized + if request.VPCRepair != repairPath || + (request.VPCRepair != (request.SupersedesOperationID != "")) { + writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation") + return + } action := "apply-vpc-network:" + vpcKey expected := &operation.ExpectedState{ Kind: operation.ExpectedVPCNetwork, VPCKey: vpcKey, VPCRevision: request.Network.DesiredRevision, VPCDigest: request.DesiredDigest, VPCPresent: true, } + admission := hypervisor.VPCNetworkAdmission(vpcKey) + supersedes, ok := s.authorizeVPCRepair( + w, r, operationID, request.SupersedesOperationID, + "vpc:"+vpcKey, action, expected, admission, repairPath, + ) + if !ok { + return + } s.runMutationExpected(w, r, body, actor, operationID, "vpc:"+vpcKey, action, expected, true, http.StatusOK, - hypervisor.VPCNetworkAdmission(vpcKey), - func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyVPCNetwork(ctx, request) }) + admission, + func(ctx context.Context) (any, error) { return s.Hypervisor.ApplyVPCNetwork(ctx, request) }, supersedes...) case "remove": request, ok := decodeRequired[vpcrouting.RemoveRequest](w, body) if !ok { @@ -322,18 +342,92 @@ func (s *Server) routeVPC(w http.ResponseWriter, r *http.Request, segments []str writePreDispatchMappedError(w, hypervisor.NewError(hypervisor.ErrInvalid, publicErrorCode(err, "invalid_vpc_remove"), publicErrorMessage(err, "VPC removal request is invalid"), err)) return } + if request.VPCRepair != repairPath || + (request.VPCRepair != (request.SupersedesOperationID != "")) { + writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation") + return + } action := "remove-vpc-network:" + vpcKey expected := &operation.ExpectedState{ Kind: operation.ExpectedVPCNetwork, VPCKey: vpcKey, VPCRevision: request.ExpectedRevision, VPCDigest: request.ExpectedDigest, VPCPresent: false, } + admission := hypervisor.VPCNetworkAdmission(vpcKey) + supersedes, ok := s.authorizeVPCRepair( + w, r, operationID, request.SupersedesOperationID, + "vpc:"+vpcKey, action, expected, admission, repairPath, + ) + if !ok { + return + } s.runMutationExpected(w, r, body, actor, operationID, "vpc:"+vpcKey, action, expected, true, http.StatusOK, - hypervisor.VPCNetworkAdmission(vpcKey), - func(ctx context.Context) (any, error) { return s.Hypervisor.RemoveVPCNetwork(ctx, request) }) + admission, + func(ctx context.Context) (any, error) { return s.Hypervisor.RemoveVPCNetwork(ctx, request) }, supersedes...) } } +func (s *Server) authorizeVPCRepair(w http.ResponseWriter, r *http.Request, + operationID, predecessorID, resourceID, action string, + expected *operation.ExpectedState, admission hypervisor.MutationAdmission, + repair bool) ([]string, bool) { + if !repair { + return nil, true + } + if !operationIDPattern.MatchString(predecessorID) || + operationID == predecessorID { + writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair_predecessor", "VPC repair requires a different valid predecessor operation") + return nil, false + } + trustedPrior, trusted := trustedSupersededOperation( + r.Context(), predecessorID, + ) + existing, existingErr := s.Journal.Get(operationID) + if existingErr == nil { + if existing.Action != action || existing.ResourceID != resourceID || + existing.SupersedesOperationID != predecessorID { + writePreDispatchError(w, http.StatusConflict, "operation_id_conflict", "operation_id is already bound to a different request") + return nil, false + } + } else if !errors.Is(existingErr, operation.ErrNotFound) { + writeMappedError(w, existingErr, nil) + return nil, false + } else { + prior := trustedPrior + var err error + if !trusted { + prior, err = s.Journal.Get(predecessorID) + } + spec := operation.Spec{ + OperationID: operationID, ResourceID: resourceID, + Action: action, ExpectedState: expected, + LockKeys: append([]string(nil), admission.LockKeys...), + SupersedesOperationID: predecessorID, + } + if err != nil || !operation.ValidVPCRepairSupersession(spec, prior) { + writePreDispatchError(w, http.StatusConflict, "vpc_repair_not_authorized", "VPC repair no longer matches one exact fenced predecessor") + return nil, false + } + if !trusted { + _, incident, incidentErr := s.Journal.CompletionDeliveryIncident( + predecessorID, + ) + if incidentErr != nil { + writeMappedError(w, incidentErr, nil) + return nil, false + } + if incident { + writePreDispatchError(w, http.StatusConflict, "vpc_repair_not_authorized", "VPC repair is blocked by a completion-delivery incident") + return nil, false + } + } + } + if trusted { + return nil, true + } + return []string{predecessorID}, true +} + func (s *Server) routeStorage(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor) { if len(segments) != 3 || segments[2] != "configuration" { writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") @@ -1010,7 +1104,9 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str return } case "vpc-port": - if len(segments) == 4 && (r.Method == http.MethodPost || r.Method == http.MethodDelete) { + repairPath := len(segments) == 5 && segments[4] == "repair" + if (len(segments) == 4 || repairPath) && + (r.Method == http.MethodPost || r.Method == http.MethodDelete) { if !requireManage(w, actor) { return } @@ -1029,6 +1125,11 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str writePreDispatchMappedError(w, err) return } + if request.VPCRepair != repairPath || + (request.VPCRepair != (request.SupersedesOperationID != "")) { + writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_vpc_repair", "VPC repair requires the repair endpoint and one predecessor operation") + return + } bridge, _ := vpcrouting.BridgeName(request.VPCKey) present := r.Method == http.MethodPost action := "apply-vpc-port" @@ -1043,15 +1144,23 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str VPCBridge: bridge, VPCRevision: request.HostRevision, VPCDigest: request.HostDigest, } + admission := hypervisor.VPCPortAdmission(uuid, request) + supersedes, ok := s.authorizeVPCRepair( + w, r, operationID, request.SupersedesOperationID, + uuid, action, expected, admission, repairPath, + ) + if !ok { + return + } s.runMutationExpected(w, r, body, actor, operationID, uuid, action, expected, true, http.StatusOK, - hypervisor.VPCPortAdmission(uuid, request), + admission, func(ctx context.Context) (any, error) { if present { return s.Hypervisor.ApplyVPCPort(ctx, uuid, request) } return s.Hypervisor.RemoveVPCPort(ctx, uuid, request) - }) + }, supersedes...) return } case "service": @@ -2134,6 +2243,9 @@ func (s *Server) runIsolatedMutation( return } request.SupersededOperation = &prior + if isVPCRepairTarget(request.RequestTarget) { + request.SupersededExpectedState = prior.ExpectedState + } } dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout) record, replayed, done, err := s.Journal.ExecuteBackground( diff --git a/agent/api/server_test.go b/agent/api/server_test.go index e73575ab..36598f10 100644 --- a/agent/api/server_test.go +++ b/agent/api/server_test.go @@ -41,6 +41,21 @@ type testHypervisor struct { inspectionErr error } +type apiFailure struct { + code string + effect bool + safe bool + fence bool +} + +func (e apiFailure) Error() string { return e.code } +func (e apiFailure) PublicCode() string { return e.code } +func (e apiFailure) PublicMessage() string { return e.code } +func (e apiFailure) FailureSemantics() (bool, bool) { + return e.effect, e.safe +} +func (e apiFailure) RequiresExecutionFence() bool { return e.fence } + func (h *testHypervisor) Ping(context.Context) error { return nil } func (h *testHypervisor) ListVMs(context.Context) ([]hypervisor.VMMetadata, error) { @@ -283,6 +298,93 @@ func TestVPCNetworkMutationUsesCanonicalDigestAndDurableExpectedState(t *testing } } +func TestVPCNetworkRepairCarriesExactSupersededGeneration(t *testing.T) { + api := newTestAPI(t) + const ( + vpcKey = "0123456789abcdef0123456789abcdef" + priorID = "operation-vpc-repair-prior" + repairID = "operation-vpc-repair-next" + ) + network := vpcrouting.Network{ + VPCKey: vpcKey, DesiredRevision: 9, + CIDR: "10.89.0.0/24", Gateway: "10.89.0.1", + Transport: vpcrouting.Transport{ + AddressCIDR: "172.16.50.11/24", Interface: "br-private", MTU: 1500, + }, + LocalPorts: []vpcrouting.LocalPort{{IP: "10.89.0.2", MAC: "02:00:00:00:00:12"}}, + } + digest, err := vpcrouting.Digest(network) + if err != nil { + t.Fatal(err) + } + prior := operation.Spec{ + OperationID: priorID, ResourceID: "vpc:" + vpcKey, + Action: "apply-vpc-network:" + vpcKey, + Fingerprint: operation.Fingerprint( + http.MethodPost, "/v1/vpc/networks/"+vpcKey+"/apply", []byte(`{}`), + ), + Destructive: true, + ExpectedState: &operation.ExpectedState{ + Kind: operation.ExpectedVPCNetwork, TransitionID: priorID, + VPCKey: vpcKey, VPCRevision: 9, + VPCDigest: digest, VPCPresent: true, + }, + LockKeys: hypervisor.VPCNetworkAdmission(vpcKey).LockKeys, + } + if _, _, err := api.journal.Execute( + context.Background(), prior, + func(context.Context) (json.RawMessage, error) { + return nil, apiFailure{ + code: "vpc_reconcile_unconfirmed", effect: true, + safe: false, fence: true, + } + }, + ); err == nil { + t.Fatal("VPC repair predecessor unexpectedly succeeded") + } + request := vpcrouting.ApplyRequest{ + OperationID: repairID, DesiredDigest: digest, Network: network, + VPCRepair: true, SupersedesOperationID: priorID, + } + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + bridge, _ := vpcrouting.BridgeName(vpcKey) + result, _ := json.Marshal(vpcrouting.Result{ + VPCKey: vpcKey, DesiredRevision: 9, DesiredDigest: digest, + Bridge: bridge, Applied: true, + }) + executor := &fakeMutationExecutor{result: result} + if err := api.handler.SetMutationExecutor(executor); err != nil { + t.Fatal(err) + } + response := api.request(t, http.MethodPost, + "/v1/vpc/networks/"+vpcKey+"/apply/repair", body, + auth.Actor{ + UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, + OperationID: repairID, + }, + ) + if response.Code != http.StatusAccepted && response.Code != http.StatusOK { + t.Fatalf("VPC repair status=%d body=%s", response.Code, response.Body.String()) + } + record := waitForTerminal(t, api.journal, repairID) + if record.State != operation.StateSucceeded || + record.SupersedesOperationID != priorID { + t.Fatalf("VPC repair record = %#v", record) + } + executor.mu.Lock() + defer executor.mu.Unlock() + if executor.calls != 1 || len(executor.requests) != 1 || + executor.requests[0].SupersededOperation == nil || + executor.requests[0].SupersededOperation.OperationID != priorID || + executor.requests[0].SupersededExpectedState == nil || + executor.requests[0].SupersededExpectedState.VPCDigest != digest { + t.Fatalf("VPC repair worker envelope = %#v", executor.requests) + } +} + func TestVPCPortMutationBindsGuestAndHostProjectionIdentity(t *testing.T) { api := newTestAPI(t) const ( diff --git a/agent/cmd/virt-agent/main.go b/agent/cmd/virt-agent/main.go index 8ed0ba80..386c2190 100644 --- a/agent/cmd/virt-agent/main.go +++ b/agent/cmd/virt-agent/main.go @@ -460,9 +460,11 @@ func runOperationWorker(configPath, journalPath, receiptPath string) error { }).WithContext(ctx) responseCapture := newBoundedWorkerResponse(worker.MaxWorkerResponse) if request.SupersededOperation != nil { + prior := *request.SupersededOperation + prior.ExpectedState = request.SupersededExpectedState handler.ServeTrustedWorkerRequest( responseCapture, httpRequest, []byte(request.Body), request.Actor, - *request.SupersededOperation, + prior, ) } else { handler.ServeTrustedWorkerRequest( diff --git a/agent/hypervisor/feature_helpers.go b/agent/hypervisor/feature_helpers.go index a7ff4e8d..23f3dcf3 100644 --- a/agent/hypervisor/feature_helpers.go +++ b/agent/hypervisor/feature_helpers.go @@ -525,6 +525,56 @@ func mutateSeedVPCPort(contents []byte, port VPCPortMetadata, return buildSeedISO(files) } +// repairSeedVPCPort rewrites only Virt Cloud's reserved private0 entry. A +// different valid VPC identity proves that panel intent has moved on and is +// rejected; a missing or malformed reserved entry is the partial-state case +// this explicit repair operation exists to converge. +func repairSeedVPCPort(contents []byte, port VPCPortMetadata, + transitionID string, present bool) ([]byte, error) { + if !operationIdentifierPattern.MatchString(transitionID) { + return nil, NewError(ErrInvalid, "invalid_operation_id", "VPC port transition id is invalid", nil) + } + files, err := readSeedISO(contents) + if err != nil { + return nil, err + } + var network map[string]any + if err := json.Unmarshal(bytes.TrimSpace(files.NetworkConfig), &network); err != nil { + return nil, NewError(ErrConflict, "invalid_seed", "NoCloud network-config is invalid", err) + } + ethernets, err := managedSeedEthernets(network) + if err != nil { + return nil, err + } + if existing, exists := ethernets[seedVPCInterfaceKey]; exists { + if parsed, ok := parseSeedVPCInterface(existing); ok && parsed != (seedVPCNetwork{ + MAC: port.MAC, PrivateIP: port.PrivateIP, + CIDR: port.CIDR, Gateway: port.Gateway, + }) { + return nil, NewError(ErrConflict, "vpc_repair_intent_changed", "NoCloud contains a different VPC identity", nil) + } + } + if present { + ethernets[seedVPCInterfaceKey] = seedVPCInterface(port) + } else { + delete(ethernets, seedVPCInterfaceKey) + } + files.NetworkConfig, err = marshalJSONLine(network) + if err != nil { + return nil, err + } + var metadata map[string]any + if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil { + return nil, NewError(ErrConflict, "invalid_seed", "NoCloud meta-data is invalid", err) + } + metadata[vpcPortOperationMarker] = transitionID + files.MetaData, err = marshalJSONLine(metadata) + if err != nil { + return nil, err + } + return buildSeedISO(files) +} + func seedVPCPort(contents []byte) (*seedVPCNetwork, error) { files, err := readSeedISO(contents) if err != nil { diff --git a/agent/hypervisor/feature_helpers_test.go b/agent/hypervisor/feature_helpers_test.go index e2ccf10b..e879358a 100644 --- a/agent/hypervisor/feature_helpers_test.go +++ b/agent/hypervisor/feature_helpers_test.go @@ -495,6 +495,52 @@ func TestMutateSeedVPCPortPreservesPublicNetworkAndBindsMarker(t *testing.T) { } } +func TestRepairSeedVPCPortConvergesOnlyTheSameManagedIdentity(t *testing.T) { + port := validVPCPortMetadata() + original := managedSeedForTest(t) + attached, err := repairSeedVPCPort( + original, port, "operation-vpc-repair-attach", true, + ) + if err != nil { + t.Fatal(err) + } + repaired, err := repairSeedVPCPort( + attached, port, "operation-vpc-repair-rebind", true, + ) + if err != nil { + t.Fatal(err) + } + private, err := seedVPCPort(repaired) + want := seedVPCNetwork{ + MAC: port.MAC, PrivateIP: port.PrivateIP, + CIDR: port.CIDR, Gateway: port.Gateway, + } + if err != nil || private == nil || *private != want { + t.Fatalf("repaired private network = %#v err=%v", private, err) + } + changed := port + changed.PrivateIP = "10.60.0.10" + if _, err := repairSeedVPCPort( + repaired, changed, "operation-vpc-repair-changed", true, + ); err == nil { + t.Fatal("repair adopted a different VPC identity") + } + detached, err := repairSeedVPCPort( + repaired, port, "operation-vpc-repair-detach", false, + ) + if err != nil { + t.Fatal(err) + } + if detached, err = repairSeedVPCPort( + detached, port, "operation-vpc-repair-detach-again", false, + ); err != nil { + t.Fatalf("repair could not converge an already absent seed port: %v", err) + } + if private, err := seedVPCPort(detached); err != nil || private != nil { + t.Fatalf("repair detach left private network = %#v err=%v", private, err) + } +} + func TestPreservedVPCCloudInitNetworkRejectsCrossLayerMismatch(t *testing.T) { port := validVPCPortMetadata() attached, err := mutateSeedVPCPort( diff --git a/agent/hypervisor/libvirt_vpc_port.go b/agent/hypervisor/libvirt_vpc_port.go index 161a336a..e1c9bcd4 100644 --- a/agent/hypervisor/libvirt_vpc_port.go +++ b/agent/hypervisor/libvirt_vpc_port.go @@ -94,20 +94,35 @@ func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string, if err != nil { return VPCPortResult{}, err } - if !managedNetworkLayoutMatches(preflightDefinition, preflightMetadata) { - return VPCPortResult{}, NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its durable metadata", nil) - } - if present { - if preflightMetadata.VPCPort != nil { - return VPCPortResult{}, NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil) + if request.VPCRepair { + if err := validateRepairableVPCPortDomain( + preflightDefinition, preflightMetadata, port, + ); err != nil { + return VPCPortResult{}, err + } + } else { + if !managedNetworkLayoutMatches(preflightDefinition, preflightMetadata) { + return VPCPortResult{}, NewError(ErrConflict, "network_layout_changed", "managed VM network layout no longer matches its durable metadata", nil) + } + if present { + if preflightMetadata.VPCPort != nil { + return VPCPortResult{}, NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil) + } + } else if preflightMetadata.VPCPort == nil || + !sameVPCPortIdentity(preflightMetadata.VPCPort.public(), port) { + return VPCPortResult{}, NewError(ErrConflict, "vpc_port_identity_changed", "VM VPC port no longer matches the requested detach identity", nil) } - } else if preflightMetadata.VPCPort == nil || - !sameVPCPortIdentity(preflightMetadata.VPCPort.public(), port) { - return VPCPortResult{}, NewError(ErrConflict, "vpc_port_identity_changed", "VM VPC port no longer matches the requested detach identity", nil) } - replacementSeed, err := mutateSeedVPCPort( - originalSeed, port, expected.TransitionID, present, - ) + var replacementSeed []byte + if request.VPCRepair { + replacementSeed, err = repairSeedVPCPort( + originalSeed, port, expected.TransitionID, present, + ) + } else { + replacementSeed, err = mutateSeedVPCPort( + originalSeed, port, expected.TransitionID, present, + ) + } if err != nil { return VPCPortResult{}, err } @@ -124,6 +139,15 @@ func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string, } updatedXML, err := mutateDomainDocument(originalXML, func(definition *libvirtxml.Domain, metadata *domainMetadataXML) error { + if request.VPCRepair { + if err := repairVPCPortDomain( + definition, metadata, port, present, + ); err != nil { + return err + } + metadata.VPCTransitionID = expected.TransitionID + return nil + } if !managedNetworkLayoutMatches(definition, metadata) { return NewError(ErrConflict, "network_layout_changed", "managed VM network layout changed while applying the VPC port", nil) } @@ -131,28 +155,10 @@ func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string, if metadata.VPCPort != nil { return NewError(ErrConflict, "vpc_port_already_attached", "VM already has a VPC port", nil) } - linkDown := definitionNetworkSuspended(definition) - queues := uint(1) - if definition.VCPU != nil { - queues = min(definition.VCPU.Value, 16) - } - iface := libvirtxml.DomainInterface{ - MAC: &libvirtxml.DomainInterfaceMAC{Address: port.MAC}, - Model: &libvirtxml.DomainInterfaceModel{Type: "virtio"}, - Driver: &libvirtxml.DomainInterfaceDriver{Queues: queues}, - Source: &libvirtxml.DomainInterfaceSource{Bridge: &libvirtxml.DomainInterfaceSourceBridge{Bridge: port.Bridge}}, - } - if metadata.BandwidthMbps > 0 { - rate := bandwidthRateKiBPerSecond(metadata.BandwidthMbps) - iface.Bandwidth = &libvirtxml.DomainInterfaceBandwidth{ - Inbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate}, - Outbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate}, - } - } - if linkDown { - iface.Link = &libvirtxml.DomainInterfaceLink{State: "down"} - } - definition.Devices.Interfaces = append(definition.Devices.Interfaces, iface) + definition.Devices.Interfaces = append( + definition.Devices.Interfaces, + canonicalVPCInterface(definition, metadata, port), + ) stored := domainVPCPortFromPublic(port) metadata.VPCPort = &stored } else { @@ -194,6 +200,93 @@ func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string, return result, nil } +// validateRepairableVPCPortDomain proves that every non-public interface is +// either the exact panel-owned VPC NIC or is already bound to that identity by +// durable metadata. It never adopts or deletes an unrelated guest interface. +func validateRepairableVPCPortDomain(definition *libvirtxml.Domain, + metadata *domainMetadataXML, port VPCPortMetadata) error { + if definition == nil || definition.Devices == nil || metadata == nil { + return NewError(ErrConflict, "network_layout_changed", "managed VM network layout is incomplete", nil) + } + publicIndex, found := managedPublicInterfaceIndex( + definition, networkFilterName(definition.UUID), metadata.PublicIPCIDR, + ) + if !found || len(definition.Devices.Interfaces) > 2 { + return NewError(ErrConflict, "network_layout_changed", "managed VM has an unrepairable network layout", nil) + } + metadataOwnsPort := metadata.VPCPort != nil + if metadataOwnsPort && + !sameVPCPortIdentity(metadata.VPCPort.public(), port) { + return NewError(ErrConflict, "vpc_repair_intent_changed", "domain metadata contains a different VPC identity", nil) + } + if len(definition.Devices.Interfaces) == 2 { + privateIndex := 1 - publicIndex + iface := definition.Devices.Interfaces[privateIndex] + exactInterface := iface.MAC != nil && + strings.EqualFold(iface.MAC.Address, port.MAC) && + iface.Source != nil && iface.Source.Bridge != nil && + iface.Source.Bridge.Bridge == port.Bridge && + iface.Source.Network == nil && iface.FilterRef == nil + if !metadataOwnsPort && !exactInterface { + return NewError(ErrConflict, "vpc_repair_unowned_interface", "unrelated private interface cannot be changed by VPC repair", nil) + } + } + return nil +} + +func repairVPCPortDomain(definition *libvirtxml.Domain, + metadata *domainMetadataXML, port VPCPortMetadata, present bool) error { + if err := validateRepairableVPCPortDomain(definition, metadata, port); err != nil { + return err + } + publicIndex, _ := managedPublicInterfaceIndex( + definition, networkFilterName(definition.UUID), metadata.PublicIPCIDR, + ) + if len(definition.Devices.Interfaces) == 2 { + privateIndex := 1 - publicIndex + definition.Devices.Interfaces = append( + definition.Devices.Interfaces[:privateIndex], + definition.Devices.Interfaces[privateIndex+1:]..., + ) + } + if present { + definition.Devices.Interfaces = append( + definition.Devices.Interfaces, + canonicalVPCInterface(definition, metadata, port), + ) + stored := domainVPCPortFromPublic(port) + metadata.VPCPort = &stored + } else { + metadata.VPCPort = nil + } + return nil +} + +func canonicalVPCInterface(definition *libvirtxml.Domain, + metadata *domainMetadataXML, port VPCPortMetadata) libvirtxml.DomainInterface { + queues := uint(1) + if definition.VCPU != nil { + queues = min(definition.VCPU.Value, 16) + } + iface := libvirtxml.DomainInterface{ + MAC: &libvirtxml.DomainInterfaceMAC{Address: port.MAC}, + Model: &libvirtxml.DomainInterfaceModel{Type: "virtio"}, + Driver: &libvirtxml.DomainInterfaceDriver{Queues: queues}, + Source: &libvirtxml.DomainInterfaceSource{Bridge: &libvirtxml.DomainInterfaceSourceBridge{Bridge: port.Bridge}}, + } + if metadata.BandwidthMbps > 0 { + rate := bandwidthRateKiBPerSecond(metadata.BandwidthMbps) + iface.Bandwidth = &libvirtxml.DomainInterfaceBandwidth{ + Inbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate}, + Outbound: &libvirtxml.DomainInterfaceBandwidthParams{Average: &rate, Peak: &rate}, + } + } + if definitionNetworkSuspended(definition) { + iface.Link = &libvirtxml.DomainInterfaceLink{State: "down"} + } + return iface +} + func (l *Libvirt) vpcPortIdentityConflict(ctx context.Context, requestedUUID string, port VPCPortMetadata) (bool, error) { domains, err := l.conn.ListAllDomains(0) diff --git a/agent/hypervisor/types.go b/agent/hypervisor/types.go index 5862bdf9..baacf211 100644 --- a/agent/hypervisor/types.go +++ b/agent/hypervisor/types.go @@ -224,16 +224,18 @@ type VPCPortMetadata struct { } type VPCPortRequest struct { - OperationID string `json:"operation_id,omitempty"` - PortKey string `json:"port_key"` - VPCKey string `json:"vpc_key"` - PrivateIP string `json:"private_ip"` - CIDR string `json:"cidr"` - Gateway string `json:"gateway"` - MAC string `json:"mac"` - HostRevision uint64 `json:"host_revision"` - HostDigest string `json:"host_digest"` - ConfirmRestart bool `json:"confirm_restart"` + OperationID string `json:"operation_id,omitempty"` + PortKey string `json:"port_key"` + VPCKey string `json:"vpc_key"` + PrivateIP string `json:"private_ip"` + CIDR string `json:"cidr"` + Gateway string `json:"gateway"` + MAC string `json:"mac"` + HostRevision uint64 `json:"host_revision"` + HostDigest string `json:"host_digest"` + ConfirmRestart bool `json:"confirm_restart"` + VPCRepair bool `json:"vpc_repair,omitempty"` + SupersedesOperationID string `json:"supersedes_operation_id,omitempty"` } type VPCPortResult struct { diff --git a/agent/hypervisor/vpc_port.go b/agent/hypervisor/vpc_port.go index c30e99aa..1743852a 100644 --- a/agent/hypervisor/vpc_port.go +++ b/agent/hypervisor/vpc_port.go @@ -19,6 +19,11 @@ func NormalizeVPCPortRequest(request VPCPortRequest) (VPCPortRequest, error) { !digestPattern.MatchString(request.HostDigest) || !request.ConfirmRestart { return VPCPortRequest{}, NewError(ErrInvalid, "invalid_vpc_port_request", "VPC port identity, host projection, and restart confirmation are required", nil) } + if request.VPCRepair != (request.SupersedesOperationID != "") || + (request.VPCRepair && (!operationIdentifierPattern.MatchString(request.SupersedesOperationID) || + request.SupersedesOperationID == request.OperationID)) { + return VPCPortRequest{}, NewError(ErrInvalid, "invalid_vpc_repair", "VPC repair requires one different valid predecessor operation", nil) + } prefix, err := netip.ParsePrefix(request.CIDR) if err != nil || !prefix.Addr().Is4() || !prefix.Addr().IsPrivate() || prefix != prefix.Masked() || prefix.Bits() < 16 || prefix.Bits() > 28 { diff --git a/agent/hypervisor/vpc_port_test.go b/agent/hypervisor/vpc_port_test.go index 96e1c336..bf707e8f 100644 --- a/agent/hypervisor/vpc_port_test.go +++ b/agent/hypervisor/vpc_port_test.go @@ -45,6 +45,16 @@ func TestNormalizeVPCPortRequestRequiresCanonicalRoutedIdentity(t *testing.T) { "multicast MAC": func(value *VPCPortRequest) { value.MAC = "03:00:00:00:00:09" }, "upper-case MAC": func(value *VPCPortRequest) { value.MAC = "02:00:00:00:00:AA" }, "invalid digest": func(value *VPCPortRequest) { value.HostDigest = "bad" }, + "repair without predecessor": func(value *VPCPortRequest) { + value.VPCRepair = true + }, + "predecessor without repair": func(value *VPCPortRequest) { + value.SupersedesOperationID = "operation-vpc-port-prior" + }, + "repair supersedes itself": func(value *VPCPortRequest) { + value.VPCRepair = true + value.SupersedesOperationID = value.OperationID + }, } { t.Run(name, func(t *testing.T) { candidate := request @@ -54,6 +64,12 @@ func TestNormalizeVPCPortRequestRequiresCanonicalRoutedIdentity(t *testing.T) { } }) } + repair := request + repair.VPCRepair = true + repair.SupersedesOperationID = "operation-vpc-port-prior" + if _, err := NormalizeVPCPortRequest(repair); err != nil { + t.Fatalf("valid repair request was rejected: %v", err) + } } func TestValidateVPCPortMetadataRequiresDeterministicBridge(t *testing.T) { diff --git a/agent/operation/journal.go b/agent/operation/journal.go index 969e7868..0679132d 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -177,8 +177,7 @@ type Record struct { LockKeys []string `json:"-"` HeavyIO bool `json:"-"` // SupersedesOperationID is valid only for an explicitly reviewed successor: - // force-delete-vm after force_required, or a plan-bound PeerMigration target - // cleanup after one exact failed receive operation. + // force-delete-vm, plan-bound cleanup, or an exact VPC repair generation. SupersedesOperationID string `json:"-"` // InspectionReceiptRecovered and InspectionEvidenceUnavailable are // ephemeral read-path markers. They are never persisted or exposed by the @@ -2440,8 +2439,9 @@ func (j *Journal) updateRecordWhen(operationID string, eligible func(Record) boo if err := syncResourceBlockers(blockers, &previous, &record); err != nil { return err } - if record.State == StateSucceeded && record.Action == "force-delete-vm" && record.SupersedesOperationID != "" { - if err := closeForceDeletePredecessor(bucket, blockers, record); err != nil { + if record.State == StateSucceeded && record.SupersedesOperationID != "" && + (record.Action == "force-delete-vm" || isVPCAction(record.Action)) { + if err := closeSuccessfulSupersessionPredecessor(bucket, blockers, record); err != nil { return err } } @@ -2466,11 +2466,10 @@ func (j *Journal) updateRecordWhen(operationID string, eligible func(Record) boo return cloneRecord(result), err } -// closeForceDeletePredecessor runs in the same Bolt transaction which marks -// the explicit force operation succeeded. It removes only the predecessor's -// blocker index entries; the failed operation and its completion receipt stay -// byte-for-byte immutable, with the successor retaining the audit link. -func closeForceDeletePredecessor(operations, blockers *bolt.Bucket, successor Record) error { +// closeSuccessfulSupersessionPredecessor runs in the same Bolt transaction +// which marks the reviewed successor successful. It removes only the prior +// blocker index entries; terminal evidence remains byte-for-byte immutable. +func closeSuccessfulSupersessionPredecessor(operations, blockers *bolt.Bucket, successor Record) error { encoded := operations.Get([]byte(successor.SupersedesOperationID)) if encoded == nil { return ErrInvalidSupersession @@ -2480,15 +2479,31 @@ func closeForceDeletePredecessor(operations, blockers *bolt.Bucket, successor Re return err } lockKeys := effectiveLockKeys(successor.OperationID, successor.ResourceID, successor.Action, successor.LockKeys) - if !validForceDeleteSupersession(Spec{ + spec := Spec{ OperationID: successor.OperationID, ResourceID: successor.ResourceID, - Action: successor.Action, SupersedesOperationID: successor.SupersedesOperationID, - }, lockKeys, predecessor) { + Action: successor.Action, ExpectedState: successor.ExpectedState, + SupersedesOperationID: successor.SupersedesOperationID, + } + if validForceDeleteSupersession(spec, lockKeys, predecessor) { + for _, lockKey := range effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys) { + if err := blockers.Delete(blockerEntryKey(lockKey, predecessor.OperationID)); err != nil { + return err + } + } + return nil + } + if !isVPCAction(successor.Action) { return ErrInvalidSupersession } - for _, lockKey := range effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys) { - if err := blockers.Delete(blockerEntryKey(lockKey, predecessor.OperationID)); err != nil { - return err + chain, err := vpcSupersessionChain(operations, spec, lockKeys) + if err != nil { + return err + } + for _, prior := range chain { + for _, lockKey := range effectiveLockKeys(prior.OperationID, prior.ResourceID, prior.Action, prior.LockKeys) { + if err := blockers.Delete(blockerEntryKey(lockKey, prior.OperationID)); err != nil { + return err + } } } return nil @@ -2715,10 +2730,120 @@ func validBackupWorkspaceCleanupSupersession(spec Spec, lockKeys []string, prede return lockKeysOverlap(lockKeys, predecessorKeys) } +func isVPCAction(action string) bool { + return strings.HasPrefix(action, "apply-vpc-network:") || + strings.HasPrefix(action, "remove-vpc-network:") || + action == "apply-vpc-port" || action == "remove-vpc-port" +} + +func sameVPCRepairIntent(current, predecessor *ExpectedState) bool { + if current == nil || predecessor == nil || current.Kind != predecessor.Kind || + current.VPCKey != predecessor.VPCKey || + current.VPCRevision != predecessor.VPCRevision || + current.VPCDigest != predecessor.VPCDigest || + current.VPCPresent != predecessor.VPCPresent { + return false + } + if current.Kind == ExpectedVPCNetwork { + return true + } + return current.Kind == ExpectedVPCPort && + current.VPCPortKey == predecessor.VPCPortKey && + current.VPCCIDR == predecessor.VPCCIDR && + current.VPCGateway == predecessor.VPCGateway && + current.VPCPrivateIP == predecessor.VPCPrivateIP && + current.VPCMAC == predecessor.VPCMAC && + current.VPCBridge == predecessor.VPCBridge +} + +func validVPCRepairSupersession(spec Spec, lockKeys []string, + predecessor Record) bool { + if spec.SupersedesOperationID == "" || + spec.SupersedesOperationID == spec.OperationID || + predecessor.OperationID != spec.SupersedesOperationID || + predecessor.ResourceID != spec.ResourceID || + predecessor.Action != spec.Action || !isVPCAction(spec.Action) || + predecessor.State != StateFailed || !predecessor.EffectMayHaveApplied || + predecessor.SafeToResubmit || !predecessor.ExecutionFenced || + predecessor.Tombstone || + !sameVPCRepairIntent(spec.ExpectedState, predecessor.ExpectedState) { + return false + } + predecessorKeys := effectiveLockKeys( + predecessor.OperationID, predecessor.ResourceID, + predecessor.Action, predecessor.LockKeys, + ) + return lockKeysOverlap(lockKeys, predecessorKeys) +} + +// ValidVPCRepairSupersession exposes the same fail-closed predecessor check to +// the authenticated API before a repair is dispatched. Journal admission +// repeats it under the Bolt transaction, so this preflight is not authority. +func ValidVPCRepairSupersession(spec Spec, predecessor Record) bool { + lockKeys, err := normalizeLockKeys(spec.LockKeys) + return err == nil && validVPCRepairSupersession( + spec, lockKeys, predecessor, + ) +} + func validExplicitSupersession(spec Spec, lockKeys []string, predecessor Record) bool { return validForceDeleteSupersession(spec, lockKeys, predecessor) || validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor) || - validBackupWorkspaceCleanupSupersession(spec, lockKeys, predecessor) + validBackupWorkspaceCleanupSupersession(spec, lockKeys, predecessor) || + validVPCRepairSupersession(spec, lockKeys, predecessor) +} + +const maxVPCSupersessionDepth = 32 + +// vpcSupersessionChain validates every durable predecessor generation. A +// repair that itself lost its response may be followed by another exact +// repair, but it may never skip a changed intent or grow without a bound. +func vpcSupersessionChain(operations *bolt.Bucket, spec Spec, + lockKeys []string) ([]Record, error) { + if operations == nil || !isVPCAction(spec.Action) || + spec.SupersedesOperationID == "" { + return nil, ErrInvalidSupersession + } + currentSpec := spec + currentKeys := lockKeys + seen := map[string]struct{}{spec.OperationID: {}} + chain := make([]Record, 0, 2) + for len(chain) < maxVPCSupersessionDepth { + predecessorID := currentSpec.SupersedesOperationID + if predecessorID == "" { + return chain, nil + } + if _, exists := seen[predecessorID]; exists { + return nil, ErrInvalidSupersession + } + seen[predecessorID] = struct{}{} + encoded := operations.Get([]byte(predecessorID)) + if encoded == nil { + return nil, ErrInvalidSupersession + } + predecessor, err := decodeRecord(encoded) + if err != nil || !validVPCRepairSupersession( + currentSpec, currentKeys, predecessor, + ) { + return nil, ErrInvalidSupersession + } + chain = append(chain, predecessor) + if predecessor.SupersedesOperationID == "" { + return chain, nil + } + currentSpec = Spec{ + OperationID: predecessor.OperationID, + ResourceID: predecessor.ResourceID, + Action: predecessor.Action, + ExpectedState: predecessor.ExpectedState, + SupersedesOperationID: predecessor.SupersedesOperationID, + } + currentKeys = effectiveLockKeys( + predecessor.OperationID, predecessor.ResourceID, + predecessor.Action, predecessor.LockKeys, + ) + } + return nil, ErrInvalidSupersession } // findResourceBlocker validates any explicit force-delete predecessor and @@ -2729,7 +2854,7 @@ func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, boo if operations == nil || blockers == nil { return Record{}, false, errors.New("operation resource blocker index is unavailable") } - supersededOperationID := "" + supersededOperationIDs := map[string]struct{}{} if spec.SupersedesOperationID != "" { encoded := operations.Get([]byte(spec.SupersedesOperationID)) if encoded == nil { @@ -2739,14 +2864,27 @@ func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, boo if err != nil || !validExplicitSupersession(spec, lockKeys, predecessor) { return Record{}, false, ErrInvalidSupersession } - supersededOperationID = predecessor.OperationID + if isVPCAction(spec.Action) { + chain, chainErr := vpcSupersessionChain( + operations, spec, lockKeys, + ) + if chainErr != nil { + return Record{}, false, chainErr + } + for _, prior := range chain { + supersededOperationIDs[prior.OperationID] = struct{}{} + } + } else { + supersededOperationIDs[predecessor.OperationID] = struct{}{} + } } for _, lockKey := range lockKeys { prefix := blockerEntryPrefix(lockKey) cursor := blockers.Cursor() for key, _ := cursor.Seek(prefix); key != nil && bytes.HasPrefix(key, prefix); key, _ = cursor.Next() { operationID := string(key[len(prefix):]) - if operationID == spec.OperationID || operationID == supersededOperationID { + _, explicitlySuperseded := supersededOperationIDs[operationID] + if operationID == spec.OperationID || explicitlySuperseded { continue } encoded := operations.Get([]byte(operationID)) @@ -3035,7 +3173,8 @@ func validateSpec(spec Spec) error { spec.SupersedesOperationID == spec.OperationID || (spec.Action != "force-delete-vm" && !cleanupPeerTargetActionPattern.MatchString(spec.Action) && - !cleanupBackupActionPattern.MatchString(spec.Action)) { + !cleanupBackupActionPattern.MatchString(spec.Action) && + !isVPCAction(spec.Action)) { return ErrInvalidSupersession } } diff --git a/agent/operation/journal_test.go b/agent/operation/journal_test.go index c5ecb42c..5e2a5898 100644 --- a/agent/operation/journal_test.go +++ b/agent/operation/journal_test.go @@ -1763,6 +1763,187 @@ func TestSuccessfulForceSuccessorOnlyRemovesPredecessorBlockerIndex(t *testing.T } } +func TestSuccessfulVPCRepairSupersedesOnlyExactFencedIntent(t *testing.T) { + j := openTestJournal(t) + const ( + vpcKey = "0123456789abcdef0123456789abcdef" + digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ) + makeSpec := func(id string) Spec { + return Spec{ + OperationID: id, + ResourceID: "vpc:" + vpcKey, + Action: "apply-vpc-network:" + vpcKey, + Fingerprint: Fingerprint("POST", "/v1/vpc/networks/"+vpcKey+"/apply/repair", []byte(id)), + Destructive: true, + ExpectedState: &ExpectedState{ + Kind: ExpectedVPCNetwork, TransitionID: id, + VPCKey: vpcKey, VPCRevision: 7, + VPCDigest: digest, VPCPresent: true, + }, + LockKeys: []string{"catalog:vpc-host/state", "catalog:vpc/" + vpcKey}, + } + } + predecessor := makeSpec("operation-vpc-repair-predecessor") + if _, _, err := j.Execute(context.Background(), predecessor, + func(context.Context) (json.RawMessage, error) { + return nil, testFailure{ + code: "vpc_reconcile_unconfirmed", message: "VPC state is inconsistent", + effect: true, safe: false, fence: true, + } + }); err == nil { + t.Fatal("VPC predecessor unexpectedly succeeded") + } + prior, err := j.Get(predecessor.OperationID) + if err != nil || !prior.ExecutionFenced { + t.Fatalf("VPC predecessor is not fenced: %#v err=%v", prior, err) + } + var before []byte + if err := j.db.View(func(tx *bolt.Tx) error { + before = append([]byte(nil), tx.Bucket(operationsBucket).Get( + []byte(predecessor.OperationID), + )...) + return nil + }); err != nil { + t.Fatal(err) + } + + successor := makeSpec("operation-vpc-repair-successor") + successor.SupersedesOperationID = predecessor.OperationID + if record, _, err := j.Execute(context.Background(), successor, + func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"applied":true}`), nil + }); err != nil || record.State != StateSucceeded { + t.Fatalf("VPC repair successor = %#v err=%v", record, err) + } + var after []byte + if err := j.db.View(func(tx *bolt.Tx) error { + after = append([]byte(nil), tx.Bucket(operationsBucket).Get( + []byte(predecessor.OperationID), + )...) + return nil + }); err != nil { + t.Fatal(err) + } + if !bytes.Equal(before, after) { + t.Fatal("VPC repair rewrote predecessor terminal evidence") + } + + followup := makeSpec("operation-vpc-after-repair") + followup.ExpectedState.VPCRevision = 8 + followup.ExpectedState.VPCDigest = strings.Repeat("b", 64) + if _, _, err := j.Execute(context.Background(), followup, + func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"applied":true}`), nil + }); err != nil { + t.Fatalf("VPC predecessor blocker survived repair: %v", err) + } + + invalid := makeSpec("operation-vpc-repair-wrong-intent") + invalid.ExpectedState.VPCDigest = strings.Repeat("c", 64) + invalid.SupersedesOperationID = predecessor.OperationID + if _, _, err := j.Execute(context.Background(), invalid, + func(context.Context) (json.RawMessage, error) { return nil, nil }); !errors.Is(err, ErrInvalidSupersession) { + t.Fatalf("VPC repair accepted changed intent: %v", err) + } +} + +func TestSuccessfulVPCRepairClosesExactUnknownRepairChain(t *testing.T) { + j := openTestJournal(t) + const ( + vpcKey = "1123456789abcdef0123456789abcdef" + digest = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + ) + makeSpec := func(id string) Spec { + return Spec{ + OperationID: id, + ResourceID: "vpc:" + vpcKey, + Action: "apply-vpc-network:" + vpcKey, + Fingerprint: Fingerprint( + "POST", "/v1/vpc/networks/"+vpcKey+"/apply/repair", + []byte(id), + ), + Destructive: true, + ExpectedState: &ExpectedState{ + Kind: ExpectedVPCNetwork, TransitionID: id, + VPCKey: vpcKey, VPCRevision: 9, + VPCDigest: digest, VPCPresent: true, + }, + LockKeys: []string{ + "catalog:vpc-host/state", "catalog:vpc/" + vpcKey, + }, + } + } + failUnknown := func(spec Spec) { + if _, _, err := j.Execute(context.Background(), spec, + func(context.Context) (json.RawMessage, error) { + return nil, testFailure{ + code: "vpc_reconcile_unconfirmed", + message: "repair result was not confirmed", + effect: true, safe: false, fence: true, + } + }); err == nil { + t.Fatalf("VPC generation %s unexpectedly succeeded", spec.OperationID) + } + } + + original := makeSpec("operation-vpc-chain-original") + failUnknown(original) + firstRepair := makeSpec("operation-vpc-chain-repair-one") + firstRepair.SupersedesOperationID = original.OperationID + failUnknown(firstRepair) + + var originalBefore, repairBefore []byte + if err := j.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket(operationsBucket) + originalBefore = append( + []byte(nil), bucket.Get([]byte(original.OperationID))..., + ) + repairBefore = append( + []byte(nil), bucket.Get([]byte(firstRepair.OperationID))..., + ) + return nil + }); err != nil { + t.Fatal(err) + } + + secondRepair := makeSpec("operation-vpc-chain-repair-two") + secondRepair.SupersedesOperationID = firstRepair.OperationID + if record, _, err := j.Execute( + context.Background(), secondRepair, + func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"applied":true}`), nil + }, + ); err != nil || record.State != StateSucceeded { + t.Fatalf("second repair = %#v err=%v", record, err) + } + + if err := j.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket(operationsBucket) + if !bytes.Equal( + originalBefore, bucket.Get([]byte(original.OperationID)), + ) || !bytes.Equal( + repairBefore, bucket.Get([]byte(firstRepair.OperationID)), + ) { + t.Fatal("successful repair rewrote predecessor evidence") + } + return nil + }); err != nil { + t.Fatal(err) + } + followup := makeSpec("operation-vpc-chain-followup") + followup.ExpectedState.VPCRevision = 10 + followup.ExpectedState.VPCDigest = strings.Repeat("d", 64) + if _, _, err := j.Execute( + context.Background(), followup, + func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"applied":true}`), nil + }, + ); err != nil { + t.Fatalf("repair chain retained a stale blocker: %v", err) + } +} + func TestPeerMigrationTargetCleanupMayCrossOnlyItsExactReceiveFence(t *testing.T) { j := openTestJournal(t) migrationID := "0123456789abcdef0123456789abcdef" diff --git a/agent/vpcrouting/manager.go b/agent/vpcrouting/manager.go index 3f0cc46b..46e0eb1f 100644 --- a/agent/vpcrouting/manager.go +++ b/agent/vpcrouting/manager.go @@ -133,6 +133,11 @@ func (m *Manager) Apply(ctx context.Context, request ApplyRequest) (Result, erro return loadErr } before := document.networkMap() + if normalized.VPCRepair { + return m.repairApplyLocked( + ctx, document, before, normalized, &result, + ) + } if existing, exists := document.network(normalized.Network.VPCKey); exists { switch { case existing.Network.DesiredRevision == normalized.Network.DesiredRevision && existing.Digest == normalized.DesiredDigest: @@ -202,6 +207,11 @@ func (m *Manager) Remove(ctx context.Context, request RemoveRequest) (Result, er return loadErr } existing, exists := document.network(request.VPCKey) + if request.VPCRepair { + return m.repairRemoveLocked( + ctx, document, existing, exists, request, &result, + ) + } if !exists || existing.Network.DesiredRevision != request.ExpectedRevision || existing.Digest != request.ExpectedDigest { return invalid("vpc_remove_precondition_failed", "VPC removal no longer matches the applied host state") } @@ -230,6 +240,82 @@ func (m *Manager) Remove(ctx context.Context, request RemoveRequest) (Result, er return result, err } +// repairApplyLocked deliberately re-runs the complete host projection even +// when the durable document already matches. It is reachable only through an +// Agent-authorized successor bound to one exact fenced predecessor; ordinary +// retries never enter this path. +func (m *Manager) repairApplyLocked(ctx context.Context, document stateDocument, + before map[string]Network, request ApplyRequest, result *Result) error { + if existing, exists := document.network(request.Network.VPCKey); exists && + (existing.Network.DesiredRevision != request.Network.DesiredRevision || + existing.Digest != request.DesiredDigest) { + return invalid("vpc_repair_projection_changed", "VPC repair no longer matches the durable host projection") + } + after := cloneNetworkMap(before) + after[request.Network.VPCKey] = request.Network + if err := validateUniqueBridgeNames(after); err != nil { + return unavailable("vpc_bridge_collision", "VPC bridge identity could not be allocated", err) + } + if err := m.driver.Reconcile(ctx, before, after); err != nil { + return m.rollback(before, after, err) + } + document.putNetwork(storedNetwork{ + Network: request.Network, Digest: request.DesiredDigest, + OperationID: request.OperationID, + }) + if err := m.save(document); err != nil { + if operation.IsUncertain(err) { + return err + } + return m.rollback(before, after, err) + } + bridge, _ := BridgeName(request.Network.VPCKey) + *result = Result{ + VPCKey: request.Network.VPCKey, + DesiredRevision: request.Network.DesiredRevision, + DesiredDigest: request.DesiredDigest, + Bridge: bridge, Applied: true, + } + return nil +} + +// repairRemoveLocked converges the host to an absent projection even when a +// prior partial removal already deleted the durable entry. Conflicting newer +// projections remain fail-closed. +func (m *Manager) repairRemoveLocked(ctx context.Context, document stateDocument, + existing storedNetwork, exists bool, request RemoveRequest, + result *Result) error { + if exists && (existing.Network.DesiredRevision != request.ExpectedRevision || + existing.Digest != request.ExpectedDigest) { + return invalid("vpc_repair_projection_changed", "VPC repair no longer matches the durable host projection") + } + before := document.networkMap() + after := cloneNetworkMap(before) + delete(after, request.VPCKey) + if err := m.driver.Reconcile(ctx, before, after); err != nil { + return m.rollback(before, after, err) + } + document.removeNetwork(removalMarker{ + VPCKey: request.VPCKey, + PreviousRevision: request.ExpectedRevision, + PreviousDigest: request.ExpectedDigest, + OperationID: request.OperationID, + }) + if err := m.save(document); err != nil { + if operation.IsUncertain(err) { + return err + } + return m.rollback(before, after, err) + } + *result = Result{ + VPCKey: request.VPCKey, + DesiredRevision: request.ExpectedRevision, + DesiredDigest: request.ExpectedDigest, + Removed: true, + } + return nil +} + func (m *Manager) Inspect(operationID, vpcKey string, revision uint64, digest string, present bool) (Inspection, error) { if m == nil || operationID == "" || !vpcKeyPattern.MatchString(vpcKey) || revision == 0 || !lowerHexDigest.MatchString(digest) { return Inspection{}, nil diff --git a/agent/vpcrouting/types.go b/agent/vpcrouting/types.go index 1e501e4b..a4c53ed0 100644 --- a/agent/vpcrouting/types.go +++ b/agent/vpcrouting/types.go @@ -58,16 +58,20 @@ type Network struct { } type ApplyRequest struct { - OperationID string `json:"operation_id,omitempty"` - DesiredDigest string `json:"desired_digest"` - Network Network `json:"network"` + OperationID string `json:"operation_id,omitempty"` + DesiredDigest string `json:"desired_digest"` + Network Network `json:"network"` + VPCRepair bool `json:"vpc_repair,omitempty"` + SupersedesOperationID string `json:"supersedes_operation_id,omitempty"` } type RemoveRequest struct { - OperationID string `json:"operation_id,omitempty"` - VPCKey string `json:"vpc_key"` - ExpectedRevision uint64 `json:"expected_revision"` - ExpectedDigest string `json:"expected_digest"` + OperationID string `json:"operation_id,omitempty"` + VPCKey string `json:"vpc_key"` + ExpectedRevision uint64 `json:"expected_revision"` + ExpectedDigest string `json:"expected_digest"` + VPCRepair bool `json:"vpc_repair,omitempty"` + SupersedesOperationID string `json:"supersedes_operation_id,omitempty"` } type Result struct { @@ -222,6 +226,11 @@ func ValidateApplyRequest(request ApplyRequest) (ApplyRequest, error) { if !operationIDPattern.MatchString(request.OperationID) { return ApplyRequest{}, invalid("invalid_operation_id", "operation_id must contain 8-80 URL-safe characters") } + if request.VPCRepair != (request.SupersedesOperationID != "") || + (request.VPCRepair && (!operationIDPattern.MatchString(request.SupersedesOperationID) || + request.SupersedesOperationID == request.OperationID)) { + return ApplyRequest{}, invalid("invalid_vpc_repair", "VPC repair requires one different valid predecessor operation") + } network, err := Normalize(request.Network) if err != nil { return ApplyRequest{}, err @@ -241,6 +250,11 @@ func ValidateRemoveRequest(request RemoveRequest) error { if !operationIDPattern.MatchString(request.OperationID) || !vpcKeyPattern.MatchString(request.VPCKey) || request.ExpectedRevision == 0 || !lowerHexDigest.MatchString(request.ExpectedDigest) { return invalid("invalid_vpc_remove", "VPC removal identity, revision, or digest is invalid") } + if request.VPCRepair != (request.SupersedesOperationID != "") || + (request.VPCRepair && (!operationIDPattern.MatchString(request.SupersedesOperationID) || + request.SupersedesOperationID == request.OperationID)) { + return invalid("invalid_vpc_repair", "VPC repair requires one different valid predecessor operation") + } return nil } diff --git a/agent/vpcrouting/types_test.go b/agent/vpcrouting/types_test.go index 7a1e07d8..f81d6250 100644 --- a/agent/vpcrouting/types_test.go +++ b/agent/vpcrouting/types_test.go @@ -80,6 +80,35 @@ func TestNormalizeRejectsAmbiguousOrUnsafeRoutes(t *testing.T) { } } +func TestValidateRequestsRejectMalformedRepairIdentity(t *testing.T) { + apply := applyRequest(t, 1, "operation-vpc-repair-current") + apply.VPCRepair = true + if _, err := ValidateApplyRequest(apply); err == nil { + t.Fatal("repair apply without predecessor was accepted") + } + apply.SupersedesOperationID = apply.OperationID + if _, err := ValidateApplyRequest(apply); err == nil { + t.Fatal("repair apply superseding itself was accepted") + } + apply.SupersedesOperationID = "operation-vpc-repair-prior" + if _, err := ValidateApplyRequest(apply); err != nil { + t.Fatalf("valid repair apply was rejected: %v", err) + } + + remove := RemoveRequest{ + OperationID: "operation-vpc-repair-remove", VPCKey: testVPCKey, + ExpectedRevision: 1, ExpectedDigest: apply.DesiredDigest, + SupersedesOperationID: "operation-vpc-repair-prior", + } + if err := ValidateRemoveRequest(remove); err == nil { + t.Fatal("remove predecessor without repair marker was accepted") + } + remove.VPCRepair = true + if err := ValidateRemoveRequest(remove); err != nil { + t.Fatalf("valid repair remove was rejected: %v", err) + } +} + type fakeDriver struct { calls [][2]map[string]Network errors []error @@ -141,6 +170,47 @@ func TestManagerAppliesReplaysAndRemovesWholeState(t *testing.T) { } } +func TestManagerRepairForcesRuntimeConvergenceAndRebindsMarker(t *testing.T) { + driver := &fakeDriver{} + manager := testManager(t, driver) + original := applyRequest(t, 5, "operation-vpc-repair-original") + if _, err := manager.Apply(t.Context(), original); err != nil { + t.Fatal(err) + } + repair := original + repair.OperationID = "operation-vpc-repair-apply" + repair.VPCRepair = true + repair.SupersedesOperationID = original.OperationID + if result, err := manager.Apply(t.Context(), repair); err != nil || + !result.Applied || len(driver.calls) != 2 || + !reflect.DeepEqual(driver.calls[1][0], driver.calls[1][1]) { + t.Fatalf("repair apply=%#v calls=%#v err=%v", result, driver.calls, err) + } + inspection, err := manager.Inspect( + repair.OperationID, testVPCKey, 5, repair.DesiredDigest, true, + ) + if err != nil || !inspection.Applied { + t.Fatalf("repair marker was not rebound: %#v err=%v", inspection, err) + } + remove := RemoveRequest{ + OperationID: "operation-vpc-repair-remove-source", + VPCKey: testVPCKey, ExpectedRevision: 5, + ExpectedDigest: repair.DesiredDigest, + } + if _, err := manager.Remove(t.Context(), remove); err != nil { + t.Fatal(err) + } + repairRemove := remove + repairRemove.OperationID = "operation-vpc-repair-remove" + repairRemove.VPCRepair = true + repairRemove.SupersedesOperationID = remove.OperationID + if result, err := manager.Remove(t.Context(), repairRemove); err != nil || + !result.Removed || len(driver.calls) != 4 || + len(driver.calls[3][0]) != 0 || len(driver.calls[3][1]) != 0 { + t.Fatalf("repair remove=%#v calls=%#v err=%v", result, driver.calls, err) + } +} + func TestManagerRestoresOnlyDurablyCommittedProjection(t *testing.T) { path := t.TempDir() + "/vpc-state.json" applyDriver := &fakeDriver{} diff --git a/agent/worker/protocol.go b/agent/worker/protocol.go index 719f5e81..03b562b7 100644 --- a/agent/worker/protocol.go +++ b/agent/worker/protocol.go @@ -42,7 +42,8 @@ type Request struct { // SupersededOperation is a non-secret immutable snapshot injected only by // the parent Agent after it authorizes an exceptional successor. The child // uses a separate Journal, so it cannot load the predecessor itself. - SupersededOperation *operation.Record `json:"superseded_operation,omitempty"` + SupersededOperation *operation.Record `json:"superseded_operation,omitempty"` + SupersededExpectedState *operation.ExpectedState `json:"superseded_expected_state,omitempty"` } // RequestIdentity is the non-secret, immutable subset persisted beside an @@ -126,14 +127,20 @@ func (r Request) Validate() error { } if r.SupersededOperation != nil { prior := r.SupersededOperation + vpcRepair := strings.HasSuffix(r.RequestTarget, "/repair") if prior.OperationID == "" || prior.OperationID == r.OperationID || !operation.ValidOperationIdentity(prior.Action, prior.ResourceID, prior.Fingerprint) || prior.State != operation.StateFailed || !prior.EffectMayHaveApplied || prior.SafeToResubmit || prior.Tombstone || !(strings.HasSuffix(r.RequestTarget, "/force-delete") || - strings.HasSuffix(r.RequestTarget, "/peer-migration/cleanup-target")) { + strings.HasSuffix(r.RequestTarget, "/peer-migration/cleanup-target") || + vpcRepair) || + (vpcRepair != (r.SupersededExpectedState != nil)) || + (vpcRepair && !prior.ExecutionFenced) { return errors.New("worker superseded operation is invalid") } + } else if r.SupersededExpectedState != nil { + return errors.New("worker superseded expected state has no operation") } if Fingerprint(r.Method, r.RequestTarget, []byte(r.Body)) != r.Fingerprint { return errors.New("worker request fingerprint does not match target and body") diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index 07abcf88..c92be0da 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -8,7 +8,7 @@ use App\Services\Config; /** Initializes only the clean-break PHP business and Agent log schemas. */ final class CloudSetup { - private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v5-fences'; + private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v6-repair'; private const MAIN_TABLES = [ 'cloud_regions', 'cloud_network_lines', 'cloud_resource_pools', 'cloud_hosts', 'cloud_host_agents', @@ -323,6 +323,7 @@ final class CloudSetup 'cloud_job_backup', 'cloud_job_backup_restore', 'cloud_job_backup_recovery', 'cloud_job_vpc', 'cloud_job_vpc_port', 'cloud_job_vpc_batch', + 'cloud_job_supersedes', 'cloud_job_vpc_repair_root', ], 'cloud_agent_completion_events' => [ 'PRIMARY', 'uniq_cloud_completion_operation', diff --git a/web/app/Controllers/CloudAdminOperationsController.php b/web/app/Controllers/CloudAdminOperationsController.php index fe832a22..41a56879 100644 --- a/web/app/Controllers/CloudAdminOperationsController.php +++ b/web/app/Controllers/CloudAdminOperationsController.php @@ -88,4 +88,17 @@ final class CloudAdminOperationsController 202 )->withHeader('Cache-Control', 'private, no-store'); } + + public function repairVpc($request, $response, $args) + { + return $response->withJson( + CloudAdminOperationRecoveryService::repairVpc( + $request->getAttribute('cloud_actor') + ?? CloudAccess::actor(false), + (string) ($args['id'] ?? ''), + CloudInput::object($request) + ), + 202 + )->withHeader('Cache-Control', 'private, no-store'); + } } diff --git a/web/app/Models/CloudJob.php b/web/app/Models/CloudJob.php index 2c8d2120..fb407920 100644 --- a/web/app/Models/CloudJob.php +++ b/web/app/Models/CloudJob.php @@ -14,6 +14,8 @@ final class CloudJob extends Model 'vpc_id' => 'int', 'vpc_port_id' => 'int', 'vpc_batch_id' => 'int', + 'supersedes_job_id' => 'int', + 'vpc_repair_root_job_id' => 'int', ]; /** diff --git a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php index 8fe1608e..7f130079 100644 --- a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php @@ -532,6 +532,37 @@ final class CloudAdminOperationRecoveryService ]; } + /** Queue a new VPC convergence operation after one fresh Agent inspection. */ + public static function repairVpc( + array $actor, + string $id, + array $input + ): array { + CloudInput::only($input, ['acknowledge_repair']); + if (($input['acknowledge_repair'] ?? null) !== true) { + throw new CloudError('必须明确确认将创建新的 VPC 修复任务', 409); + } + if (!($actor['is_admin'] ?? false) + || !($actor['is_super_admin'] ?? false) + || !($actor['can_manage'] ?? false)) { + throw new CloudError('只有超级管理员可以修复 VPC 状态', 403); + } + CloudAccess::manage($actor); + if (preg_match('/^[1-9][0-9]{0,17}$/D', $id) !== 1) { + throw new CloudError('编号无效', 400); + } + $inspection = self::inspect($actor, $id); + $job = CloudVpcPortService::repairUnknownEffect( + $actor, + (int) $id, + $inspection + ); + return [ + 'task_id' => (string) $job->id, + 'status_url' => '/api/v1/tasks/' . (int) $job->id, + ]; + } + /** @return array{0:CloudJob,1:CloudAgentClient} */ private static function context( array $actor, @@ -565,7 +596,9 @@ final class CloudAdminOperationRecoveryService 'effect_resolution', 'effect_resolution_reason', 'dispatched_at', 'finalization_incident', 'finalization_error_code', 'result_json', 'sealed_payload', 'instance_id', 'snapshot_id', 'backup_id', - 'backup_restore_id', 'parent_job_id', + 'backup_restore_id', 'backup_recovery_id', 'parent_job_id', + 'supersedes_job_id', 'vpc_repair_root_job_id', + 'vpc_id', 'vpc_port_id', 'vpc_batch_id', 'user_id', 'order_id', ])->where('id', (int) $id)->first(); if ($job === null) { diff --git a/web/app/Services/Cloud/CloudAdminOperationsService.php b/web/app/Services/Cloud/CloudAdminOperationsService.php index 03f28a60..b098ec15 100644 --- a/web/app/Services/Cloud/CloudAdminOperationsService.php +++ b/web/app/Services/Cloud/CloudAdminOperationsService.php @@ -325,7 +325,8 @@ final class CloudAdminOperationsService foreach ([ 'id', 'operation_id', 'request_idempotency_key', 'request_fingerprint', 'user_id', 'host_id', 'kind', - 'parent_job_id', 'execution_mode', 'invoice_id', 'phase', + 'parent_job_id', 'supersedes_job_id', + 'vpc_repair_root_job_id', 'execution_mode', 'invoice_id', 'phase', 'order_id', 'instance_id', 'snapshot_id', 'outcome', 'status', 'attempts', 'dispatch_attempts', 'error_code', 'dispatched_at', 'dispatch_deadline_at', 'dispatch_started_at', @@ -440,6 +441,9 @@ final class CloudAdminOperationsService [$canForceDelete, $forceDeleteReason] = self::forceDeleteable( $job, $canManage ); + [$canRepairVpc, $repairVpcReason] = self::vpcRepairable( + $job, $canManage + ); $recovery = CloudJobService::recoveryPresentation($job, true); $effectScope = $recovery === null ? CloudJobService::effectScope($job) : null; @@ -477,6 +481,10 @@ final class CloudAdminOperationsService 'resolve_effect_reason' => $resolveReason, 'can_force_delete' => $canForceDelete, 'force_delete_reason' => $forceDeleteReason, + 'can_repair_vpc' => $canRepairVpc, + 'repair_vpc_reason' => $repairVpcReason, + 'supersedes_job_id' => $job->supersedes_job_id === null + ? null : (string) $job->supersedes_job_id, 'dispatch_attempts' => $job->dispatch_attempts === null ? null : CloudDispatchLedger::attempts($job), 'dispatch_state' => CloudDispatchLedger::isInvalid($job) @@ -681,6 +689,35 @@ final class CloudAdminOperationsService return [true, '将创建新的任务并再次确认数据丢失风险']; } + /** @return array{0:bool,1:string} */ + private static function vpcRepairable($job, bool $canManage): array + { + if (!$canManage) { + return [false, '只有超级管理员可以创建 VPC 修复任务']; + } + if (!in_array((string) $job->kind, [ + 'vpc_host_apply', 'vpc_host_remove', + 'vpc_port_attach', 'vpc_port_detach', + ], true)) { + return [false, '当前任务不是 VPC 操作']; + } + if (CloudDispatchLedger::isInvalid($job) + || CloudDispatchLedger::hasRepairIncident($job) + || (bool) ($job->finalization_incident ?? false) + || (string) $job->execution_mode !== 'agent' + || !CloudDispatchLedger::isDispatched($job) + || (string) $job->phase !== 'terminal' + || (string) $job->outcome !== 'failed' + || (string) $job->status !== 'failed' + || !(bool) $job->effect_may_have_applied + || (bool) $job->safe_to_resubmit + || (string) $job->effect_resolution !== 'unknown') { + return [false, 'VPC 任务不处于可修复的影响未知状态']; + } + return [true, + '服务端会先读取宿主机新鲜证据,再创建具有新操作标识的精确修复任务']; + } + private static function timestamp($value): ?int { return $value !== null && (int) $value > 0 ? (int) $value : null; diff --git a/web/app/Services/Cloud/CloudJobService.php b/web/app/Services/Cloud/CloudJobService.php index c3e3a916..dfe31990 100644 --- a/web/app/Services/Cloud/CloudJobService.php +++ b/web/app/Services/Cloud/CloudJobService.php @@ -544,6 +544,16 @@ final class CloudJobService 'vpc_route_predecessors.kind', $routeKinds ) + ->where(static function ($outsideRepair): void { + $outsideRepair->whereNull( + 'cloud_jobs.vpc_repair_root_job_id' + )->orWhereRaw( + 'COALESCE(' + . 'vpc_route_predecessors.vpc_repair_root_job_id, ' + . 'vpc_route_predecessors.id) <> ' + . 'cloud_jobs.vpc_repair_root_job_id' + ); + }) ->where(static function ($blocking): void { $blocking->where( 'vpc_route_predecessors.phase', diff --git a/web/app/Services/Cloud/CloudVpcPortService.php b/web/app/Services/Cloud/CloudVpcPortService.php index d47b6397..d3493fb4 100644 --- a/web/app/Services/Cloud/CloudVpcPortService.php +++ b/web/app/Services/Cloud/CloudVpcPortService.php @@ -10,6 +10,8 @@ use App\Models\CloudJob; use App\Models\CloudResourcePool; use App\Models\CloudVpc; use App\Models\CloudVpcHostState; +use App\Models\CloudVpcMembershipBatch; +use App\Models\CloudVpcMembershipBatchItem; use App\Models\CloudVpcPort; /** @@ -802,6 +804,40 @@ final class CloudVpcPortService || ($body['operation_id'] ?? null) !== (string) $job->operation_id) { throw new CloudError('VPC 任务与资源账本不一致', 409); } + $repair = $job->supersedes_job_id !== null; + $prior = null; + if ($repair) { + $prior = CloudJob::query()->where( + 'id', (int) $job->supersedes_job_id + )->first(); + $rootId = (int) ($job->vpc_repair_root_job_id ?? 0); + if ($prior === null || $rootId < 1 + || ($payload['vpc_repair'] ?? null) !== true + || (int) ($payload['vpc_repair_root_job_id'] ?? 0) + !== $rootId + || ($body['vpc_repair'] ?? null) !== true + || ($body['supersedes_operation_id'] ?? null) + !== (string) $prior->operation_id + || (int) $prior->instance_id !== (int) $job->instance_id + || (int) $prior->vpc_id !== (int) $job->vpc_id + || (int) $prior->vpc_port_id !== (int) $job->vpc_port_id + || (int) $prior->host_id !== (int) $job->host_id + || (string) $prior->kind !== $kind + || (string) $prior->phase !== 'terminal' + || (string) $prior->outcome !== 'failed' + || !(bool) $prior->effect_may_have_applied + || (bool) $prior->safe_to_resubmit + || (string) $prior->effect_resolution !== 'unknown' + || (bool) $prior->finalization_incident + || (int) ($prior->vpc_repair_root_job_id ?? $prior->id) + !== $rootId) { + throw new CloudError('VPC 修复任务继任账本不一致', 409); + } + } elseif (($payload['vpc_repair'] ?? false) === true + || array_key_exists('vpc_repair', $body) + || array_key_exists('supersedes_operation_id', $body)) { + throw new CloudError('普通 VPC 任务包含修复字段', 409); + } if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) { $state = CloudVpcHostState::query() ->where('vpc_id', (int) $vpc->id) @@ -830,12 +866,18 @@ final class CloudVpcPortService } $expected = self::portBody($job, $vpc, $port, $state); } + if ($repair) { + $expected['vpc_repair'] = true; + $expected['supersedes_operation_id'] = + (string) $prior->operation_id; + } $expectedTarget = in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true) ? '/v1/vpc/networks/' . rawurlencode((string) $vpc->vpc_key) . ($kind === 'vpc_host_apply' ? '/apply' : '/remove') + . ($repair ? '/repair' : '') : '/v1/vms/' . rawurlencode(strtolower((string) $instance->uuid)) - . '/vpc-port'; + . '/vpc-port' . ($repair ? '/repair' : ''); $expectedMethod = match ($kind) { 'vpc_host_apply', 'vpc_host_remove', 'vpc_port_attach' => 'POST', 'vpc_port_detach' => 'DELETE', @@ -882,6 +924,16 @@ final class CloudVpcPortService if ($port === null || $vpc === null || $instance === null) { throw new CloudError('VPC 任务资源账本已丢失', 409); } + if ($job->supersedes_job_id !== null) { + if ($status === 'succeeded') { + self::settleRepairChainLocked($job); + } elseif ($status === 'failed') { + self::restoreRepairPredecessorLocked( + $job, $vpc, $port + ); + return; + } + } if ($job->vpc_batch_id !== null) { CloudVpcMembershipBatchService::finishChildLocked( $job, $status, $result, $operation, $vpc, $port, $instance @@ -904,6 +956,134 @@ final class CloudVpcPortService ); } + /** Resolve only VPC safety metadata after the successor proves convergence. */ + private static function settleRepairChainLocked(CloudJob $job): void + { + $rootId = (int) ($job->vpc_repair_root_job_id ?? 0); + $priorId = (int) ($job->supersedes_job_id ?? 0); + if ($rootId < 1 || $priorId < 1) { + throw new CloudError('VPC 修复继任链缺失', 409); + } + $seen = [(int) $job->id => true]; + $now = time(); + for ($depth = 0; $depth < 32; $depth++) { + if ($priorId < 1 || isset($seen[$priorId])) { + throw new CloudError('VPC 修复继任链包含循环', 409); + } + $seen[$priorId] = true; + $prior = CloudJob::query()->where('id', $priorId) + ->lockForUpdate()->first(); + if ($prior === null + || (string) $prior->kind !== (string) $job->kind + || (int) $prior->instance_id !== (int) $job->instance_id + || (int) $prior->vpc_id !== (int) $job->vpc_id + || (int) $prior->vpc_port_id !== (int) $job->vpc_port_id + || (int) $prior->host_id !== (int) $job->host_id + || (string) $prior->phase !== 'terminal' + || (string) $prior->outcome !== 'failed' + || !(bool) $prior->effect_may_have_applied + || (bool) $prior->safe_to_resubmit + || (string) $prior->effect_resolution !== 'unknown' + || (bool) $prior->finalization_incident + || (int) ($prior->vpc_repair_root_job_id ?? $prior->id) + !== $rootId) { + throw new CloudError('VPC 修复继任链已变化', 409); + } + $nextId = (int) ($prior->supersedes_job_id ?? 0); + $prior->effect_resolution = 'applied'; + $prior->effect_resolved_at = $now; + $prior->effect_resolved_by = null; + $prior->effect_resolution_reason = substr( + 'superseded_by_vpc_repair:' . (int) $job->id . ':' + . (string) $job->operation_id, + 0, + 500 + ); + $prior->available_at = 0; + $prior->read_at = null; + $prior->updated_at = $now; + $prior->save(); + if ($nextId < 1) { + if ((int) $prior->id !== $rootId) { + throw new CloudError('VPC 修复根任务身份不一致', 409); + } + return; + } + $priorId = $nextId; + } + throw new CloudError('VPC 修复继任链超过安全上限', 409); + } + + /** A proved no-effect repair restores the still-fenced predecessor tip. */ + private static function restoreRepairPredecessorLocked( + CloudJob $job, + CloudVpc $vpc, + CloudVpcPort $port + ): void { + $prior = CloudJob::query()->where( + 'id', (int) $job->supersedes_job_id + )->lockForUpdate()->first(); + if ($prior === null + || (string) $prior->kind !== (string) $job->kind + || (string) $prior->effect_resolution !== 'unknown' + || (string) $prior->phase !== 'terminal' + || (string) $prior->outcome !== 'failed') { + throw new CloudError('VPC 修复前序任务已变化', 409); + } + $now = time(); + if (in_array((string) $job->kind, + ['vpc_host_apply', 'vpc_host_remove'], true)) { + $state = CloudVpcHostState::query() + ->where('vpc_id', (int) $job->vpc_id) + ->where('host_id', (int) $job->host_id) + ->lockForUpdate()->first(); + if ($state === null || !hash_equals( + (string) $state->last_operation_id, + (string) $job->operation_id + )) { + throw new CloudError('VPC 宿主机修复回退代次不一致', 409); + } + $state->last_operation_id = (string) $prior->operation_id; + $state->sync_state = 'unknown'; + $state->last_error_code = 'vpc_result_uncertain'; + $state->last_failure_at = $now; + $state->updated_at = $now; + $state->save(); + } else { + if (!hash_equals( + (string) $port->last_operation_id, + (string) $job->operation_id + )) { + throw new CloudError('VPC 网卡修复回退代次不一致', 409); + } + $port->last_operation_id = (string) $prior->operation_id; + $port->last_error_code = 'vpc_result_uncertain'; + $port->updated_at = $now; + $port->save(); + } + if ($job->vpc_batch_id !== null + && in_array((string) $job->kind, + ['vpc_port_attach', 'vpc_port_detach'], true)) { + $item = CloudVpcMembershipBatchItem::query() + ->where('batch_id', (int) $job->vpc_batch_id) + ->where('instance_id', (int) $job->instance_id) + ->where('vpc_port_id', (int) $job->vpc_port_id) + ->lockForUpdate()->first(); + if ($item === null + || (int) $item->child_job_id !== (int) $job->id) { + throw new CloudError('VPC 批次修复回退代次不一致', 409); + } + $item->child_job_id = (int) $prior->id; + $item->state = 'unknown'; + $item->last_error_code = 'vpc_result_uncertain'; + $item->updated_at = $now; + $item->save(); + } + $vpc->state = 'degraded'; + $vpc->updated_at = $now; + $vpc->save(); + } + public static function markJobUncertain(CloudJob $job): void { $kind = (string) $job->kind; @@ -946,6 +1126,296 @@ final class CloudVpcPortService } } + /** + * Queue one new, explicitly linked repair generation for an exact VPC + * operation whose Agent journal is reachable but still inconclusive. + * The failed operation is never replayed. + */ + public static function repairUnknownEffect( + array $actor, + int $jobId, + array $inspection + ): CloudJob { + CloudAccess::manage($actor); + $reference = CloudJob::query()->where('id', $jobId)->first(); + if ($reference === null) { + throw new CloudError('任务不存在', 404); + } + if (!in_array((string) $reference->kind, self::JOB_KINDS, true) + || (int) ($reference->instance_id ?? 0) < 1 + || (int) ($reference->vpc_id ?? 0) < 1 + || (int) ($reference->vpc_port_id ?? 0) < 1 + || (int) ($reference->host_id ?? 0) < 1) { + throw new CloudError('该任务不是可修复的 VPC 操作', 409); + } + self::assertRepairInspection($reference, $inspection); + + return (new CloudJob())->getConnection()->transaction( + function () use ($reference, $inspection): CloudJob { + $instance = CloudVpcLockOrder::instance( + (int) $reference->instance_id + ); + $vpc = CloudVpcLockOrder::vpcById( + (int) $reference->vpc_id + ); + $port = CloudVpcLockOrder::portById( + (int) $reference->vpc_port_id + ); + $state = CloudVpcHostState::query() + ->where('vpc_id', (int) $reference->vpc_id) + ->where('host_id', (int) $reference->host_id) + ->lockForUpdate()->first(); + if ($instance === null || $vpc === null || $port === null + || $state === null + || (int) $port->instance_id !== (int) $instance->id + || (int) $port->vpc_id !== (int) $vpc->id + || (int) $reference->host_id + !== (int) $state->host_id) { + throw new CloudError('VPC 修复资源账本不完整', 409); + } + + $batch = null; + $item = null; + if ($reference->vpc_batch_id !== null) { + $batch = CloudVpcMembershipBatch::query() + ->where('id', (int) $reference->vpc_batch_id) + ->lockForUpdate()->first(); + if ($batch === null + || (string) $batch->state !== 'blocked' + || (int) $batch->vpc_id !== (int) $vpc->id) { + throw new CloudError('VPC 批次不处于可修复状态', 409); + } + if (in_array((string) $reference->kind, + ['vpc_port_attach', 'vpc_port_detach'], true)) { + $item = CloudVpcMembershipBatchItem::query() + ->where('batch_id', (int) $batch->id) + ->where('instance_id', (int) $instance->id) + ->where('vpc_port_id', (int) $port->id) + ->lockForUpdate()->first(); + } + } + + $prior = CloudJob::query()->where( + 'id', (int) $reference->id + )->lockForUpdate()->first(); + if ($prior === null) { + throw new CloudError('待修复任务已不存在', 409); + } + self::assertRepairCandidateLocked( + $prior, $instance, $vpc, $port, $state, + $batch, $item, $inspection + ); + $rootId = (int) ($prior->vpc_repair_root_job_id + ?? $prior->id); + $chain = CloudJob::query() + ->where(static function ($query) use ($rootId): void { + $query->where('id', $rootId) + ->orWhere('vpc_repair_root_job_id', $rootId); + })->orderBy('id')->lockForUpdate()->get(); + $unknownGenerations = $chain->filter( + static fn (CloudJob $candidate): bool => + (string) $candidate->effect_resolution === 'unknown' + ); + $latestUnknown = $unknownGenerations->last(); + if ($chain->isEmpty() || $unknownGenerations->count() > 32 + || $chain->contains(static fn (CloudJob $candidate): bool => + (string) $candidate->phase !== 'terminal') + || !$latestUnknown instanceof CloudJob + || (int) $latestUnknown->id !== (int) $prior->id) { + throw new CloudError('VPC 修复链已有其他活动代次', 409); + } + foreach ($chain as $candidate) { + if ((int) $candidate->instance_id !== (int) $instance->id + || (int) $candidate->vpc_id !== (int) $vpc->id + || (int) $candidate->vpc_port_id !== (int) $port->id + || (int) $candidate->host_id !== (int) $prior->host_id + || (string) $candidate->kind + !== (string) $prior->kind) { + throw new CloudError('VPC 修复链资源身份不一致', 409); + } + } + + $payload = CloudSecrets::decrypt( + (string) $prior->sealed_payload, 'job-payload' + ); + self::assertJobRunnable($prior, $payload); + $job = self::baseJob( + (string) $prior->kind, + $vpc, + $port, + (int) $prior->host_id, + $prior->vpc_batch_id === null + ? null : (int) $prior->vpc_batch_id + ); + $job->supersedes_job_id = (int) $prior->id; + $job->vpc_repair_root_job_id = $rootId; + $extra = []; + foreach ([ + 'maintenance_migration_id', 'vpc_migration_direction', + 'vpc_batch_id', 'vpc_batch_revision', + ] as $key) { + if (array_key_exists($key, $payload)) { + $extra[$key] = $payload[$key]; + } + } + if ((string) $prior->kind === 'vpc_host_apply' + && is_array($payload['host_projection'] ?? null) + && !array_is_list($payload['host_projection'])) { + $extra['host_projection'] = $payload['host_projection']; + } + if ($job->vpc_batch_id !== null + && in_array((string) $job->kind, + ['vpc_host_apply', 'vpc_host_remove'], true)) { + $revision = (int) ($extra['vpc_batch_revision'] + ?? $batch?->desired_revision ?? 0); + if ($revision < 1) { + throw new CloudError('VPC 批次修复缺少投影代次', 409); + } + $job->request_idempotency_key = substr( + self::batchProjectionRequestPrefix( + (int) $job->vpc_batch_id, + $revision + ) . (string) $job->kind . '-' + . (int) $job->host_id . '-repair-' + . (string) $job->operation_id, + 0, + 128 + ); + } + if (in_array((string) $job->kind, + ['vpc_host_apply', 'vpc_host_remove'], true)) { + $body = (string) $job->kind === 'vpc_host_apply' + ? self::hostApplyBody($job, $vpc, $state, $payload) + : self::hostRemoveBody($job, $vpc, $state); + $target = '/v1/vpc/networks/' + . rawurlencode((string) $vpc->vpc_key) + . ((string) $job->kind === 'vpc_host_apply' + ? '/apply/repair' : '/remove/repair'); + $method = 'POST'; + } else { + if (!self::hostReady($state)) { + throw new CloudError( + 'VPC 网卡修复所需宿主机路由未就绪', 409 + ); + } + $body = self::portBody($job, $vpc, $port, $state); + $target = '/v1/vms/' . rawurlencode( + strtolower((string) $instance->uuid) + ) . '/vpc-port/repair'; + $method = (string) $job->kind === 'vpc_port_attach' + ? 'POST' : 'DELETE'; + } + $body['vpc_repair'] = true; + $body['supersedes_operation_id'] = + (string) $prior->operation_id; + self::sealJob( + $job, $vpc, $port, $method, $target, $body, + $extra + [ + 'vpc_repair' => true, + 'vpc_repair_root_job_id' => $rootId, + ] + ); + if (in_array((string) $job->kind, + ['vpc_host_apply', 'vpc_host_remove'], true)) { + $state->last_operation_id = (string) $job->operation_id; + $state->sync_state = 'pending'; + $state->last_error_code = ''; + $state->updated_at = time(); + $state->save(); + } else { + $port->last_operation_id = (string) $job->operation_id; + $port->last_error_code = ''; + $port->updated_at = time(); + $port->save(); + if ($item !== null) { + $item->child_job_id = (int) $job->id; + $item->state = 'queued'; + $item->last_error_code = ''; + $item->updated_at = time(); + $item->save(); + } + } + return $job; + }, + 3 + ); + } + + private static function assertRepairInspection( + CloudJob $job, + array $inspection + ): void { + if (($inspection['operation_id'] ?? null) + !== (string) $job->operation_id + || ($inspection['journal_status'] ?? null) !== 'failed' + || ($inspection['outcome'] ?? null) !== 'inconclusive' + || ($inspection['effect_resolution'] ?? null) !== 'inconclusive' + || ($inspection['fence_state'] ?? null) !== 'active' + || ($inspection['fence_resolution_required'] ?? null) !== true + || array_key_exists('completion_delivery_state', $inspection) + || !is_int($inspection['checked_at'] ?? null) + || $inspection['checked_at'] < time() - 120 + || $inspection['checked_at'] > time() + 5) { + throw new CloudError( + '宿主机没有提供可执行 VPC 修复的新鲜不一致证据', 409 + ); + } + } + + private static function assertRepairCandidateLocked( + CloudJob $job, + CloudInstance $instance, + CloudVpc $vpc, + CloudVpcPort $port, + CloudVpcHostState $state, + ?CloudVpcMembershipBatch $batch, + ?CloudVpcMembershipBatchItem $item, + array $inspection + ): void { + self::assertRepairInspection($job, $inspection); + if (!in_array((string) $job->kind, self::JOB_KINDS, true) + || (string) $job->execution_mode !== 'agent' + || (string) $job->phase !== 'terminal' + || (string) $job->outcome !== 'failed' + || (string) $job->status !== 'failed' + || (int) $job->dispatch_attempts !== 1 + || !(bool) $job->effect_may_have_applied + || (bool) $job->safe_to_resubmit + || (string) $job->effect_resolution !== 'unknown' + || (bool) $job->finalization_incident + || (int) $job->instance_id !== (int) $instance->id + || (int) $job->vpc_id !== (int) $vpc->id + || (int) $job->vpc_port_id !== (int) $port->id + || (int) $job->host_id !== (int) $state->host_id + || (int) $job->user_id !== (int) $port->user_id) { + throw new CloudError('VPC 任务已不处于可修复状态', 409); + } + if (in_array((string) $job->kind, + ['vpc_host_apply', 'vpc_host_remove'], true)) { + if (!hash_equals( + (string) $state->last_operation_id, + (string) $job->operation_id + ) || (string) $state->sync_state !== 'unknown') { + throw new CloudError('VPC 宿主机修复代次已过期', 409); + } + } elseif (!hash_equals( + (string) $port->last_operation_id, + (string) $job->operation_id + ) || !in_array((string) $port->state, + ['preparing', 'detaching'], true)) { + throw new CloudError('VPC 网卡修复代次已过期', 409); + } + if ($job->vpc_batch_id !== null + && ($batch === null || (string) $batch->state !== 'blocked' + || (in_array((string) $job->kind, + ['vpc_port_attach', 'vpc_port_detach'], true) + && ($item === null + || (int) $item->child_job_id !== (int) $job->id + || (string) $item->state !== 'unknown')))) { + throw new CloudError('VPC 批次修复子任务已过期', 409); + } + } + private static function finishHostJobLocked(CloudJob $job, string $status, $result, array $operation, CloudVpc $vpc, CloudVpcPort $port, CloudInstance $instance): void diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index fe520dfe..9994a5b3 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -142,6 +142,8 @@ $app->group('/api/v1', function () { $this->post('/admin/operations/jobs/{id:[0-9]+}/resolve-effect', 'App\Controllers\CloudAdminOperationsController:resolveEffect'); $this->post('/admin/operations/jobs/{id:[0-9]+}/force-delete', 'App\Controllers\CloudAdminOperationsController:forceDelete'); + $this->post('/admin/operations/jobs/{id:[0-9]+}/repair-vpc', + 'App\Controllers\CloudAdminOperationsController:repairVpc'); $this->get('/admin/billing/invoices', 'App\Controllers\CloudAdminBillingController:invoices'); $this->get('/admin/billing/invoices/{id:[0-9]+}', diff --git a/web/public/cloud/admin-operations.js b/web/public/cloud/admin-operations.js index 43cecfa8..b6a7378a 100644 --- a/web/public/cloud/admin-operations.js +++ b/web/public/cloud/admin-operations.js @@ -24,6 +24,7 @@ const state = { opener: null, scroll: null, actionRequest: null, forceDeleteAccepted: false, forceDeleteBlocked: false, forceDeleteAcceptedIds: new Set(), forceDeleteBlockedIds: new Set(), forceDeleteObservers: new Map(), + vpcRepairAccepted: false, vpcRepairAcceptedIds: new Set(), inspection: null, effectResolved: false, }; @@ -394,6 +395,11 @@ function renderDetailActions() { inspect.hidden = state.detailJob?.can_inspect !== true; inspect.disabled = busy; inspect.textContent = busy && state.actionRequest?.kind === 'inspect' ? '正在检查…' : '只读检查宿主机'; + const repairVpc = byId('operation-detail-repair-vpc'); + repairVpc.hidden = !canManage() || state.detailJob?.can_repair_vpc !== true; + repairVpc.disabled = busy || state.vpcRepairAccepted; + repairVpc.textContent = state.vpcRepairAccepted ? '修复任务已接收' + : busy && state.actionRequest?.kind === 'repair-vpc' ? '正在创建…' : '创建 VPC 修复任务'; const force = byId('operation-detail-force-delete'); force.hidden = !canManage() || state.detailJob?.can_force_delete !== true; force.disabled = busy || state.forceDeleteAccepted || state.forceDeleteBlocked; @@ -583,6 +589,52 @@ async function forceDelete(id) { } } +async function repairVpc(id) { + if (!canManage() || state.detailId !== String(id) + || state.detailJob?.can_repair_vpc !== true || state.actionRequest + || state.vpcRepairAccepted) return; + if (!window.confirm('系统将先执行一次只读宿主机检查;只有原 VPC 操作仍处于影响未知且 Agent 可达时,才会创建新的修复任务。\n\n原操作不会重放,修复任务使用新的操作标识并保留继任关系。确认继续?')) return; + const controller = new AbortController(); + controller.kind = 'repair-vpc'; state.actionRequest = controller; + renderDetailActions(); detailNotice('正在检查宿主机并创建精确 VPC 修复任务…'); + try { + const payload = await acceptedTask(`/api/v1/admin/operations/jobs/${encodeURIComponent(id)}/repair-vpc`, controller, 45000, { + method: 'POST', body: JSON.stringify({ acknowledge_repair: true }), + }); + state.vpcRepairAccepted = true; + state.vpcRepairAcceptedIds.add(String(id)); + announceTaskAccepted(payload, { source: 'admin-vpc-repair' }); + const taskId = String(payload.task_id || '').trim(); + detailNotice(`VPC 修复任务已接收${taskId ? `(#${taskId})` : ''};原任务不会重放。`); + notice(`VPC 修复任务已接收${taskId ? `(#${taskId})` : ''}。`); + if (taskId) { + const handle = observeTask(payload, { + onTerminal: task => { + const succeeded = taskSucceededWithoutRecovery(task); + const message = succeeded + ? `VPC 修复任务 #${task.id} 已成功收敛。` + : taskFailureMessage(task, `VPC 修复任务 #${task.id} 失败。`); + void (async () => { + await loadPage(listUrl(), 'none'); + if (byId('operation-detail').open) await openDetail(task.id, null); + notice(message, !succeeded); + if (byId('operation-detail').open) detailNotice(message, !succeeded); + })(); + }, + }); + void handle.done; + } + } catch (error) { + if (controller.signal.aborted && controller.signal.reason !== 'timeout') return; + detailNotice(`VPC 修复任务未创建:${error.message}`, true); + notice(`VPC 修复请求未确认:${error.message}`, true); + } finally { + if (state.actionRequest === controller) { + state.actionRequest = null; renderDetailActions(); + } + } +} + function finishDetailClose(id) { state.detailClosing = false; document.documentElement.classList.remove('admin-instance-detail-open'); @@ -602,6 +654,7 @@ function closeDetail() { state.actionRequest?.abort('closed'); state.actionRequest = null; state.detailId = null; state.detailJob = null; state.forceDeleteAccepted = false; state.forceDeleteBlocked = false; + state.vpcRepairAccepted = false; state.inspection = null; state.effectResolved = false; state.detailClosing = true; void window.VenixDrawerMotion.closeDrawerWithMotion(byId('operation-detail')).then(closed => { @@ -615,11 +668,13 @@ async function openDetail(id, opener) { state.detailRequest?.abort('replaced'); state.detailId = id; state.detailJob = null; state.opener = opener; state.forceDeleteAccepted = state.forceDeleteAcceptedIds.has(String(id)); state.forceDeleteBlocked = state.forceDeleteBlockedIds.has(String(id)); + state.vpcRepairAccepted = state.vpcRepairAcceptedIds.has(String(id)); state.inspection = null; state.effectResolved = false; const controller = new AbortController(); state.detailRequest = controller; byId('operation-detail-title').textContent = `任务 #${id}`; byId('operation-detail-body').replaceChildren(); byId('operation-detail-links').replaceChildren(); byId('operation-detail-inspect').hidden = true; + byId('operation-detail-repair-vpc').hidden = true; byId('operation-detail-force-delete').hidden = true; byId('operation-effect-resolution').hidden = true; byId('operation-effect-value').value = ''; @@ -672,6 +727,9 @@ if (root) { byId('operation-detail-force-delete').addEventListener('click', event => { event.preventDefault(); if (state.detailId) void forceDelete(state.detailId); }); + byId('operation-detail-repair-vpc').addEventListener('click', event => { + event.preventDefault(); if (state.detailId) void repairVpc(state.detailId); + }); byId('operation-effect-submit').addEventListener('click', event => { event.preventDefault(); if (state.detailId) void resolveEffect(state.detailId); }); diff --git a/web/resources/views/venix/admin/cloud/operations.tpl b/web/resources/views/venix/admin/cloud/operations.tpl index bd8d6e8e..0719fece 100644 --- a/web/resources/views/venix/admin/cloud/operations.tpl +++ b/web/resources/views/venix/admin/cloud/operations.tpl @@ -39,7 +39,7 @@ 任务详情任务 正在读取任务详情…人工收敛任务级资源安全栅栏仅在已取得可审计的真实效果证据后使用。原任务仍保持失败,不会重放 Agent 或支付请求;服务端只会执行该任务类型已实现的原子账本收尾。人工核实结果请选择操作效果已经发生操作效果未发生审计证据与说明(20–500 字符)人工确认并收敛安全栅栏 - + diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index edb3cc15..50d3bf1b 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026102906' scope='global'} +{assign var=venix_asset_version value='2026102907' scope='global'} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 39df172c..60d13cbd 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -1261,6 +1261,8 @@ CREATE TABLE cloud_jobs ( request_idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, request_fingerprint CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, parent_job_id BIGINT UNSIGNED DEFAULT NULL, + supersedes_job_id BIGINT UNSIGNED DEFAULT NULL, + vpc_repair_root_job_id BIGINT UNSIGNED DEFAULT NULL, order_id BIGINT UNSIGNED DEFAULT NULL, invoice_id BIGINT UNSIGNED DEFAULT NULL, instance_id BIGINT UNSIGNED DEFAULT NULL, @@ -1397,7 +1399,9 @@ CREATE TABLE cloud_jobs ( KEY cloud_job_vpc_batch (vpc_batch_id, id), KEY cloud_job_overview_state (kind, order_id, phase, outcome, terminal_at, created_at), KEY cloud_job_owner (user_id, id), - KEY cloud_job_parent (parent_job_id, id) + KEY cloud_job_parent (parent_job_id, id), + KEY cloud_job_supersedes (supersedes_job_id, id), + KEY cloud_job_vpc_repair_root (vpc_repair_root_job_id, id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- Agent terminal receipts are immutable transport events. Retransmission of diff --git a/web/tests/AdminOperationsViewTest.php b/web/tests/AdminOperationsViewTest.php index 851020fb..0a1c2a72 100644 --- a/web/tests/AdminOperationsViewTest.php +++ b/web/tests/AdminOperationsViewTest.php @@ -158,6 +158,7 @@ $assert($xpath->query('//*[@id="operations-health-body"]//p')->length === 1 && !str_contains($html, '数据库可读'), 'SSR must not pretend to have checked runtime health.'); $assert($xpath->query('//button[contains(@id,"retry") or contains(@id,"refund") or contains(@id,"stop-auto") or contains(@id,"abort-job") or contains(@id,"cleanup")]')->length === 0 && $xpath->query('//button[@id="operation-detail-inspect" and @hidden]')->length === 1 + && $xpath->query('//button[@id="operation-detail-repair-vpc" and @hidden]')->length === 1 && $xpath->query('//button[@id="operation-detail-force-delete" and @hidden]')->length === 1 && $xpath->query('//*[@id="operation-effect-resolution" and @hidden]')->length === 1, 'The task list must expose no replay/stop mutation and keep audited fence resolution inside detail.'); diff --git a/web/tests/CloudAdminOperationsServiceTest.php b/web/tests/CloudAdminOperationsServiceTest.php index 93563109..8913559d 100644 --- a/web/tests/CloudAdminOperationsServiceTest.php +++ b/web/tests/CloudAdminOperationsServiceTest.php @@ -76,6 +76,8 @@ namespace { $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); + $table->integer('supersedes_job_id')->nullable(); + $table->integer('vpc_repair_root_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); diff --git a/web/tests/CloudFreshSchemaContractTest.php b/web/tests/CloudFreshSchemaContractTest.php index 6b046ebd..b07bc794 100644 --- a/web/tests/CloudFreshSchemaContractTest.php +++ b/web/tests/CloudFreshSchemaContractTest.php @@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false $assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema), 'cloud.sql retained an engine-prefixed identifier.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v5-fences'") + "SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v6-repair'") && !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"), 'Outbound Agent schema must reject the incompatible earlier Go database.'); $assert(str_contains($schema, @@ -64,6 +64,10 @@ $assert(str_contains($schema, 'CONSTRAINT chk_cloud_job_finalization_incident CHECK (') && str_contains($schema, 'KEY cloud_job_overview_state (kind, order_id, phase, outcome, terminal_at, created_at)') + && str_contains($schema, + 'KEY cloud_job_supersedes (supersedes_job_id, id)') + && str_contains($schema, + 'KEY cloud_job_vpc_repair_root (vpc_repair_root_job_id, id)') && str_contains($schema, 'UNIQUE KEY uniq_cloud_job_request (user_id, request_idempotency_key)') && str_contains($schema, diff --git a/web/tests/CloudVpcHostDispatchQueueTest.php b/web/tests/CloudVpcHostDispatchQueueTest.php index c761de94..6ac31884 100644 --- a/web/tests/CloudVpcHostDispatchQueueTest.php +++ b/web/tests/CloudVpcHostDispatchQueueTest.php @@ -25,6 +25,7 @@ $capsule->schema('default')->create('cloud_jobs', static function ( $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('phase'); + $table->integer('vpc_repair_root_job_id')->nullable(); $table->string('effect_resolution')->default('not_applicable'); $table->boolean('finalization_incident')->default(0); }); @@ -35,6 +36,7 @@ $insert = static function (int $id, int $hostId, string $kind): void { 'host_id' => $hostId, 'kind' => $kind, 'phase' => 'accepted', + 'vpc_repair_root_job_id' => null, 'effect_resolution' => 'not_applicable', 'finalization_incident' => 0, ]); @@ -82,6 +84,20 @@ Capsule::table('cloud_jobs')->where('id', 1)->update([ $assert($eligibleIds() === [2, 3, 4], 'Resolved route predecessor did not release exactly one host successor.'); +Capsule::table('cloud_jobs')->insert([ + ['id' => 6, 'host_id' => 20, 'kind' => 'vpc_host_apply', + 'phase' => 'terminal', 'effect_resolution' => 'unknown', + 'finalization_incident' => 0, 'vpc_repair_root_job_id' => null], + ['id' => 7, 'host_id' => 20, 'kind' => 'vpc_host_apply', + 'phase' => 'terminal', 'effect_resolution' => 'unknown', + 'finalization_incident' => 0, 'vpc_repair_root_job_id' => 6], + ['id' => 8, 'host_id' => 20, 'kind' => 'vpc_host_apply', + 'phase' => 'accepted', 'effect_resolution' => 'not_applicable', + 'finalization_incident' => 0, 'vpc_repair_root_job_id' => 6], +]); +$assert(in_array(8, $eligibleIds(), true), + 'An exact VPC repair successor could not cross its own predecessor chain.'); + $worker = file_get_contents( dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php' ); diff --git a/web/tests/CloudVpcPortWorkflowTest.php b/web/tests/CloudVpcPortWorkflowTest.php index b5ba6281..653357a4 100644 --- a/web/tests/CloudVpcPortWorkflowTest.php +++ b/web/tests/CloudVpcPortWorkflowTest.php @@ -197,6 +197,8 @@ namespace { $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); + $table->integer('supersedes_job_id')->nullable(); + $table->integer('vpc_repair_root_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); @@ -1211,6 +1213,135 @@ namespace { !CloudJobService::blockingQueryForInstance($id)->exists()), 'Retried rollback left an instance operation fence behind.'); + // A reachable but inconclusive VPC generation is repaired by a new + // operation. If the first repair also loses its result, a second exact + // repair may supersede the complete chain without rewriting history. + $repairInstance = $newInstance(40, 1); + $repairAttach = CloudVpcPortService::attach($repairInstance, [ + 'user_id' => 7, + 'request_idempotency_key' => 'vpc-repair-fixture-attach-0001', + ], [ + 'vpc_key' => (string) $vpc->vpc_key, + 'confirm_restart' => true, + ]); + $repairOriginal = CloudJob::where('kind', 'vpc_host_apply') + ->where('instance_id', 40)->where('phase', 'accepted')->first(); + $assert($repairAttach['job'] !== null && $repairOriginal !== null, + 'VPC repair fixture did not create its host generation.'); + $repairFinishedAt = time(); + $repairOriginal->phase = 'terminal'; + $repairOriginal->outcome = $repairOriginal->status = 'failed'; + $repairOriginal->attempts = $repairOriginal->dispatch_attempts = 1; + $repairOriginal->dispatch_started_at = + $repairOriginal->dispatched_at = $repairFinishedAt - 2; + $repairOriginal->agent_accepted_at = $repairFinishedAt - 1; + $repairOriginal->receipt_deadline_at = $repairFinishedAt + 900; + $repairOriginal->terminal_at = $repairFinishedAt; + $repairOriginal->effect_may_have_applied = 1; + $repairOriginal->safe_to_resubmit = 0; + $repairOriginal->effect_resolution = 'unknown'; + $repairOriginal->error_code = 'operation_effect_unknown'; + $repairOriginal->updated_at = $repairFinishedAt; + $repairOriginal->save(); + CloudVpcPortService::markJobUncertain($repairOriginal); + $repairInspection = static function (CloudJob $job): array { + return [ + 'operation_id' => (string) $job->operation_id, + 'journal_status' => 'failed', + 'checked_at' => time(), + 'outcome' => 'inconclusive', + 'effect_resolution' => 'inconclusive', + 'fence_state' => 'active', + 'fence_resolution_required' => true, + ]; + }; + $repairActor = [ + 'user_id' => 1, 'is_admin' => true, + 'is_super_admin' => true, 'can_manage' => true, + ]; + $repairOne = CloudVpcPortService::repairUnknownEffect( + $repairActor, + (int) $repairOriginal->id, + $repairInspection($repairOriginal) + ); + $repairOnePayload = CloudSecrets::decrypt( + (string) $repairOne->sealed_payload, 'job-payload' + ); + $repairState = CloudVpcHostState::where('vpc_id', 1) + ->where('host_id', 1)->first(); + $assert((int) $repairOne->supersedes_job_id + === (int) $repairOriginal->id + && (int) $repairOne->vpc_repair_root_job_id + === (int) $repairOriginal->id + && ($repairOnePayload['body']['vpc_repair'] ?? false) === true + && ($repairOnePayload['body']['supersedes_operation_id'] ?? null) + === (string) $repairOriginal->operation_id + && str_ends_with( + (string) $repairOnePayload['request_target'], '/repair' + ) + && (string) $repairState->last_operation_id + === (string) $repairOne->operation_id, + 'First VPC repair was not bound to the exact predecessor generation.'); + CloudVpcPortService::finishJobLocked( + $repairOne, + 'uncertain', + null, + $repairOnePayload, + [ + 'operation_id' => (string) $repairOne->operation_id, + 'error_code' => 'vpc_reconcile_unconfirmed', + ] + ); + $repairOne->phase = 'terminal'; + $repairOne->outcome = $repairOne->status = 'failed'; + $repairOne->attempts = $repairOne->dispatch_attempts = 1; + $repairOne->dispatch_started_at = + $repairOne->dispatched_at = time() - 2; + $repairOne->agent_accepted_at = time() - 1; + $repairOne->receipt_deadline_at = time() + 900; + $repairOne->terminal_at = time(); + $repairOne->effect_may_have_applied = 1; + $repairOne->safe_to_resubmit = 0; + $repairOne->effect_resolution = 'unknown'; + $repairOne->error_code = 'operation_effect_unknown'; + $repairOne->updated_at = time(); + $repairOne->save(); + + $repairTwo = CloudVpcPortService::repairUnknownEffect( + $repairActor, + (int) $repairOne->id, + $repairInspection($repairOne) + ); + $repairTwoPayload = CloudSecrets::decrypt( + (string) $repairTwo->sealed_payload, 'job-payload' + ); + $repairState = CloudVpcHostState::where('vpc_id', 1) + ->where('host_id', 1)->first(); + CloudVpcPortService::finishJobLocked($repairTwo, 'succeeded', [ + 'vpc_key' => (string) $vpc->vpc_key, + 'desired_revision' => (int) $repairState->desired_revision, + 'desired_digest' => (string) $repairState->desired_digest, + 'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key), + 'applied' => true, + ], $repairTwoPayload, [ + 'operation_id' => (string) $repairTwo->operation_id, + ]); + $terminalize($repairTwo); + $assert((int) $repairTwo->supersedes_job_id === (int) $repairOne->id + && (int) $repairTwo->vpc_repair_root_job_id + === (int) $repairOriginal->id + && (string) $repairOriginal->fresh()->effect_resolution === 'applied' + && str_starts_with( + (string) $repairOriginal->fresh()->effect_resolution_reason, + 'superseded_by_vpc_repair:' + ) + && (string) $repairOne->fresh()->effect_resolution === 'applied' + && (string) CloudVpcHostState::where('vpc_id', 1) + ->where('host_id', 1)->value('sync_state') === 'ready' + && CloudJob::where('kind', 'vpc_port_attach') + ->where('instance_id', 40)->where('phase', 'accepted')->exists(), + 'A successful chained VPC repair did not converge every fence and ledger.'); + $worker = file_get_contents( dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php' ); diff --git a/web/tests/CloudVpcSchemaContractTest.php b/web/tests/CloudVpcSchemaContractTest.php index e3ec7e9a..1b3a93c5 100644 --- a/web/tests/CloudVpcSchemaContractTest.php +++ b/web/tests/CloudVpcSchemaContractTest.php @@ -88,17 +88,26 @@ $jobs = $jobsStart === false || $jobsEnd === false ? '' $assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL') && str_contains($jobs, 'vpc_port_id BIGINT UNSIGNED DEFAULT NULL') && str_contains($jobs, 'vpc_batch_id BIGINT UNSIGNED DEFAULT NULL') + && str_contains($jobs, 'supersedes_job_id BIGINT UNSIGNED DEFAULT NULL') + && str_contains($jobs, + 'vpc_repair_root_job_id BIGINT UNSIGNED DEFAULT NULL') && str_contains($jobs, 'KEY cloud_job_vpc (vpc_id, id)') && str_contains($jobs, 'KEY cloud_job_vpc_port (vpc_port_id, id)') - && str_contains($jobs, 'KEY cloud_job_vpc_batch (vpc_batch_id, id)'), + && str_contains($jobs, 'KEY cloud_job_vpc_batch (vpc_batch_id, id)') + && str_contains($jobs, + 'KEY cloud_job_supersedes (supersedes_job_id, id)') + && str_contains($jobs, + 'KEY cloud_job_vpc_repair_root (vpc_repair_root_job_id, id)'), 'Cloud jobs cannot be correlated to VPC or VPC-port work.'); $assert(str_contains($setup, - "private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v5-fences'") + "private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v6-repair'") && str_contains($setup, "'name' => 'cloud_vpc_enabled'") && str_contains($setup, "'value' => 'false'") && str_contains($setup, - "'cloud_job_vpc', 'cloud_job_vpc_port', 'cloud_job_vpc_batch'") + "'cloud_job_vpc', 'cloud_job_vpc_port', 'cloud_job_vpc_batch',") + && str_contains($setup, + "'cloud_job_supersedes', 'cloud_job_vpc_repair_root'") && str_contains($setup, "'chk_cloud_vpc_port_release'"), 'cloudInit does not publish and validate the disabled VPC schema.'); @@ -130,7 +139,9 @@ foreach ($models as $class => $table) { $jobModel = file_get_contents($root . '/app/Models/CloudJob.php'); $assert(str_contains($jobModel, "'vpc_id' => 'int'") && str_contains($jobModel, "'vpc_port_id' => 'int'") - && str_contains($jobModel, "'vpc_batch_id' => 'int'"), + && str_contains($jobModel, "'vpc_batch_id' => 'int'") + && str_contains($jobModel, "'supersedes_job_id' => 'int'") + && str_contains($jobModel, "'vpc_repair_root_job_id' => 'int'"), 'CloudJob does not cast VPC correlations.'); echo "VPC schema, model and feature-switch contracts passed.\n";
任务详情
正在读取任务详情…
仅在已取得可审计的真实效果证据后使用。原任务仍保持失败,不会重放 Agent 或支付请求;服务端只会执行该任务类型已实现的原子账本收尾。