From 6f4ee55a693681837cd27d26680f9fd02b2d274a Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 28 Oct 2026 03:36:13 +0800 Subject: [PATCH] fix: reconcile peer migration target residue --- agent/api/server.go | 147 ++++++++++- agent/api/server_test.go | 182 ++++++++++++- agent/cmd/virt-agent/main.go | 11 +- agent/hypervisor/admission.go | 11 + agent/hypervisor/libvirt_connection.go | 6 + agent/hypervisor/libvirt_operation_admin.go | 28 +- agent/hypervisor/libvirt_peer_migration.go | 85 ++++++ agent/hypervisor/peer_migration.go | 11 + agent/hypervisor/types.go | 16 ++ agent/operation/journal.go | 32 ++- agent/operation/journal_test.go | 46 ++++ agent/operation/recovery_contract.go | 7 +- agent/operation/recovery_contract.json | 13 + agent/worker/protocol.go | 16 ++ .../CloudAdminOperationRecoveryService.php | 9 +- ...CloudAutomaticOperationRecoveryService.php | 8 + .../Cloud/CloudInstanceMigrationService.php | 150 +++++++++++ web/app/Services/Cloud/CloudJobService.php | 2 + .../Services/Cloud/CloudOperationCatalog.php | 2 + .../Cloud/CloudOperationErrorCatalog.php | 1 + .../Cloud/CloudOperationRecoveryContract.php | 7 + .../Cloud/CloudOperationRecoveryIdentity.php | 43 +++ .../CloudOperationRecoveryMatrixTest.php | 248 ++++++++++++++++++ .../CloudOperationRecoveryPolicyTest.php | 5 +- web/tests/CloudSnapshotWorkflowTest.php | 3 +- 25 files changed, 1067 insertions(+), 22 deletions(-) diff --git a/agent/api/server.go b/agent/api/server.go index edced5ed..f9c79430 100644 --- a/agent/api/server.go +++ b/agent/api/server.go @@ -66,6 +66,7 @@ type Server struct { } type trustedWorkerRequestKey struct{} +type trustedSupersededOperationKey struct{} // SetMutationExecutor enables the clean-break process-isolated mutation path. // It must be called during startup before request admission begins. @@ -81,7 +82,8 @@ func (s *Server) SetMutationExecutor(executor worker.Executor) error { // already authenticated and validated by the parent Agent. Hidden workers do // not have access to the panel secret through HTTP headers and cannot recurse // because they never install a MutationExecutor. -func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor) { +func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor, + superseded ...operation.Record) { setResponseHeaders(w) if !s.beginRequest() { writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down") @@ -93,9 +95,18 @@ func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Reques return } trusted := context.WithValue(r.Context(), trustedWorkerRequestKey{}, struct{}{}) + if len(superseded) > 0 { + prior := superseded[0] + trusted = context.WithValue(trusted, trustedSupersededOperationKey{}, prior) + } s.route(w, r.WithContext(trusted), body, actor) } +func trustedSupersededOperation(ctx context.Context, operationID string) (operation.Record, bool) { + prior, ok := ctx.Value(trustedSupersededOperationKey{}).(operation.Record) + return prior, ok && prior.OperationID == operationID +} + type errorBody struct { Code string `json:"code"` Message string `json:"message"` @@ -610,6 +621,88 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str func(ctx context.Context) (any, error) { return s.Hypervisor.CleanupPeerMigration(ctx, uuid, request) }) + case "cleanup-target": + if r.Method != http.MethodPost { + methodOrNotFound(w, r, true, http.MethodPost) + return + } + request, ok := decodeRequired[hypervisor.CleanupPeerMigrationTargetRequest](w, body) + if !ok { + return + } + operationID, err := resolveOperationID(actor.OperationID, request.OperationID) + if err != nil { + writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error()) + return + } + request.OperationID = operationID + if err := hypervisor.ValidateCleanupPeerMigrationTargetRequest(request); err != nil { + writePreDispatchMappedError(w, err) + return + } + action := "cleanup-peer-migration-target:" + request.MigrationID + existingReplay := false + trustedPrior, trustedPriorFound := trustedSupersededOperation( + r.Context(), request.PriorOperationID, + ) + existing, existingErr := s.Journal.Get(operationID) + if existingErr == nil { + existingReplay = existing.Action == action && existing.ResourceID == uuid && + existing.SupersedesOperationID == request.PriorOperationID + } else if !errors.Is(existingErr, operation.ErrNotFound) { + writeMappedError(w, existingErr, nil) + return + } + if !existingReplay { + prior := trustedPrior + var priorErr error + if !trustedPriorFound { + prior, priorErr = s.Journal.Get(request.PriorOperationID) + } + if priorErr != nil || prior.ResourceID != uuid || + prior.Action != "receive-peer-migration:"+request.MigrationID || + prior.State != operation.StateFailed || !prior.EffectMayHaveApplied || + prior.SafeToResubmit || prior.ExecutionFenced { + writePreDispatchError(w, http.StatusConflict, "peer_migration_target_cleanup_not_authorized", "target cleanup requires the exact safely reaped failed receive operation") + return + } + if !trustedPriorFound { + _, incidentFound, incidentErr := s.Journal.CompletionDeliveryIncident(prior.OperationID) + if incidentErr != nil { + writeMappedError(w, incidentErr, nil) + return + } + if incidentFound { + writePreDispatchError(w, http.StatusConflict, "peer_migration_target_cleanup_not_authorized", "target cleanup is blocked by a completion-delivery incident") + return + } + } + ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) + inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, prior) + cancel() + if inspectErr != nil { + writeMappedError(w, inspectErr, nil) + return + } + if !peerMigrationTargetCleanupInspectionMatches(prior, request.CleanupPlanID, inspection) { + writePreDispatchError(w, http.StatusConflict, "peer_migration_cleanup_plan_changed", "target cleanup plan is stale or no longer safe") + return + } + request.PriorOperation = prior + } + supersedes := []string{request.PriorOperationID} + if trustedPriorFound { + // The isolated worker has its own Journal. The parent Journal + // already admitted this exact supersession and retains the audit + // link; the child receives the immutable predecessor separately. + supersedes = nil + } + s.runMutationExpected(w, r, body, actor, operationID, uuid, action, nil, + true, http.StatusOK, + hypervisor.CleanupPeerMigrationTargetAdmission(uuid, request.MigrationID), + func(ctx context.Context) (any, error) { + return s.Hypervisor.CleanupPeerMigrationTarget(ctx, uuid, request) + }, supersedes...) default: writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } @@ -645,6 +738,9 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str return } existingReplay := false + trustedPrior, trustedPriorFound := trustedSupersededOperation( + r.Context(), request.PriorOperationID, + ) existing, existingErr := s.Journal.Get(operationID) if existingErr == nil { existingReplay = existing.Action == "force-delete-vm" && existing.ResourceID == uuid && @@ -654,7 +750,11 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str return } if !existingReplay { - prior, priorErr := s.Journal.Get(request.PriorOperationID) + prior := trustedPrior + var priorErr error + if !trustedPriorFound { + prior, priorErr = s.Journal.Get(request.PriorOperationID) + } priorActionAllowed := priorErr == nil && (prior.Action == "delete-vm" || prior.Action == "service-suspend") if !priorActionAllowed || prior.ResourceID != uuid || prior.State != operation.StateFailed || prior.Error == nil || prior.Error.Code != "force_required" { @@ -662,11 +762,18 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str return } } + supersedes := []string{request.PriorOperationID} + if trustedPriorFound { + // The parent Journal already authorized and records the force + // successor. The isolated worker Journal receives the immutable + // predecessor separately and must not try to load it locally. + supersedes = nil + } s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "force-delete-vm", nil, true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid), func(ctx context.Context) (any, error) { return s.Hypervisor.ForceDeleteVM(ctx, uuid, request) - }, request.PriorOperationID) + }, supersedes...) return } case "state": @@ -943,6 +1050,32 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found") } +func peerMigrationTargetCleanupInspectionMatches(prior operation.Record, planID string, + inspection hypervisor.OperationInspection) bool { + if inspection.OperationID != prior.OperationID || inspection.Action != prior.Action || + inspection.ResourceID != prior.ResourceID || inspection.RequestFingerprint != prior.Fingerprint || + inspection.OperationStatus != prior.State || inspection.CheckedAt <= 0 || + inspection.SummaryCode != hypervisor.InspectionSummaryCleanupRequired || + inspection.EffectResolution != hypervisor.InspectionEffectPartial || + inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" || + inspection.AbortJob.Available || !inspection.Cleanup.Available || + !strings.EqualFold(inspection.Cleanup.PlanID, planID) || len(inspection.Cleanup.Steps) == 0 { + return false + } + for _, step := range inspection.Cleanup.Steps { + switch step { + case hypervisor.CleanupStepDeleteManagedVolumes, + hypervisor.CleanupStepDeleteSeedVolume, + hypervisor.CleanupStepUndefineNetworkFilter, + hypervisor.CleanupStepDeleteNVRAM, + hypervisor.CleanupStepRemoveTrafficState: + default: + return false + } + } + return true +} + func (s *Server) routeSnapshots(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) { if len(segments) == 4 { switch r.Method { @@ -1746,6 +1879,14 @@ func (s *Server) runIsolatedMutation( RequestTarget: auth.RequestTarget(r), Body: string(body), Actor: actor, OperationID: spec.OperationID, Fingerprint: spec.Fingerprint, } + if spec.SupersedesOperationID != "" { + prior, err := s.Journal.Get(spec.SupersedesOperationID) + if err != nil { + writeMappedError(w, err, nil) + return + } + request.SupersededOperation = &prior + } dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout) record, replayed, done, err := s.Journal.ExecuteBackground( dispatchCtx, s.backgroundCtx, deadline, spec, diff --git a/agent/api/server_test.go b/agent/api/server_test.go index 3a85c878..b313ae00 100644 --- a/agent/api/server_test.go +++ b/agent/api/server_test.go @@ -28,11 +28,13 @@ import ( type testHypervisor struct { hypervisor.Unsupported - mu sync.Mutex - inspections int - configureCalls int - inspectionResult *hypervisor.OperationInspection - inspectionErr error + mu sync.Mutex + inspections int + configureCalls int + cleanupPeerTargetCalls int + cleanupPeerTargetLast hypervisor.CleanupPeerMigrationTargetRequest + inspectionResult *hypervisor.OperationInspection + inspectionErr error } func (h *testHypervisor) Ping(context.Context) error { return nil } @@ -48,6 +50,17 @@ func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageCon return hypervisor.StorageConfigurationResult{Configured: true}, nil } +func (h *testHypervisor) CleanupPeerMigrationTarget(_ context.Context, _ string, + request hypervisor.CleanupPeerMigrationTargetRequest) (hypervisor.DeleteVMResult, error) { + h.mu.Lock() + h.cleanupPeerTargetCalls++ + h.cleanupPeerTargetLast = request + h.mu.Unlock() + return hypervisor.DeleteVMResult{ + Deleted: true, ManagedArtifactsAbsent: true, Forced: false, + }, nil +} + func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) { h.mu.Lock() defer h.mu.Unlock() @@ -526,6 +539,7 @@ func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing "change-network", "network-suspend", "network-resume", "apply-firewall", "delete-vm", "force-delete-vm", "service-suspend", + "cleanup-peer-migration-target:0123456789abcdef0123456789abcdef", "create-snapshot:0123456789abcdef0123456789abcdef:filesystem", "create-snapshot:0123456789abcdef0123456789abcdef:crash", "delete-snapshot:0123456789abcdef0123456789abcdef", @@ -917,6 +931,164 @@ func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) { } } +func TestPeerMigrationTargetCleanupRequiresFreshExactPlanAndRetainsReceiveFence(t *testing.T) { + api := newTestAPI(t) + const ( + uuid = "123e4567-e89b-42d3-a456-426614174000" + migrationID = "0123456789abcdef0123456789abcdef" + priorID = "operation-peer-receive-residue" + cleanupID = "operation-peer-target-cleanup" + ) + prior := operation.Spec{ + OperationID: priorID, ResourceID: uuid, + Action: "receive-peer-migration:" + migrationID, + Fingerprint: operation.Fingerprint(http.MethodPost, + "/v1/vms/"+uuid+"/peer-migration/receive", []byte(`{}`)), + Destructive: true, + LockKeys: hypervisor.ReceivePeerMigrationAdmission(hypervisor.ReceivePeerMigrationRequest{ + CreateVMRequest: hypervisor.CreateVMRequest{UUID: uuid}, MigrationID: migrationID, + }).LockKeys, + } + if _, _, err := api.journal.Execute(context.Background(), prior, func(context.Context) (json.RawMessage, error) { + return nil, &worker.Error{Code: "result_unconfirmed", Message: "receive result unknown", EffectMayHaveApplied: true} + }); err == nil { + t.Fatal("seed receive failure unexpectedly succeeded") + } + stored, err := api.journal.Get(priorID) + if err != nil { + t.Fatal(err) + } + planID := strings.Repeat("a", 64) + api.hv.inspectionResult = &hypervisor.OperationInspection{ + OperationID: stored.OperationID, Action: stored.Action, + ResourceID: stored.ResourceID, RequestFingerprint: stored.Fingerprint, + OperationStatus: stored.State, CheckedAt: api.now.Unix(), + Outcome: hypervisor.InspectionOutcomeUncertain, + SummaryCode: hypervisor.InspectionSummaryCleanupRequired, + EffectResolution: hypervisor.InspectionEffectPartial, + Cleanup: hypervisor.OperationCleanupPlan{Available: true, PlanID: planID, + Steps: []string{hypervisor.CleanupStepDeleteManagedVolumes}}, + } + executor := &fakeMutationExecutor{result: json.RawMessage(`{"deleted":true,"managed_artifacts_absent":true,"forced":false}`)} + if err := api.handler.SetMutationExecutor(executor); err != nil { + t.Fatal(err) + } + request := hypervisor.CleanupPeerMigrationTargetRequest{ + OperationID: cleanupID, MigrationID: migrationID, + PriorOperationID: priorID, CleanupPlanID: planID, + } + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: cleanupID} + response := api.request(t, http.MethodPost, + "/v1/vms/"+uuid+"/peer-migration/cleanup-target", body, actor) + if response.Code != http.StatusAccepted { + t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String()) + } + terminal := waitForTerminal(t, api.journal, cleanupID) + if terminal.State != operation.StateSucceeded || terminal.SupersedesOperationID != priorID { + t.Fatalf("cleanup terminal = %#v", terminal) + } + executor.mu.Lock() + if len(executor.requests) != 1 || executor.requests[0].SupersededOperation == nil || + executor.requests[0].SupersededOperation.OperationID != priorID { + executor.mu.Unlock() + t.Fatalf("cleanup worker predecessor = %#v", executor.requests) + } + executor.mu.Unlock() + if state, err := api.journal.FenceState(priorID); err != nil || state != operation.FenceStateActive { + t.Fatalf("receive fence state=%q err=%v", state, err) + } + + stale := request + stale.OperationID = "operation-peer-target-cleanup-stale" + stale.CleanupPlanID = strings.Repeat("b", 64) + staleBody, _ := json.Marshal(stale) + staleActor := actor + staleActor.OperationID = stale.OperationID + denied := api.request(t, http.MethodPost, + "/v1/vms/"+uuid+"/peer-migration/cleanup-target", staleBody, staleActor) + if denied.Code != http.StatusConflict || + !bytes.Contains(denied.Body.Bytes(), []byte(`"code":"peer_migration_cleanup_plan_changed"`)) { + t.Fatalf("stale cleanup = %d %s", denied.Code, denied.Body.String()) + } +} + +func TestTrustedWorkerPeerMigrationTargetCleanupUsesParentPredecessor(t *testing.T) { + parent := newTestAPI(t) + child := newTestAPI(t) + const ( + uuid = "123e4567-e89b-42d3-a456-426614174000" + migrationID = "0123456789abcdef0123456789abcdef" + priorID = "operation-peer-receive-worker-residue" + cleanupID = "operation-peer-target-worker-cleanup" + ) + priorSpec := operation.Spec{ + OperationID: priorID, ResourceID: uuid, + Action: "receive-peer-migration:" + migrationID, + Fingerprint: operation.Fingerprint(http.MethodPost, + "/v1/vms/"+uuid+"/peer-migration/receive", []byte(`{}`)), + Destructive: true, + LockKeys: hypervisor.ReceivePeerMigrationAdmission(hypervisor.ReceivePeerMigrationRequest{ + CreateVMRequest: hypervisor.CreateVMRequest{UUID: uuid}, MigrationID: migrationID, + }).LockKeys, + } + if _, _, err := parent.journal.Execute(context.Background(), priorSpec, func(context.Context) (json.RawMessage, error) { + return nil, &worker.Error{Code: "result_unconfirmed", Message: "receive result unknown", EffectMayHaveApplied: true} + }); err == nil { + t.Fatal("seed receive failure unexpectedly succeeded") + } + prior, err := parent.journal.Get(priorID) + if err != nil { + t.Fatal(err) + } + planID := strings.Repeat("c", 64) + child.hv.inspectionResult = &hypervisor.OperationInspection{ + OperationID: prior.OperationID, Action: prior.Action, + ResourceID: prior.ResourceID, RequestFingerprint: prior.Fingerprint, + OperationStatus: prior.State, CheckedAt: child.now.Unix(), + Outcome: hypervisor.InspectionOutcomeUncertain, + SummaryCode: hypervisor.InspectionSummaryCleanupRequired, + EffectResolution: hypervisor.InspectionEffectPartial, + Cleanup: hypervisor.OperationCleanupPlan{Available: true, PlanID: planID, + Steps: []string{hypervisor.CleanupStepDeleteManagedVolumes}}, + } + requestBody, err := json.Marshal(hypervisor.CleanupPeerMigrationTargetRequest{ + OperationID: cleanupID, MigrationID: migrationID, + PriorOperationID: priorID, CleanupPlanID: planID, + }) + if err != nil { + t.Fatal(err) + } + actor := auth.Actor{ + UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, + OperationID: cleanupID, + } + request := httptest.NewRequest(http.MethodPost, + "/v1/vms/"+uuid+"/peer-migration/cleanup-target", bytes.NewReader(requestBody)) + response := httptest.NewRecorder() + child.handler.ServeTrustedWorkerRequest(response, request, requestBody, actor, prior) + if response.Code != http.StatusOK { + t.Fatalf("trusted cleanup response = %d %s", response.Code, response.Body.String()) + } + terminal, err := child.journal.Get(cleanupID) + if err != nil { + t.Fatal(err) + } + if terminal.State != operation.StateSucceeded || terminal.SupersedesOperationID != "" { + t.Fatalf("trusted cleanup terminal = %#v", terminal) + } + child.hv.mu.Lock() + defer child.hv.mu.Unlock() + if child.hv.cleanupPeerTargetCalls != 1 || + child.hv.cleanupPeerTargetLast.PriorOperation.OperationID != priorID { + t.Fatalf("trusted cleanup calls=%d request=%#v", + child.hv.cleanupPeerTargetCalls, child.hv.cleanupPeerTargetLast) + } +} + func TestCreateBackupUsesDedicatedAsyncActionAndHeavyAdmission(t *testing.T) { api := newTestAPI(t) const ( diff --git a/agent/cmd/virt-agent/main.go b/agent/cmd/virt-agent/main.go index 1c6ffa5a..8ec7c3fa 100644 --- a/agent/cmd/virt-agent/main.go +++ b/agent/cmd/virt-agent/main.go @@ -457,7 +457,16 @@ func runOperationWorker(configPath, journalPath, receiptPath string) error { ContentLength: int64(len(request.Body)), Host: "agent.worker", }).WithContext(ctx) responseCapture := newBoundedWorkerResponse(worker.MaxWorkerResponse) - handler.ServeTrustedWorkerRequest(responseCapture, httpRequest, []byte(request.Body), request.Actor) + if request.SupersededOperation != nil { + handler.ServeTrustedWorkerRequest( + responseCapture, httpRequest, []byte(request.Body), request.Actor, + *request.SupersededOperation, + ) + } else { + handler.ServeTrustedWorkerRequest( + responseCapture, httpRequest, []byte(request.Body), request.Actor, + ) + } response := waitForWorkerTerminal(ctx, journal, request, responseCapture) if err := worker.WriteReceipt(receiptPath, response); err != nil { diff --git a/agent/hypervisor/admission.go b/agent/hypervisor/admission.go index e9a61ac9..88800b73 100644 --- a/agent/hypervisor/admission.go +++ b/agent/hypervisor/admission.go @@ -172,6 +172,17 @@ func ReceivePeerMigrationAdmission(request ReceivePeerMigrationRequest) Mutation }} } +func CleanupPeerMigrationTargetAdmission(uuid, migrationID string) MutationAdmission { + uuid = strings.ToLower(uuid) + return MutationAdmission{HeavyIO: true, LockKeys: []string{ + domainLockKey(uuid), vmManagedVolumesLockKey(uuid), + volumeLockKey(managedStoragePoolName, rootVolumeName(uuid)), + volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)), + nwfilterLockKey(networkFilterName(uuid)), + catalogLockKey("peer-migration", migrationID), + }} +} + func DeleteSnapshotAdmission(uuid, snapshotKey string) MutationAdmission { uuid = strings.ToLower(uuid) return MutationAdmission{LockKeys: []string{ diff --git a/agent/hypervisor/libvirt_connection.go b/agent/hypervisor/libvirt_connection.go index 432451ef..a7b47aba 100644 --- a/agent/hypervisor/libvirt_connection.go +++ b/agent/hypervisor/libvirt_connection.go @@ -602,6 +602,12 @@ func (m *managedLibvirt) CleanupPeerMigration(ctx context.Context, uuid string, }) } +func (m *managedLibvirt) CleanupPeerMigrationTarget(ctx context.Context, uuid string, request CleanupPeerMigrationTargetRequest) (DeleteVMResult, error) { + return managedCall(m, ctx, true, func(h Hypervisor) (DeleteVMResult, error) { + return h.CleanupPeerMigrationTarget(ctx, uuid, request) + }) +} + func (m *managedLibvirt) DeleteVM(ctx context.Context, uuid string, request DeleteVMRequest) (DeleteVMResult, error) { return managedCall(m, ctx, true, func(h Hypervisor) (DeleteVMResult, error) { return h.DeleteVM(ctx, uuid, request) diff --git a/agent/hypervisor/libvirt_operation_admin.go b/agent/hypervisor/libvirt_operation_admin.go index 2cc83ed4..43cd9fa1 100644 --- a/agent/hypervisor/libvirt_operation_admin.go +++ b/agent/hypervisor/libvirt_operation_admin.go @@ -169,13 +169,24 @@ func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record opera var resolution operation.Resolution var cleanup operationCleanupInspection switch { + case strings.HasPrefix(record.Action, "receive-peer-migration:"): + plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil) + if err != nil { + return OperationInspection{}, err + } + if len(plan.steps) != 0 { + cleanup = plan + break + } + resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, true, false) case operation.IsCreateLikeAction(record.Action): absent, err := l.deleteArtifactsAbsent(record.ResourceID) if err != nil { return OperationInspection{}, err } resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, absent, l.traffic.contains(record.ResourceID)) - case record.Action == "delete-vm" || record.Action == "force-delete-vm": + case record.Action == "delete-vm" || record.Action == "force-delete-vm" || + strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"): plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil) if err != nil { return OperationInspection{}, err @@ -210,6 +221,12 @@ func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePres func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection, error) { + if strings.HasPrefix(record.Action, "cleanup-peer-migration-target:") { + // This successor is authorized only to clean an absent target. A domain + // appearing before or after dispatch is never converted into a delete + // plan by reconciliation. + return operation.Resolution{}, operationCleanupInspection{}, nil + } if record.Action == "delete-vm" || record.Action == "force-delete-vm" { _, managed, err := requireManagedDomain(domain) if err != nil { @@ -1596,6 +1613,15 @@ func operationEffectProjection(record operation.Record, raw []byte) json.RawMess return nil } projection = value + case strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"): + parts := strings.Split(record.Action, ":") + var value DeleteVMResult + if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) || + !decodeOperationEffectResult(raw, &value) || !value.Deleted || + !value.ManagedArtifactsAbsent || value.Forced { + return nil + } + projection = value case strings.HasPrefix(record.Action, "activate-peer-migration:") || strings.HasPrefix(record.Action, "rollback-peer-migration:"): parts := strings.Split(record.Action, ":") diff --git a/agent/hypervisor/libvirt_peer_migration.go b/agent/hypervisor/libvirt_peer_migration.go index 4e1133ab..a826bc79 100644 --- a/agent/hypervisor/libvirt_peer_migration.go +++ b/agent/hypervisor/libvirt_peer_migration.go @@ -547,6 +547,91 @@ func (l *Libvirt) CleanupPeerMigration(ctx context.Context, uuid string, return result, nil } +// CleanupPeerMigrationTarget removes only UUID-owned artifacts left by one +// failed target receive. It never destroys or undefines a domain. The exact +// inventory is inspected twice: once by the API before Journal admission and +// again here under the successor's resource leases, closing the TOCTOU gap. +func (l *Libvirt) CleanupPeerMigrationTarget(ctx context.Context, uuid string, + request CleanupPeerMigrationTargetRequest) (_ DeleteVMResult, resultErr error) { + mutationStarted := false + defer func() { + if !mutationStarted { + resultErr = operation.MarkNotApplied(resultErr) + } + }() + if err := ValidateCleanupPeerMigrationTargetRequest(request); err != nil { + return DeleteVMResult{}, err + } + prior := request.PriorOperation + expectedAction := "receive-peer-migration:" + request.MigrationID + if prior.OperationID != request.PriorOperationID || prior.ResourceID != uuid || + prior.Action != expectedAction || prior.State != operation.StateFailed || + !prior.EffectMayHaveApplied || prior.SafeToResubmit || prior.ExecutionFenced { + return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_cleanup_not_authorized", "peer migration target cleanup is not authorized by the prior receive", nil) + } + domain, err := l.domainByUUID(ctx, uuid) + if err == nil { + _ = domain.Free() + return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_domain_present", "peer migration target cleanup refuses to remove an existing domain", nil) + } + if !errors.Is(err, ErrNotFound) { + return DeleteVMResult{}, err + } + plan, err := l.inspectDeleteCleanupSteps(ctx, uuid, nil, nil) + if err != nil { + return DeleteVMResult{}, err + } + if len(plan.steps) == 0 || len(plan.evidence) == 0 || + !hashEqualsFold(request.CleanupPlanID, + operationInspectionToken(prior, "cleanup", plan.tokenEvidence())) { + return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_changed", "peer migration target cleanup plan changed; inspect again", nil) + } + for _, step := range plan.steps { + switch step { + case CleanupStepDeleteManagedVolumes, CleanupStepDeleteSeedVolume, + CleanupStepUndefineNetworkFilter, CleanupStepDeleteNVRAM, + CleanupStepRemoveTrafficState: + default: + return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_invalid", "peer migration target cleanup plan contains an unsafe step", nil) + } + } + mutationStarted = true + completionCtx := context.WithoutCancel(ctx) + if err := l.cleanupDeletedVMArtifacts(completionCtx, uuid); err != nil { + return DeleteVMResult{}, err + } + store, err := peermigration.NewStore( + l.local.PeerMigrationStatePath, l.local.StoragePathRoots, + l.local.PeerMigrationStagingPath, + ) + if err != nil { + return DeleteVMResult{}, operation.MarkUncertain(NewError( + ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target state cleanup is incomplete", err, + )) + } + if err := store.RemoveTarget(request.MigrationID); err != nil { + return DeleteVMResult{}, operation.MarkUncertain(NewError( + ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target staging cleanup is incomplete", err, + )) + } + if err := store.RemoveTargetReceipt(request.MigrationID); err != nil { + return DeleteVMResult{}, operation.MarkUncertain(NewError( + ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target receipt cleanup is incomplete", err, + )) + } + absent, err := l.deleteArtifactsAbsent(uuid) + if err != nil || !absent { + return DeleteVMResult{}, operation.MarkUncertain(errors.Join(err, + NewError(ErrConflict, "peer_migration_target_cleanup_unconfirmed", "peer migration target artifacts are still present", nil), + )) + } + return DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true}, nil +} + +func hashEqualsFold(left, right string) bool { + return len(left) == len(right) && strings.EqualFold(left, right) +} + func (l *Libvirt) createPeerMigrationVolume(ctx context.Context, _ string, volumeName, path string, artifact peermigration.Artifact, finalCapacity uint64) (*libvirt.StorageVol, error) { if finalCapacity < artifact.VirtualSizeBytes { diff --git a/agent/hypervisor/peer_migration.go b/agent/hypervisor/peer_migration.go index e7eebdc3..a620c9bd 100644 --- a/agent/hypervisor/peer_migration.go +++ b/agent/hypervisor/peer_migration.go @@ -104,3 +104,14 @@ func ValidateCleanupPeerMigrationRequest(request CleanupPeerMigrationRequest) er } return nil } + +func ValidateCleanupPeerMigrationTargetRequest(request CleanupPeerMigrationTargetRequest) error { + if !operationIdentifierPattern.MatchString(request.OperationID) || + !operationIdentifierPattern.MatchString(request.PriorOperationID) || + request.OperationID == request.PriorOperationID || + !catalogIDPattern.MatchString(request.MigrationID) || + !sha256Pattern.MatchString(strings.ToLower(request.CleanupPlanID)) { + return NewError(ErrInvalid, "invalid_peer_migration_target_cleanup", "peer migration target cleanup identity is invalid", nil) + } + return nil +} diff --git a/agent/hypervisor/types.go b/agent/hypervisor/types.go index 95fd3ee6..8f801ce1 100644 --- a/agent/hypervisor/types.go +++ b/agent/hypervisor/types.go @@ -329,6 +329,18 @@ type CleanupPeerMigrationRequest struct { RemoveManagedDisks bool `json:"remove_managed_disks"` } +// CleanupPeerMigrationTargetRequest authorizes one narrowly scoped cleanup of +// an interrupted target receive. PriorOperation is injected by the Agent API +// from its immutable Journal and is never accepted from the wire. PlanID binds +// the mutation to one fresh, read-only inventory of UUID-owned artifacts. +type CleanupPeerMigrationTargetRequest struct { + OperationID string `json:"operation_id,omitempty"` + MigrationID string `json:"migration_id"` + PriorOperationID string `json:"prior_operation_id"` + CleanupPlanID string `json:"cleanup_plan_id"` + PriorOperation operation.Record `json:"-"` +} + // RecoverBackupRequest replaces the managed root disk of an existing VM with // one immutable portable-backup object. Network and domain identity remain // attached to the existing VM, while the Agent reduces its NoCloud seed to @@ -823,6 +835,7 @@ type Hypervisor interface { ActivatePeerMigration(context.Context, string, ActivatePeerMigrationRequest) (PeerMigrationStateResult, error) RollbackPeerMigration(context.Context, string, RollbackPeerMigrationRequest) (PeerMigrationStateResult, error) CleanupPeerMigration(context.Context, string, CleanupPeerMigrationRequest) (DeleteVMResult, error) + CleanupPeerMigrationTarget(context.Context, string, CleanupPeerMigrationTargetRequest) (DeleteVMResult, error) DeleteVM(context.Context, string, DeleteVMRequest) (DeleteVMResult, error) ForceDeleteVM(context.Context, string, ForceDeleteVMRequest) (DeleteVMResult, error) Power(context.Context, string, PowerRequest) (VMState, error) @@ -911,6 +924,9 @@ func (Unsupported) RollbackPeerMigration(context.Context, string, RollbackPeerMi func (Unsupported) CleanupPeerMigration(context.Context, string, CleanupPeerMigrationRequest) (DeleteVMResult, error) { return DeleteVMResult{}, unsupported("peer migration cleanup") } +func (Unsupported) CleanupPeerMigrationTarget(context.Context, string, CleanupPeerMigrationTargetRequest) (DeleteVMResult, error) { + return DeleteVMResult{}, unsupported("peer migration target cleanup") +} func (Unsupported) DeleteVM(context.Context, string, DeleteVMRequest) (DeleteVMResult, error) { return DeleteVMResult{}, unsupported("VM delete") } diff --git a/agent/operation/journal.go b/agent/operation/journal.go index 3370cb08..6e94ebe1 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -160,8 +160,9 @@ type Record struct { // they are never exposed by the Agent API. LockKeys []string `json:"-"` HeavyIO bool `json:"-"` - // SupersedesOperationID is only valid for an explicitly acknowledged - // force-delete-vm successor of one matching graceful/force_required record. + // SupersedesOperationID is valid only for an explicitly reviewed successor: + // force-delete-vm after force_required, or a plan-bound PeerMigration target + // cleanup after one exact failed receive operation. SupersedesOperationID string `json:"-"` // InspectionReceiptRecovered and InspectionEvidenceUnavailable are // ephemeral read-path markers. They are never persisted or exposed by the @@ -2655,6 +2656,27 @@ func validForceDeleteSupersession(spec Spec, lockKeys []string, predecessor Reco return lockKeysOverlap(lockKeys, predecessorKeys) } +func validPeerMigrationTargetCleanupSupersession(spec Spec, lockKeys []string, predecessor Record) bool { + if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID || + !cleanupPeerTargetActionPattern.MatchString(spec.Action) || + predecessor.OperationID != spec.SupersedesOperationID || + predecessor.ResourceID != spec.ResourceID || + predecessor.State != StateFailed || !predecessor.EffectMayHaveApplied || + predecessor.SafeToResubmit || predecessor.ExecutionFenced || + !receivePeerActionPattern.MatchString(predecessor.Action) || + strings.TrimPrefix(spec.Action, "cleanup-peer-migration-target:") != + strings.TrimPrefix(predecessor.Action, "receive-peer-migration:") { + return false + } + predecessorKeys := effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys) + return lockKeysOverlap(lockKeys, predecessorKeys) +} + +func validExplicitSupersession(spec Spec, lockKeys []string, predecessor Record) bool { + return validForceDeleteSupersession(spec, lockKeys, predecessor) || + validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor) +} + // findResourceBlocker validates any explicit force-delete predecessor and // reads only the compact blocker index for unresolved resource conflicts. func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, bool, error) { @@ -2670,7 +2692,7 @@ func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, boo return Record{}, false, ErrInvalidSupersession } predecessor, err := decodeRecord(encoded) - if err != nil || !validForceDeleteSupersession(spec, lockKeys, predecessor) { + if err != nil || !validExplicitSupersession(spec, lockKeys, predecessor) { return Record{}, false, ErrInvalidSupersession } supersededOperationID = predecessor.OperationID @@ -2965,7 +2987,9 @@ func validateSpec(spec Spec) error { return err } if spec.SupersedesOperationID != "" { - if !operationIDPattern.MatchString(spec.SupersedesOperationID) || spec.SupersedesOperationID == spec.OperationID || spec.Action != "force-delete-vm" { + if !operationIDPattern.MatchString(spec.SupersedesOperationID) || + spec.SupersedesOperationID == spec.OperationID || + (spec.Action != "force-delete-vm" && !cleanupPeerTargetActionPattern.MatchString(spec.Action)) { return ErrInvalidSupersession } } diff --git a/agent/operation/journal_test.go b/agent/operation/journal_test.go index 4d4c512d..d31d9109 100644 --- a/agent/operation/journal_test.go +++ b/agent/operation/journal_test.go @@ -1572,6 +1572,7 @@ func TestSystemRecoveryActionAllowlistIsExplicit(t *testing.T) { "service-suspend", "prepare-peer-migration:0123456789abcdef0123456789abcdef", "receive-peer-migration:0123456789abcdef0123456789abcdef", + "cleanup-peer-migration-target:0123456789abcdef0123456789abcdef", "activate-peer-migration:0123456789abcdef0123456789abcdef", "rollback-peer-migration:0123456789abcdef0123456789abcdef", "create-snapshot:0123456789abcdef0123456789abcdef:filesystem", @@ -1682,6 +1683,51 @@ func TestSuccessfulForceSuccessorOnlyRemovesPredecessorBlockerIndex(t *testing.T } } +func TestPeerMigrationTargetCleanupMayCrossOnlyItsExactReceiveFence(t *testing.T) { + j := openTestJournal(t) + migrationID := "0123456789abcdef0123456789abcdef" + predecessor := testSpec("operation-peer-receive-residue", "vm-peer-residue") + predecessor.Action = "receive-peer-migration:" + migrationID + predecessor.Fingerprint = Fingerprint("POST", "/v1/vms/"+predecessor.ResourceID+"/peer-migration/receive", []byte(`{}`)) + if _, _, err := j.Execute(context.Background(), predecessor, func(context.Context) (json.RawMessage, error) { + return nil, testFailure{code: "result_unconfirmed", message: "receive result unknown", effect: true, safe: false} + }); err == nil { + t.Fatal("peer receive predecessor unexpectedly succeeded") + } + + successor := testSpec("operation-peer-target-cleanup", predecessor.ResourceID) + successor.Action = "cleanup-peer-migration-target:" + migrationID + successor.Fingerprint = Fingerprint("POST", "/v1/vms/"+predecessor.ResourceID+"/peer-migration/cleanup-target", []byte(`{}`)) + successor.SupersedesOperationID = predecessor.OperationID + if record, _, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"deleted":true,"managed_artifacts_absent":true,"forced":false}`), nil + }); err != nil || record.State != StateSucceeded { + t.Fatalf("peer target cleanup successor = %#v err=%v", record, err) + } + + // Cleanup is an evidence-producing successor, not the effect verdict for + // the original receive. Keep that receive fence active until the panel + // performs one fresh absence inspection and resolves it as not applied. + if state, err := j.FenceState(predecessor.OperationID); err != nil || state != FenceStateActive { + t.Fatalf("peer receive fence state=%q err=%v", state, err) + } + followup := testSpec("operation-peer-after-cleanup", predecessor.ResourceID) + if _, _, err := j.Execute(context.Background(), followup, func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"accepted":true}`), nil + }); !errors.Is(err, ErrResourceBlocked) { + t.Fatalf("original receive fence was released before reconciliation: %v", err) + } + + wrong := testSpec("operation-peer-wrong-cleanup", predecessor.ResourceID) + wrong.Action = "cleanup-peer-migration-target:fedcba9876543210fedcba9876543210" + wrong.SupersedesOperationID = predecessor.OperationID + if _, _, err := j.Execute(context.Background(), wrong, func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"deleted":true}`), nil + }); !errors.Is(err, ErrInvalidSupersession) { + t.Fatalf("mismatched migration cleanup supersession err=%v", err) + } +} + func TestInitializeRemovesLegacyStorageBlockerAndPreservesVMFence(t *testing.T) { path := filepath.Join(t.TempDir(), "operations.db") j, err := Open(path) diff --git a/agent/operation/recovery_contract.go b/agent/operation/recovery_contract.go index c88468d1..2bd95b5b 100644 --- a/agent/operation/recovery_contract.go +++ b/agent/operation/recovery_contract.go @@ -14,9 +14,10 @@ import ( const recoveryContractSchemaVersion = 1 var ( - recoveryKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) - restoreBackupActionPattern = regexp.MustCompile(`^restore-backup:[a-f0-9]{32}$`) - receivePeerActionPattern = regexp.MustCompile(`^receive-peer-migration:[a-f0-9]{32}$`) + recoveryKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) + restoreBackupActionPattern = regexp.MustCompile(`^restore-backup:[a-f0-9]{32}$`) + receivePeerActionPattern = regexp.MustCompile(`^receive-peer-migration:[a-f0-9]{32}$`) + cleanupPeerTargetActionPattern = regexp.MustCompile(`^cleanup-peer-migration-target:[a-f0-9]{32}$`) ) //go:embed recovery_contract.json diff --git a/agent/operation/recovery_contract.json b/agent/operation/recovery_contract.json index 32bd9467..a7f03f0b 100644 --- a/agent/operation/recovery_contract.json +++ b/agent/operation/recovery_contract.json @@ -112,6 +112,19 @@ "not_applied" ] }, + { + "kind": "instance_migration_target_clean", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [], + "action_patterns": [ + "^cleanup-peer-migration-target:[a-f0-9]{32}$" + ], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, { "kind": "instance_migration_rollback", "recovery_mode": "generic", diff --git a/agent/worker/protocol.go b/agent/worker/protocol.go index ad7db485..719f5e81 100644 --- a/agent/worker/protocol.go +++ b/agent/worker/protocol.go @@ -13,6 +13,7 @@ import ( "strings" "git.luoli.live/root/virt/agent/auth" + "git.luoli.live/root/virt/agent/operation" ) const ( @@ -38,6 +39,10 @@ type Request struct { Actor auth.Actor `json:"actor"` OperationID string `json:"operation_id"` Fingerprint string `json:"fingerprint"` + // SupersededOperation is a non-secret immutable snapshot injected only by + // the parent Agent after it authorizes an exceptional successor. The child + // uses a separate Journal, so it cannot load the predecessor itself. + SupersededOperation *operation.Record `json:"superseded_operation,omitempty"` } // RequestIdentity is the non-secret, immutable subset persisted beside an @@ -119,6 +124,17 @@ func (r Request) Validate() error { if r.Actor.OperationID == "" || r.Actor.OperationID != r.OperationID { return errors.New("worker actor operation ID does not match request") } + if r.SupersededOperation != nil { + prior := r.SupersededOperation + if prior.OperationID == "" || prior.OperationID == r.OperationID || + !operation.ValidOperationIdentity(prior.Action, prior.ResourceID, prior.Fingerprint) || + prior.State != operation.StateFailed || !prior.EffectMayHaveApplied || + prior.SafeToResubmit || prior.Tombstone || + !(strings.HasSuffix(r.RequestTarget, "/force-delete") || + strings.HasSuffix(r.RequestTarget, "/peer-migration/cleanup-target")) { + return errors.New("worker superseded operation is invalid") + } + } if Fingerprint(r.Method, r.RequestTarget, []byte(r.Body)) != r.Fingerprint { return errors.New("worker request fingerprint does not match target and body") } diff --git a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php index a87e0c05..3f93bbc5 100644 --- a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php @@ -807,6 +807,9 @@ final class CloudAdminOperationRecoveryService $dto['error'] = self::INSPECTION_ERROR_LABELS[$errorCode] ?? '宿主机操作失败(错误码已记录)'; } + if (($value['cleanup']['available'] ?? false) === true) { + $dto['cleanup'] = $value['cleanup']; + } if ($includeProjection && $projection !== null) { $dto['effect_projection'] = $projection; } @@ -997,7 +1000,8 @@ final class CloudAdminOperationRecoveryService } if (in_array($kind, ['delete', 'delete_force', 'expiry_delete', - 'instance_migration_cleanup'], true)) { + 'instance_migration_cleanup', + 'instance_migration_target_clean'], true)) { return self::DELETE_PROJECTION; } if (in_array($kind, [ @@ -1098,7 +1102,8 @@ final class CloudAdminOperationRecoveryService 'network_change' => ['uuid', 'network_changed'], 'reinstall' => ['uuid', 'reinstall_committed'], 'delete', 'delete_force', 'expiry_delete', - 'instance_migration_cleanup' => [ + 'instance_migration_cleanup', + 'instance_migration_target_clean' => [ 'deleted', 'managed_artifacts_absent', 'forced', ], 'instance_migration_rollback', 'instance_migration_activate' => diff --git a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php index e70a9450..feb8c0e0 100644 --- a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php @@ -920,6 +920,14 @@ final class CloudAutomaticOperationRecoveryService } $resolution = (string) ($inspection['effect_resolution'] ?? ''); + if ($resolution === 'partial' + && (string) $job->kind === 'instance_migration_transfer' + && CloudInstanceMigrationService::ensureTargetResidueCleanup( + $job, + $inspection + )) { + return self::pending('recovery_target_cleanup_pending'); + } if (in_array($resolution, ['partial', 'inconclusive'], true)) { return self::pending( $resolution === 'partial' diff --git a/web/app/Services/Cloud/CloudInstanceMigrationService.php b/web/app/Services/Cloud/CloudInstanceMigrationService.php index 24ca050b..2cd6e4ea 100644 --- a/web/app/Services/Cloud/CloudInstanceMigrationService.php +++ b/web/app/Services/Cloud/CloudInstanceMigrationService.php @@ -25,6 +25,156 @@ final class CloudInstanceMigrationService 'attention_required', ]; + /** + * Queue one plan-bound cleanup for residue left by an interrupted target + * receive. The original receive remains fenced and is never replayed; its + * ordinary automatic recovery will prove absence and repair every PHP + * placement/migration ledger after this successor succeeds. + */ + public static function ensureTargetResidueCleanup( + CloudJob $receiveJob, + array $inspection + ): bool { + $cleanup = $inspection['cleanup'] ?? null; + $planId = is_array($cleanup) ? ($cleanup['plan_id'] ?? null) : null; + $steps = is_array($cleanup) ? ($cleanup['steps'] ?? null) : null; + $allowedSteps = [ + 'delete_managed_volumes', 'delete_seed_volume', + 'undefine_network_filter', 'delete_nvram', + 'remove_traffic_state', + ]; + if ((string) $receiveJob->kind !== 'instance_migration_transfer' + || ($cleanup['available'] ?? false) !== true + || !is_string($planId) + || preg_match('/^[a-f0-9]{64}$/D', strtolower($planId)) !== 1 + || !is_array($steps) || !array_is_list($steps) || $steps === [] + || array_diff($steps, $allowedSteps) !== [] + || count(array_unique($steps)) !== count($steps)) { + return false; + } + return $receiveJob->getConnection()->transaction( + static function () use ($receiveJob, $planId, $steps): bool { + $lockedJob = CloudJob::where( + 'id', (int) $receiveJob->id + )->lockForUpdate()->first(); + $migration = CloudInstanceMigration::where( + 'target_receive_job_id', (int) $receiveJob->id + )->lockForUpdate()->first(); + $instance = $migration === null ? null : CloudInstance::where( + 'id', (int) $migration->instance_id + )->lockForUpdate()->first(); + $restore = $migration === null ? null : CloudBackupRestore::where( + 'id', (int) $migration->restore_id + )->lockForUpdate()->first(); + if ($lockedJob === null || $migration === null + || $instance === null || $restore === null + || (string) $lockedJob->kind + !== 'instance_migration_transfer' + || (string) $lockedJob->phase !== 'terminal' + || (string) $lockedJob->outcome !== 'failed' + || (string) $lockedJob->effect_resolution !== 'unknown' + || (int) $lockedJob->effect_may_have_applied !== 1 + || (int) $lockedJob->safe_to_resubmit !== 0 + || (int) $lockedJob->host_id + !== (int) $migration->target_host_id + || (int) $lockedJob->instance_id !== (int) $instance->id + || (int) $lockedJob->backup_restore_id + !== (int) $restore->id + || (int) $restore->target_host_id + !== (int) $migration->target_host_id + || !in_array((string) $restore->state, + ['restoring', 'uncertain'], true) + || !hash_equals((string) $lockedJob->operation_id, + (string) $restore->operation_id)) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 目标残留清理账本不一致', 409 + ); + } + $priorCleanups = CloudJob::where( + 'parent_job_id', (int) $lockedJob->id + )->where('kind', 'instance_migration_target_clean') + ->orderBy('id')->lockForUpdate()->get(); + foreach ($priorCleanups as $priorCleanup) { + if ((string) $priorCleanup->phase !== 'terminal' + || (string) $priorCleanup->outcome === 'succeeded' + || (string) $priorCleanup->effect_resolution + !== 'not_applied') { + return true; + } + } + if (count($priorCleanups) >= 3) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 目标残留自动清理已达到安全重试上限', + 409 + ); + } + $generation = count($priorCleanups) + 1; + $target = '/v1/vms/' + . rawurlencode((string) $migration->target_uuid) + . '/peer-migration/cleanup-target'; + $body = [ + 'migration_id' => (string) $migration->migration_key, + 'prior_operation_id' => + (string) $lockedJob->operation_id, + 'cleanup_plan_id' => strtolower($planId), + ]; + [$cleanupJob, $created] = CloudJobService:: + reserveStandaloneAgentMutation( + [ + 'user_id' => (int) $lockedJob->user_id, + 'request_idempotency_key' => + 'migration-target-clean-' + . (string) $migration->migration_key + . '-' . $generation, + ], + (int) $migration->target_host_id, + 'instance_migration_target_clean', + 'POST', + $target, + $body, + [ + 'owner_id' => (int) $instance->user_id, + 'can_manage' => true, + 'maintenance_migration_id' => + (int) $migration->id, + 'prior_job_id' => (int) $lockedJob->id, + 'cleanup_plan_id' => strtolower($planId), + 'cleanup_steps' => array_values($steps), + ], + [ + 'migration_id' => (int) $migration->id, + 'generation' => $generation, + 'prior_job_id' => (int) $lockedJob->id, + 'prior_operation_id' => + (string) $lockedJob->operation_id, + 'cleanup_plan_id' => strtolower($planId), + 'cleanup_steps' => array_values($steps), + ], + null, + '/v1/instance-maintenance-migrations/' + . (int) $migration->id + . '/target-residue-cleanup/' . $generation + ); + if (!$created) { + return true; + } + $cleanupJob->parent_job_id = (int) $lockedJob->id; + $cleanupJob->instance_id = (int) $instance->id; + $cleanupJob->backup_restore_id = + (int) $migration->restore_id; + $cleanupJob->save(); + self::setState( + $migration, + 'attention_required', + 'target_residue_cleanup_running', + 'reconcile_transfer' + ); + return true; + }, + 3 + ); + } + public static function start( CloudInstance $instance, array $actor, diff --git a/web/app/Services/Cloud/CloudJobService.php b/web/app/Services/Cloud/CloudJobService.php index afb924bc..1ea59302 100644 --- a/web/app/Services/Cloud/CloudJobService.php +++ b/web/app/Services/Cloud/CloudJobService.php @@ -73,6 +73,8 @@ final class CloudJobService 'Agent 回执与宿主机证据暂不一致;资源保持安全锁定,系统仍会继续自动核对。', 'recovery_evidence_partial' => '宿主机目前只能确认部分操作结果;资源保持安全锁定,系统仍会继续自动核对。', + 'recovery_target_cleanup_pending' => + '系统正在自动清理维护迁移目标机残留;原迁移不会重放,清理后会继续核对并修复迁移账本。', 'recovery_evidence_inconclusive' => '宿主机证据暂不足以确认操作结果;资源保持安全锁定,系统仍会继续自动核对。', 'recovery_evidence_invalid' => diff --git a/web/app/Services/Cloud/CloudOperationCatalog.php b/web/app/Services/Cloud/CloudOperationCatalog.php index 359b8b14..fd490e47 100644 --- a/web/app/Services/Cloud/CloudOperationCatalog.php +++ b/web/app/Services/Cloud/CloudOperationCatalog.php @@ -29,6 +29,7 @@ final class CloudOperationCatalog 'backup_recover' => '从云备份恢复实例', 'instance_migration_prepare' => '准备维护迁移源实例', 'instance_migration_transfer' => '传输维护迁移数据', + 'instance_migration_target_clean' => '清理维护迁移目标残留', 'instance_migration_cleanup' => '清理维护迁移源副本', 'instance_migration_rollback' => '恢复维护迁移源实例', 'instance_migration_activate' => '恢复维护迁移目标状态', @@ -77,6 +78,7 @@ final class CloudOperationCatalog 'backup_recover', 'instance_migration_prepare', 'instance_migration_transfer', + 'instance_migration_target_clean', 'instance_migration_cleanup', 'instance_migration_rollback', 'instance_migration_activate', diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php index d26ad0dc..e407e0b7 100644 --- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php +++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php @@ -509,6 +509,7 @@ final class CloudOperationErrorCatalog 'backup_recover' => '从云备份恢复实例失败;任务已结束且不会自动重试', 'instance_migration_prepare' => '维护迁移源准备失败;请按迁移恢复出口处理', 'instance_migration_transfer' => '维护迁移传输失败;请按迁移恢复出口处理', + 'instance_migration_target_clean' => '维护迁移目标残留清理失败;原迁移仍保持锁定并继续核对', 'instance_migration_cleanup' => '维护迁移源清理失败;目标实例不受影响', 'instance_migration_rollback' => '维护迁移源恢复失败;请按迁移恢复出口处理', 'instance_migration_activate' => '维护迁移目标状态恢复失败;请按迁移恢复出口处理', diff --git a/web/app/Services/Cloud/CloudOperationRecoveryContract.php b/web/app/Services/Cloud/CloudOperationRecoveryContract.php index b44e9be3..3d2a308f 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryContract.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryContract.php @@ -63,6 +63,13 @@ final class CloudOperationRecoveryContract 'action_patterns' => ['^receive-peer-migration:[a-f0-9]{32}$'], 'system_resolutions' => ['applied', 'not_applied'], ], + 'instance_migration_target_clean' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => [], + 'action_patterns' => ['^cleanup-peer-migration-target:[a-f0-9]{32}$'], + 'system_resolutions' => ['applied', 'not_applied'], + ], 'instance_migration_rollback' => [ 'recovery_mode' => 'generic', 'identity_scope' => 'instance', diff --git a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php index 05892000..4288eefb 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php @@ -263,6 +263,7 @@ final class CloudOperationRecoveryIdentity $kind = (string) $job->kind; if (in_array($kind, ['instance_migration_prepare', 'instance_migration_transfer', + 'instance_migration_target_clean', 'instance_migration_cleanup', 'instance_migration_rollback', 'instance_migration_activate'], true) && isset($payload['maintenance_migration_id'])) { @@ -432,6 +433,48 @@ final class CloudOperationRecoveryIdentity return ['receive-peer-migration:' . (string) $migration->migration_key, $uuid]; } + if ((string) $job->kind === 'instance_migration_target_clean') { + $prior = $job->parent_job_id === null ? null : CloudJob::find( + (int) $job->parent_job_id + ); + if ($prior === null + || (int) $migration->target_receive_job_id + !== (int) $prior->id + || (string) $prior->kind + !== 'instance_migration_transfer' + || (int) $migration->target_host_id !== (int) $job->host_id + || preg_match(self::UUID_PATTERN, + strtolower((string) $migration->target_uuid)) !== 1) { + throw new CloudRecoveryInstanceStateConflict( + 'PeerMigration 目标残留清理身份不一致', 409 + ); + } + $uuid = strtolower((string) $migration->target_uuid); + self::expectRequest($method, $target, 'POST', + '/v1/vms/' . rawurlencode($uuid) + . '/peer-migration/cleanup-target'); + self::assertAllowedBody($body, [ + 'operation_id', 'migration_id', 'prior_operation_id', + 'cleanup_plan_id', + ]); + if (($body['migration_id'] ?? null) + !== (string) $migration->migration_key + || !hash_equals((string) $prior->operation_id, + (string) ($body['prior_operation_id'] ?? '')) + || !is_string($body['cleanup_plan_id'] ?? null) + || preg_match('/^[a-f0-9]{64}$/D', + (string) $body['cleanup_plan_id']) !== 1 + || !hash_equals((string) ($payload['cleanup_plan_id'] ?? ''), + (string) $body['cleanup_plan_id']) + || (int) ($payload['prior_job_id'] ?? 0) + !== (int) $prior->id) { + self::invalidPayload( + 'PeerMigration 目标残留清理确认无效' + ); + } + return ['cleanup-peer-migration-target:' + . (string) $migration->migration_key, $uuid]; + } if ((string) $job->kind === 'instance_migration_activate') { if ((int) $migration->activate_job_id !== (int) $job->id || (int) $migration->target_host_id !== (int) $job->host_id diff --git a/web/tests/CloudOperationRecoveryMatrixTest.php b/web/tests/CloudOperationRecoveryMatrixTest.php index 4e7ef6af..f426e762 100644 --- a/web/tests/CloudOperationRecoveryMatrixTest.php +++ b/web/tests/CloudOperationRecoveryMatrixTest.php @@ -833,6 +833,36 @@ namespace { null, 9002 ); + $migrationTargetClean = $insertJob( + 'instance_migration_target_clean', + static fn (int $id, string $operationId): array => [ + 'method' => 'POST', + 'request_target' => '/v1/vms/' + . rawurlencode($migrationTargetUuid) + . '/peer-migration/cleanup-target', + 'body' => [ + 'operation_id' => $operationId, + 'migration_id' => $migrationKey, + 'prior_operation_id' => + $migrationTransfer['operation_id'], + 'cleanup_plan_id' => str_repeat('c', 64), + ], + 'owner_id' => $ownerId, + 'can_manage' => true, + 'maintenance_migration_id' => 1, + 'prior_job_id' => $migrationTransfer['id'], + 'cleanup_plan_id' => str_repeat('c', 64), + 'cleanup_steps' => ['delete_managed_volumes'], + ], + $instanceId, + null, + $migrationTransfer['id'], + null, + 'agent', + $ownerId, + null, + 9002 + ); $migrationCleanup = $insertJob( 'instance_migration_cleanup', static fn (int $id, string $operationId): array => [ @@ -1017,6 +1047,14 @@ namespace { 'staged_powered_off' => true, ] ); + $addFixture( + 'instance_migration_target_clean', + 'target-residue-cleanup', + $migrationTargetClean, + 'cleanup-peer-migration-target:' . $migrationKey, + $migrationTargetUuid, + $deleteProjection + ); $addFixture( 'instance_migration_cleanup', 'source-cleanup', @@ -2181,6 +2219,34 @@ namespace { CloudAdminOperationRecoveryService::class, 'inspection' ); + $transferIdentity = CloudOperationRecoveryIdentity::expectedFromPayload( + $migrationTransfer['job'], + $migrationTransfer['payload'] + ); + $cleanupInspection = $inspectionMethod->invoke(null, [ + 'operation_id' => $migrationTransfer['operation_id'], + 'action' => $transferIdentity['action'], + 'resource_id' => $transferIdentity['resource_id'], + 'request_fingerprint' => + $transferIdentity['request_fingerprint'], + 'operation_status' => 'failed', + 'checked_at' => time(), + 'outcome' => 'uncertain', + 'summary_code' => 'cleanup_required', + 'effect_resolution' => 'partial', + 'fence_state' => 'active', + 'fence_resolution_required' => true, + 'abort_job' => ['available' => false], + 'cleanup' => [ + 'available' => true, + 'plan_id' => str_repeat('c', 64), + 'steps' => ['delete_managed_volumes'], + ], + ], $migrationTransfer['operation_id'], + 'instance_migration_transfer', true); + $assert(($cleanupInspection['cleanup']['plan_id'] ?? null) + === str_repeat('c', 64), + 'Validated target cleanup evidence was dropped before recovery.'); $upgradeNotAppliedRejected = false; try { $inspectionMethod->invoke(null, [ @@ -2727,5 +2793,187 @@ namespace { )->where('id', '<>', $migrationRollback['id'])->count() === 1, 'Abort prepare created more than one compensating rollback task.'); + // A partial target-receive inspection queues one independent, plan-bound + // cleanup generation. Re-entering automatic recovery must reuse that + // durable child rather than replaying the receive or creating duplicates. + $nextJobId = max( + $nextJobId, + (int) Capsule::table('cloud_jobs')->max('id') + 1 + ); + $residueMigrationKey = str_repeat('9', 32); + $residueTargetUuid = '88888888-8888-4888-8888-888888888888'; + $residueTransfer = $insertJob( + 'instance_migration_transfer', + static fn (int $id, string $operationId): array => [ + 'method' => 'POST', + 'request_target' => '/v1/vms/' + . rawurlencode($residueTargetUuid) + . '/peer-migration/receive', + 'body' => [ + 'operation_id' => $operationId, + 'public_ip_cidr' => '203.0.113.30/24', + ], + 'owner_id' => $ownerId, + 'can_manage' => true, + 'maintenance_migration_id' => 2, + ], + $instanceId, + null, + null, + null, + 'agent', + $ownerId, + null, + 9012 + ); + Capsule::table('cloud_backup_restores')->insert([ + 'id' => 9012, + 'restore_key' => str_repeat('f', 32), + 'backup_id' => null, + 'target_host_id' => $hostId, + 'target_instance_id' => null, + 'reservation_id' => 9004, + 'job_id' => $residueTransfer['id'], + 'operation_id' => $residueTransfer['operation_id'], + 'new_uuid' => $residueTargetUuid, + 'new_name' => 'residue-target', + 'state' => 'uncertain', + 'error_code' => 'operation_effect_unknown', + 'ready_at' => null, + 'created_at' => $now - 120, + 'updated_at' => $now, + ]); + Capsule::table('cloud_instance_migrations')->insert([ + 'id' => 2, + 'migration_key' => $residueMigrationKey, + 'instance_id' => $instanceId, + 'restore_id' => 9012, + 'source_host_id' => $hostId, + 'source_uuid' => $instanceUuid, + 'source_reservation_id' => 9003, + 'target_host_id' => $hostId, + 'target_uuid' => $residueTargetUuid, + 'target_reservation_id' => 9004, + 'source_prepare_job_id' => null, + 'target_receive_job_id' => $residueTransfer['id'], + 'firewall_job_id' => null, + 'activate_job_id' => null, + 'rollback_job_id' => null, + 'cleanup_job_id' => null, + 'cleanup_mode' => 'delete', + 'state' => 'attention_required', + 'recovery_action' => 'reconcile_transfer', + 'retry_generation' => 0, + 'source_endpoint' => '["https://192.0.2.10:9443"]', + 'peer_trust_id' => str_repeat('1', 64), + 'manifest_digest' => str_repeat('2', 64), + 'source_was_running' => true, + 'source_autostart_enabled' => true, + 'source_service_status' => 'active', + 'source_network_status' => 'active', + 'error_code' => 'operation_effect_unknown', + 'cutover_at' => null, + 'completed_at' => null, + 'created_at' => $now - 120, + 'updated_at' => $now, + ]); + Capsule::table('cloud_resource_reservations')->where('id', 9004) + ->update([ + 'backup_restore_id' => 9012, + 'operation_id' => $residueTransfer['operation_id'], + 'instance_id' => $instanceId, + 'state' => 'uncertain', + 'updated_at' => $now, + ]); + Capsule::table('cloud_public_ips')->where('id', 9007)->update([ + 'state' => 'uncertain', + 'instance_id' => null, + 'operation_id' => $residueTransfer['operation_id'], + 'updated_at' => $now, + ]); + $residueInspection = [ + 'cleanup' => [ + 'available' => true, + 'plan_id' => str_repeat('3', 64), + 'steps' => ['delete_managed_volumes'], + ], + ]; + $assert(CloudInstanceMigrationService::ensureTargetResidueCleanup( + CloudJob::findOrFail($residueTransfer['id']), + $residueInspection + ), 'A safe target-residue plan did not queue cleanup.'); + $targetCleanup = CloudJob::where( + 'parent_job_id', $residueTransfer['id'] + )->where('kind', 'instance_migration_target_clean')->first(); + $targetCleanupPayload = $targetCleanup === null ? null + : CloudSecrets::decrypt( + (string) $targetCleanup->sealed_payload, + 'job-payload' + ); + $assert($targetCleanup !== null + && (int) $targetCleanup->host_id === $hostId + && (int) $targetCleanup->instance_id === $instanceId + && ($targetCleanupPayload['body']['prior_operation_id'] ?? null) + === $residueTransfer['operation_id'] + && ($targetCleanupPayload['body']['cleanup_plan_id'] ?? null) + === str_repeat('3', 64) + && str_ends_with( + (string) ($targetCleanupPayload['request_target'] ?? ''), + '/peer-migration/cleanup-target' + ), 'Target-residue cleanup did not retain its exact plan and parent.'); + $assert(CloudInstanceMigrationService::ensureTargetResidueCleanup( + CloudJob::findOrFail($residueTransfer['id']), + $residueInspection + ) && CloudJob::where( + 'parent_job_id', $residueTransfer['id'] + )->where('kind', 'instance_migration_target_clean')->count() === 1, + 'Repeated recovery created duplicate target-residue cleanups.'); + + // Model the next recovery pass after the independent cleanup succeeds and + // the original receive inspection proves the target is absent. The normal + // transfer finalizer must restore every target ledger to one retryable + // operation identity instead of leaving uncertain capacity or IP claims. + Capsule::table('cloud_jobs')->where('id', $targetCleanup->id)->update([ + 'phase' => 'terminal', + 'outcome' => 'succeeded', + 'status' => 'succeeded', + 'effect_may_have_applied' => 0, + 'safe_to_resubmit' => 0, + 'effect_resolution' => 'applied', + 'terminal_at' => $now, + 'updated_at' => $now, + ]); + Capsule::table('cloud_jobs')->where('id', $residueTransfer['id']) + ->update([ + 'effect_may_have_applied' => 0, + 'safe_to_resubmit' => 0, + 'effect_resolution' => 'not_applied', + 'effect_resolution_reason' => 'system_inspection', + 'updated_at' => $now, + ]); + CloudInstanceMigrationService::finishJobLocked( + CloudJob::findOrFail($residueTransfer['id']), + 'failed', + null, + $residueTransfer['payload'] + ); + $repairedMigration = CloudInstanceMigration::findOrFail(2); + $repairedRestore = Capsule::table('cloud_backup_restores') + ->where('id', 9012)->first(); + $repairedReservation = Capsule::table('cloud_resource_reservations') + ->where('id', 9004)->first(); + $repairedIp = Capsule::table('cloud_public_ips') + ->where('id', 9007)->first(); + $assert((string) $repairedMigration->state === 'attention_required' + && (string) $repairedMigration->recovery_action === 'retry_transfer' + && (string) $repairedMigration->error_code + === 'peer_transfer_failed' + && (string) $repairedRestore->state === 'reserved' + && (string) $repairedRestore->error_code === 'peer_transfer_failed' + && (string) $repairedReservation->state === 'held' + && (string) $repairedIp->state === 'reserved' + && $repairedIp->instance_id === null, + 'Target cleanup did not repair migration, restore, capacity, and IP ledgers for a safe retry.'); + echo "Cloud operation recovery matrix tests passed\n"; } diff --git a/web/tests/CloudOperationRecoveryPolicyTest.php b/web/tests/CloudOperationRecoveryPolicyTest.php index 906b106c..8f698ba0 100644 --- a/web/tests/CloudOperationRecoveryPolicyTest.php +++ b/web/tests/CloudOperationRecoveryPolicyTest.php @@ -56,6 +56,7 @@ namespace { 'instance_migration_cleanup', 'instance_migration_prepare', 'instance_migration_transfer', + 'instance_migration_target_clean', 'instance_migration_rollback', 'instance_migration_activate', 'expiry_delete', @@ -81,8 +82,8 @@ namespace { ]; $assert(CloudOperationRecoveryPolicy::kinds() === $expectedKinds, 'The Agent leaf recovery policy changed without updating its contract.'); - $assert(count(CloudOperationRecoveryPolicy::kinds()) === 30, - 'The recovery policy contract must contain exactly 30 Agent leaf kinds.'); + $assert(count(CloudOperationRecoveryPolicy::kinds()) === 31, + 'The recovery policy contract must contain exactly 31 Agent leaf kinds.'); foreach ($expectedKinds as $kind) { $assert(CloudOperationRecoveryPolicy::supports($kind), 'The recovery policy omitted ' . $kind . '.'); diff --git a/web/tests/CloudSnapshotWorkflowTest.php b/web/tests/CloudSnapshotWorkflowTest.php index d83da64f..c1212cc5 100644 --- a/web/tests/CloudSnapshotWorkflowTest.php +++ b/web/tests/CloudSnapshotWorkflowTest.php @@ -634,7 +634,8 @@ namespace { 'instance_upgrade', 'firewall_apply', 'snapshot_create', 'snapshot_restore', 'snapshot_delete', 'backup_create', 'backup_restore', 'backup_recover', 'instance_migration_prepare', - 'instance_migration_transfer', 'instance_migration_cleanup', + 'instance_migration_transfer', 'instance_migration_target_clean', + 'instance_migration_cleanup', 'instance_migration_rollback', 'instance_migration_activate', 'rescue', ], 'The instance operation log does not cover every settings operation.'); $assert(in_array((string) $settingsAfterDelete->id, $operationLogIds, true),