fix: reconcile peer migration target residue

go
codex 4 weeks ago
parent fd28f475f1
commit 6f4ee55a69

@ -66,6 +66,7 @@ type Server struct {
}
type trustedWorkerRequestKey struct{}
type trustedSupersededOperationKey struct{}
// SetMutationExecutor enables the clean-break process-isolated mutation path.
// It must be called during startup before request admission begins.
@ -81,7 +82,8 @@ func (s *Server) SetMutationExecutor(executor worker.Executor) error {
// already authenticated and validated by the parent Agent. Hidden workers do
// not have access to the panel secret through HTTP headers and cannot recurse
// because they never install a MutationExecutor.
func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor) {
func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Request, body []byte, actor auth.Actor,
superseded ...operation.Record) {
setResponseHeaders(w)
if !s.beginRequest() {
writePreDispatchError(w, http.StatusServiceUnavailable, "agent_shutting_down", "Agent is shutting down")
@ -93,9 +95,18 @@ func (s *Server) ServeTrustedWorkerRequest(w http.ResponseWriter, r *http.Reques
return
}
trusted := context.WithValue(r.Context(), trustedWorkerRequestKey{}, struct{}{})
if len(superseded) > 0 {
prior := superseded[0]
trusted = context.WithValue(trusted, trustedSupersededOperationKey{}, prior)
}
s.route(w, r.WithContext(trusted), body, actor)
}
func trustedSupersededOperation(ctx context.Context, operationID string) (operation.Record, bool) {
prior, ok := ctx.Value(trustedSupersededOperationKey{}).(operation.Record)
return prior, ok && prior.OperationID == operationID
}
type errorBody struct {
Code string `json:"code"`
Message string `json:"message"`
@ -610,6 +621,88 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str
func(ctx context.Context) (any, error) {
return s.Hypervisor.CleanupPeerMigration(ctx, uuid, request)
})
case "cleanup-target":
if r.Method != http.MethodPost {
methodOrNotFound(w, r, true, http.MethodPost)
return
}
request, ok := decodeRequired[hypervisor.CleanupPeerMigrationTargetRequest](w, body)
if !ok {
return
}
operationID, err := resolveOperationID(actor.OperationID, request.OperationID)
if err != nil {
writePreDispatchError(w, http.StatusUnprocessableEntity, "invalid_operation_id", err.Error())
return
}
request.OperationID = operationID
if err := hypervisor.ValidateCleanupPeerMigrationTargetRequest(request); err != nil {
writePreDispatchMappedError(w, err)
return
}
action := "cleanup-peer-migration-target:" + request.MigrationID
existingReplay := false
trustedPrior, trustedPriorFound := trustedSupersededOperation(
r.Context(), request.PriorOperationID,
)
existing, existingErr := s.Journal.Get(operationID)
if existingErr == nil {
existingReplay = existing.Action == action && existing.ResourceID == uuid &&
existing.SupersedesOperationID == request.PriorOperationID
} else if !errors.Is(existingErr, operation.ErrNotFound) {
writeMappedError(w, existingErr, nil)
return
}
if !existingReplay {
prior := trustedPrior
var priorErr error
if !trustedPriorFound {
prior, priorErr = s.Journal.Get(request.PriorOperationID)
}
if priorErr != nil || prior.ResourceID != uuid ||
prior.Action != "receive-peer-migration:"+request.MigrationID ||
prior.State != operation.StateFailed || !prior.EffectMayHaveApplied ||
prior.SafeToResubmit || prior.ExecutionFenced {
writePreDispatchError(w, http.StatusConflict, "peer_migration_target_cleanup_not_authorized", "target cleanup requires the exact safely reaped failed receive operation")
return
}
if !trustedPriorFound {
_, incidentFound, incidentErr := s.Journal.CompletionDeliveryIncident(prior.OperationID)
if incidentErr != nil {
writeMappedError(w, incidentErr, nil)
return
}
if incidentFound {
writePreDispatchError(w, http.StatusConflict, "peer_migration_target_cleanup_not_authorized", "target cleanup is blocked by a completion-delivery incident")
return
}
}
ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, prior)
cancel()
if inspectErr != nil {
writeMappedError(w, inspectErr, nil)
return
}
if !peerMigrationTargetCleanupInspectionMatches(prior, request.CleanupPlanID, inspection) {
writePreDispatchError(w, http.StatusConflict, "peer_migration_cleanup_plan_changed", "target cleanup plan is stale or no longer safe")
return
}
request.PriorOperation = prior
}
supersedes := []string{request.PriorOperationID}
if trustedPriorFound {
// The isolated worker has its own Journal. The parent Journal
// already admitted this exact supersession and retains the audit
// link; the child receives the immutable predecessor separately.
supersedes = nil
}
s.runMutationExpected(w, r, body, actor, operationID, uuid, action, nil,
true, http.StatusOK,
hypervisor.CleanupPeerMigrationTargetAdmission(uuid, request.MigrationID),
func(ctx context.Context) (any, error) {
return s.Hypervisor.CleanupPeerMigrationTarget(ctx, uuid, request)
}, supersedes...)
default:
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
@ -645,6 +738,9 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str
return
}
existingReplay := false
trustedPrior, trustedPriorFound := trustedSupersededOperation(
r.Context(), request.PriorOperationID,
)
existing, existingErr := s.Journal.Get(operationID)
if existingErr == nil {
existingReplay = existing.Action == "force-delete-vm" && existing.ResourceID == uuid &&
@ -654,7 +750,11 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str
return
}
if !existingReplay {
prior, priorErr := s.Journal.Get(request.PriorOperationID)
prior := trustedPrior
var priorErr error
if !trustedPriorFound {
prior, priorErr = s.Journal.Get(request.PriorOperationID)
}
priorActionAllowed := priorErr == nil && (prior.Action == "delete-vm" || prior.Action == "service-suspend")
if !priorActionAllowed || prior.ResourceID != uuid ||
prior.State != operation.StateFailed || prior.Error == nil || prior.Error.Code != "force_required" {
@ -662,11 +762,18 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str
return
}
}
supersedes := []string{request.PriorOperationID}
if trustedPriorFound {
// The parent Journal already authorized and records the force
// successor. The isolated worker Journal receives the immutable
// predecessor separately and must not try to load it locally.
supersedes = nil
}
s.runMutationExpected(w, r, body, actor, request.OperationID, uuid, "force-delete-vm", nil,
true, http.StatusOK, hypervisor.DeleteVMAdmission(uuid),
func(ctx context.Context) (any, error) {
return s.Hypervisor.ForceDeleteVM(ctx, uuid, request)
}, request.PriorOperationID)
}, supersedes...)
return
}
case "state":
@ -943,6 +1050,32 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str
writePreDispatchError(w, http.StatusNotFound, "not_found", "endpoint not found")
}
func peerMigrationTargetCleanupInspectionMatches(prior operation.Record, planID string,
inspection hypervisor.OperationInspection) bool {
if inspection.OperationID != prior.OperationID || inspection.Action != prior.Action ||
inspection.ResourceID != prior.ResourceID || inspection.RequestFingerprint != prior.Fingerprint ||
inspection.OperationStatus != prior.State || inspection.CheckedAt <= 0 ||
inspection.SummaryCode != hypervisor.InspectionSummaryCleanupRequired ||
inspection.EffectResolution != hypervisor.InspectionEffectPartial ||
inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" ||
inspection.AbortJob.Available || !inspection.Cleanup.Available ||
!strings.EqualFold(inspection.Cleanup.PlanID, planID) || len(inspection.Cleanup.Steps) == 0 {
return false
}
for _, step := range inspection.Cleanup.Steps {
switch step {
case hypervisor.CleanupStepDeleteManagedVolumes,
hypervisor.CleanupStepDeleteSeedVolume,
hypervisor.CleanupStepUndefineNetworkFilter,
hypervisor.CleanupStepDeleteNVRAM,
hypervisor.CleanupStepRemoveTrafficState:
default:
return false
}
}
return true
}
func (s *Server) routeSnapshots(w http.ResponseWriter, r *http.Request, segments []string, body []byte, actor auth.Actor, uuid string) {
if len(segments) == 4 {
switch r.Method {
@ -1746,6 +1879,14 @@ func (s *Server) runIsolatedMutation(
RequestTarget: auth.RequestTarget(r), Body: string(body), Actor: actor,
OperationID: spec.OperationID, Fingerprint: spec.Fingerprint,
}
if spec.SupersedesOperationID != "" {
prior, err := s.Journal.Get(spec.SupersedesOperationID)
if err != nil {
writeMappedError(w, err, nil)
return
}
request.SupersededOperation = &prior
}
dispatchCtx, dispatchCancel := context.WithTimeout(r.Context(), s.AdmissionTimeout)
record, replayed, done, err := s.Journal.ExecuteBackground(
dispatchCtx, s.backgroundCtx, deadline, spec,

@ -28,11 +28,13 @@ import (
type testHypervisor struct {
hypervisor.Unsupported
mu sync.Mutex
inspections int
configureCalls int
inspectionResult *hypervisor.OperationInspection
inspectionErr error
mu sync.Mutex
inspections int
configureCalls int
cleanupPeerTargetCalls int
cleanupPeerTargetLast hypervisor.CleanupPeerMigrationTargetRequest
inspectionResult *hypervisor.OperationInspection
inspectionErr error
}
func (h *testHypervisor) Ping(context.Context) error { return nil }
@ -48,6 +50,17 @@ func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageCon
return hypervisor.StorageConfigurationResult{Configured: true}, nil
}
func (h *testHypervisor) CleanupPeerMigrationTarget(_ context.Context, _ string,
request hypervisor.CleanupPeerMigrationTargetRequest) (hypervisor.DeleteVMResult, error) {
h.mu.Lock()
h.cleanupPeerTargetCalls++
h.cleanupPeerTargetLast = request
h.mu.Unlock()
return hypervisor.DeleteVMResult{
Deleted: true, ManagedArtifactsAbsent: true, Forced: false,
}, nil
}
func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) {
h.mu.Lock()
defer h.mu.Unlock()
@ -526,6 +539,7 @@ func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing
"change-network", "network-suspend", "network-resume",
"apply-firewall", "delete-vm", "force-delete-vm",
"service-suspend",
"cleanup-peer-migration-target:0123456789abcdef0123456789abcdef",
"create-snapshot:0123456789abcdef0123456789abcdef:filesystem",
"create-snapshot:0123456789abcdef0123456789abcdef:crash",
"delete-snapshot:0123456789abcdef0123456789abcdef",
@ -917,6 +931,164 @@ func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) {
}
}
func TestPeerMigrationTargetCleanupRequiresFreshExactPlanAndRetainsReceiveFence(t *testing.T) {
api := newTestAPI(t)
const (
uuid = "123e4567-e89b-42d3-a456-426614174000"
migrationID = "0123456789abcdef0123456789abcdef"
priorID = "operation-peer-receive-residue"
cleanupID = "operation-peer-target-cleanup"
)
prior := operation.Spec{
OperationID: priorID, ResourceID: uuid,
Action: "receive-peer-migration:" + migrationID,
Fingerprint: operation.Fingerprint(http.MethodPost,
"/v1/vms/"+uuid+"/peer-migration/receive", []byte(`{}`)),
Destructive: true,
LockKeys: hypervisor.ReceivePeerMigrationAdmission(hypervisor.ReceivePeerMigrationRequest{
CreateVMRequest: hypervisor.CreateVMRequest{UUID: uuid}, MigrationID: migrationID,
}).LockKeys,
}
if _, _, err := api.journal.Execute(context.Background(), prior, func(context.Context) (json.RawMessage, error) {
return nil, &worker.Error{Code: "result_unconfirmed", Message: "receive result unknown", EffectMayHaveApplied: true}
}); err == nil {
t.Fatal("seed receive failure unexpectedly succeeded")
}
stored, err := api.journal.Get(priorID)
if err != nil {
t.Fatal(err)
}
planID := strings.Repeat("a", 64)
api.hv.inspectionResult = &hypervisor.OperationInspection{
OperationID: stored.OperationID, Action: stored.Action,
ResourceID: stored.ResourceID, RequestFingerprint: stored.Fingerprint,
OperationStatus: stored.State, CheckedAt: api.now.Unix(),
Outcome: hypervisor.InspectionOutcomeUncertain,
SummaryCode: hypervisor.InspectionSummaryCleanupRequired,
EffectResolution: hypervisor.InspectionEffectPartial,
Cleanup: hypervisor.OperationCleanupPlan{Available: true, PlanID: planID,
Steps: []string{hypervisor.CleanupStepDeleteManagedVolumes}},
}
executor := &fakeMutationExecutor{result: json.RawMessage(`{"deleted":true,"managed_artifacts_absent":true,"forced":false}`)}
if err := api.handler.SetMutationExecutor(executor); err != nil {
t.Fatal(err)
}
request := hypervisor.CleanupPeerMigrationTargetRequest{
OperationID: cleanupID, MigrationID: migrationID,
PriorOperationID: priorID, CleanupPlanID: planID,
}
body, err := json.Marshal(request)
if err != nil {
t.Fatal(err)
}
actor := auth.Actor{UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true, OperationID: cleanupID}
response := api.request(t, http.MethodPost,
"/v1/vms/"+uuid+"/peer-migration/cleanup-target", body, actor)
if response.Code != http.StatusAccepted {
t.Fatalf("cleanup response = %d %s", response.Code, response.Body.String())
}
terminal := waitForTerminal(t, api.journal, cleanupID)
if terminal.State != operation.StateSucceeded || terminal.SupersedesOperationID != priorID {
t.Fatalf("cleanup terminal = %#v", terminal)
}
executor.mu.Lock()
if len(executor.requests) != 1 || executor.requests[0].SupersededOperation == nil ||
executor.requests[0].SupersededOperation.OperationID != priorID {
executor.mu.Unlock()
t.Fatalf("cleanup worker predecessor = %#v", executor.requests)
}
executor.mu.Unlock()
if state, err := api.journal.FenceState(priorID); err != nil || state != operation.FenceStateActive {
t.Fatalf("receive fence state=%q err=%v", state, err)
}
stale := request
stale.OperationID = "operation-peer-target-cleanup-stale"
stale.CleanupPlanID = strings.Repeat("b", 64)
staleBody, _ := json.Marshal(stale)
staleActor := actor
staleActor.OperationID = stale.OperationID
denied := api.request(t, http.MethodPost,
"/v1/vms/"+uuid+"/peer-migration/cleanup-target", staleBody, staleActor)
if denied.Code != http.StatusConflict ||
!bytes.Contains(denied.Body.Bytes(), []byte(`"code":"peer_migration_cleanup_plan_changed"`)) {
t.Fatalf("stale cleanup = %d %s", denied.Code, denied.Body.String())
}
}
func TestTrustedWorkerPeerMigrationTargetCleanupUsesParentPredecessor(t *testing.T) {
parent := newTestAPI(t)
child := newTestAPI(t)
const (
uuid = "123e4567-e89b-42d3-a456-426614174000"
migrationID = "0123456789abcdef0123456789abcdef"
priorID = "operation-peer-receive-worker-residue"
cleanupID = "operation-peer-target-worker-cleanup"
)
priorSpec := operation.Spec{
OperationID: priorID, ResourceID: uuid,
Action: "receive-peer-migration:" + migrationID,
Fingerprint: operation.Fingerprint(http.MethodPost,
"/v1/vms/"+uuid+"/peer-migration/receive", []byte(`{}`)),
Destructive: true,
LockKeys: hypervisor.ReceivePeerMigrationAdmission(hypervisor.ReceivePeerMigrationRequest{
CreateVMRequest: hypervisor.CreateVMRequest{UUID: uuid}, MigrationID: migrationID,
}).LockKeys,
}
if _, _, err := parent.journal.Execute(context.Background(), priorSpec, func(context.Context) (json.RawMessage, error) {
return nil, &worker.Error{Code: "result_unconfirmed", Message: "receive result unknown", EffectMayHaveApplied: true}
}); err == nil {
t.Fatal("seed receive failure unexpectedly succeeded")
}
prior, err := parent.journal.Get(priorID)
if err != nil {
t.Fatal(err)
}
planID := strings.Repeat("c", 64)
child.hv.inspectionResult = &hypervisor.OperationInspection{
OperationID: prior.OperationID, Action: prior.Action,
ResourceID: prior.ResourceID, RequestFingerprint: prior.Fingerprint,
OperationStatus: prior.State, CheckedAt: child.now.Unix(),
Outcome: hypervisor.InspectionOutcomeUncertain,
SummaryCode: hypervisor.InspectionSummaryCleanupRequired,
EffectResolution: hypervisor.InspectionEffectPartial,
Cleanup: hypervisor.OperationCleanupPlan{Available: true, PlanID: planID,
Steps: []string{hypervisor.CleanupStepDeleteManagedVolumes}},
}
requestBody, err := json.Marshal(hypervisor.CleanupPeerMigrationTargetRequest{
OperationID: cleanupID, MigrationID: migrationID,
PriorOperationID: priorID, CleanupPlanID: planID,
})
if err != nil {
t.Fatal(err)
}
actor := auth.Actor{
UserID: 1, IsAdmin: true, IsSuperAdmin: true, CanManage: true,
OperationID: cleanupID,
}
request := httptest.NewRequest(http.MethodPost,
"/v1/vms/"+uuid+"/peer-migration/cleanup-target", bytes.NewReader(requestBody))
response := httptest.NewRecorder()
child.handler.ServeTrustedWorkerRequest(response, request, requestBody, actor, prior)
if response.Code != http.StatusOK {
t.Fatalf("trusted cleanup response = %d %s", response.Code, response.Body.String())
}
terminal, err := child.journal.Get(cleanupID)
if err != nil {
t.Fatal(err)
}
if terminal.State != operation.StateSucceeded || terminal.SupersedesOperationID != "" {
t.Fatalf("trusted cleanup terminal = %#v", terminal)
}
child.hv.mu.Lock()
defer child.hv.mu.Unlock()
if child.hv.cleanupPeerTargetCalls != 1 ||
child.hv.cleanupPeerTargetLast.PriorOperation.OperationID != priorID {
t.Fatalf("trusted cleanup calls=%d request=%#v",
child.hv.cleanupPeerTargetCalls, child.hv.cleanupPeerTargetLast)
}
}
func TestCreateBackupUsesDedicatedAsyncActionAndHeavyAdmission(t *testing.T) {
api := newTestAPI(t)
const (

@ -457,7 +457,16 @@ func runOperationWorker(configPath, journalPath, receiptPath string) error {
ContentLength: int64(len(request.Body)), Host: "agent.worker",
}).WithContext(ctx)
responseCapture := newBoundedWorkerResponse(worker.MaxWorkerResponse)
handler.ServeTrustedWorkerRequest(responseCapture, httpRequest, []byte(request.Body), request.Actor)
if request.SupersededOperation != nil {
handler.ServeTrustedWorkerRequest(
responseCapture, httpRequest, []byte(request.Body), request.Actor,
*request.SupersededOperation,
)
} else {
handler.ServeTrustedWorkerRequest(
responseCapture, httpRequest, []byte(request.Body), request.Actor,
)
}
response := waitForWorkerTerminal(ctx, journal, request, responseCapture)
if err := worker.WriteReceipt(receiptPath, response); err != nil {

@ -172,6 +172,17 @@ func ReceivePeerMigrationAdmission(request ReceivePeerMigrationRequest) Mutation
}}
}
func CleanupPeerMigrationTargetAdmission(uuid, migrationID string) MutationAdmission {
uuid = strings.ToLower(uuid)
return MutationAdmission{HeavyIO: true, LockKeys: []string{
domainLockKey(uuid), vmManagedVolumesLockKey(uuid),
volumeLockKey(managedStoragePoolName, rootVolumeName(uuid)),
volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)),
nwfilterLockKey(networkFilterName(uuid)),
catalogLockKey("peer-migration", migrationID),
}}
}
func DeleteSnapshotAdmission(uuid, snapshotKey string) MutationAdmission {
uuid = strings.ToLower(uuid)
return MutationAdmission{LockKeys: []string{

@ -602,6 +602,12 @@ func (m *managedLibvirt) CleanupPeerMigration(ctx context.Context, uuid string,
})
}
func (m *managedLibvirt) CleanupPeerMigrationTarget(ctx context.Context, uuid string, request CleanupPeerMigrationTargetRequest) (DeleteVMResult, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (DeleteVMResult, error) {
return h.CleanupPeerMigrationTarget(ctx, uuid, request)
})
}
func (m *managedLibvirt) DeleteVM(ctx context.Context, uuid string, request DeleteVMRequest) (DeleteVMResult, error) {
return managedCall(m, ctx, true, func(h Hypervisor) (DeleteVMResult, error) {
return h.DeleteVM(ctx, uuid, request)

@ -169,13 +169,24 @@ func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record opera
var resolution operation.Resolution
var cleanup operationCleanupInspection
switch {
case strings.HasPrefix(record.Action, "receive-peer-migration:"):
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil)
if err != nil {
return OperationInspection{}, err
}
if len(plan.steps) != 0 {
cleanup = plan
break
}
resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, true, false)
case operation.IsCreateLikeAction(record.Action):
absent, err := l.deleteArtifactsAbsent(record.ResourceID)
if err != nil {
return OperationInspection{}, err
}
resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, absent, l.traffic.contains(record.ResourceID))
case record.Action == "delete-vm" || record.Action == "force-delete-vm":
case record.Action == "delete-vm" || record.Action == "force-delete-vm" ||
strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"):
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil)
if err != nil {
return OperationInspection{}, err
@ -210,6 +221,12 @@ func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePres
func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record operation.Record,
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection, error) {
if strings.HasPrefix(record.Action, "cleanup-peer-migration-target:") {
// This successor is authorized only to clean an absent target. A domain
// appearing before or after dispatch is never converted into a delete
// plan by reconciliation.
return operation.Resolution{}, operationCleanupInspection{}, nil
}
if record.Action == "delete-vm" || record.Action == "force-delete-vm" {
_, managed, err := requireManagedDomain(domain)
if err != nil {
@ -1596,6 +1613,15 @@ func operationEffectProjection(record operation.Record, raw []byte) json.RawMess
return nil
}
projection = value
case strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"):
parts := strings.Split(record.Action, ":")
var value DeleteVMResult
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
!decodeOperationEffectResult(raw, &value) || !value.Deleted ||
!value.ManagedArtifactsAbsent || value.Forced {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "activate-peer-migration:") ||
strings.HasPrefix(record.Action, "rollback-peer-migration:"):
parts := strings.Split(record.Action, ":")

@ -547,6 +547,91 @@ func (l *Libvirt) CleanupPeerMigration(ctx context.Context, uuid string,
return result, nil
}
// CleanupPeerMigrationTarget removes only UUID-owned artifacts left by one
// failed target receive. It never destroys or undefines a domain. The exact
// inventory is inspected twice: once by the API before Journal admission and
// again here under the successor's resource leases, closing the TOCTOU gap.
func (l *Libvirt) CleanupPeerMigrationTarget(ctx context.Context, uuid string,
request CleanupPeerMigrationTargetRequest) (_ DeleteVMResult, resultErr error) {
mutationStarted := false
defer func() {
if !mutationStarted {
resultErr = operation.MarkNotApplied(resultErr)
}
}()
if err := ValidateCleanupPeerMigrationTargetRequest(request); err != nil {
return DeleteVMResult{}, err
}
prior := request.PriorOperation
expectedAction := "receive-peer-migration:" + request.MigrationID
if prior.OperationID != request.PriorOperationID || prior.ResourceID != uuid ||
prior.Action != expectedAction || prior.State != operation.StateFailed ||
!prior.EffectMayHaveApplied || prior.SafeToResubmit || prior.ExecutionFenced {
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_cleanup_not_authorized", "peer migration target cleanup is not authorized by the prior receive", nil)
}
domain, err := l.domainByUUID(ctx, uuid)
if err == nil {
_ = domain.Free()
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_target_domain_present", "peer migration target cleanup refuses to remove an existing domain", nil)
}
if !errors.Is(err, ErrNotFound) {
return DeleteVMResult{}, err
}
plan, err := l.inspectDeleteCleanupSteps(ctx, uuid, nil, nil)
if err != nil {
return DeleteVMResult{}, err
}
if len(plan.steps) == 0 || len(plan.evidence) == 0 ||
!hashEqualsFold(request.CleanupPlanID,
operationInspectionToken(prior, "cleanup", plan.tokenEvidence())) {
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_changed", "peer migration target cleanup plan changed; inspect again", nil)
}
for _, step := range plan.steps {
switch step {
case CleanupStepDeleteManagedVolumes, CleanupStepDeleteSeedVolume,
CleanupStepUndefineNetworkFilter, CleanupStepDeleteNVRAM,
CleanupStepRemoveTrafficState:
default:
return DeleteVMResult{}, NewError(ErrConflict, "peer_migration_cleanup_plan_invalid", "peer migration target cleanup plan contains an unsafe step", nil)
}
}
mutationStarted = true
completionCtx := context.WithoutCancel(ctx)
if err := l.cleanupDeletedVMArtifacts(completionCtx, uuid); err != nil {
return DeleteVMResult{}, err
}
store, err := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
)
if err != nil {
return DeleteVMResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target state cleanup is incomplete", err,
))
}
if err := store.RemoveTarget(request.MigrationID); err != nil {
return DeleteVMResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target staging cleanup is incomplete", err,
))
}
if err := store.RemoveTargetReceipt(request.MigrationID); err != nil {
return DeleteVMResult{}, operation.MarkUncertain(NewError(
ErrUnavailable, "peer_migration_state_cleanup_pending", "peer migration target receipt cleanup is incomplete", err,
))
}
absent, err := l.deleteArtifactsAbsent(uuid)
if err != nil || !absent {
return DeleteVMResult{}, operation.MarkUncertain(errors.Join(err,
NewError(ErrConflict, "peer_migration_target_cleanup_unconfirmed", "peer migration target artifacts are still present", nil),
))
}
return DeleteVMResult{Deleted: true, ManagedArtifactsAbsent: true}, nil
}
func hashEqualsFold(left, right string) bool {
return len(left) == len(right) && strings.EqualFold(left, right)
}
func (l *Libvirt) createPeerMigrationVolume(ctx context.Context, _ string, volumeName,
path string, artifact peermigration.Artifact, finalCapacity uint64) (*libvirt.StorageVol, error) {
if finalCapacity < artifact.VirtualSizeBytes {

@ -104,3 +104,14 @@ func ValidateCleanupPeerMigrationRequest(request CleanupPeerMigrationRequest) er
}
return nil
}
func ValidateCleanupPeerMigrationTargetRequest(request CleanupPeerMigrationTargetRequest) error {
if !operationIdentifierPattern.MatchString(request.OperationID) ||
!operationIdentifierPattern.MatchString(request.PriorOperationID) ||
request.OperationID == request.PriorOperationID ||
!catalogIDPattern.MatchString(request.MigrationID) ||
!sha256Pattern.MatchString(strings.ToLower(request.CleanupPlanID)) {
return NewError(ErrInvalid, "invalid_peer_migration_target_cleanup", "peer migration target cleanup identity is invalid", nil)
}
return nil
}

@ -329,6 +329,18 @@ type CleanupPeerMigrationRequest struct {
RemoveManagedDisks bool `json:"remove_managed_disks"`
}
// CleanupPeerMigrationTargetRequest authorizes one narrowly scoped cleanup of
// an interrupted target receive. PriorOperation is injected by the Agent API
// from its immutable Journal and is never accepted from the wire. PlanID binds
// the mutation to one fresh, read-only inventory of UUID-owned artifacts.
type CleanupPeerMigrationTargetRequest struct {
OperationID string `json:"operation_id,omitempty"`
MigrationID string `json:"migration_id"`
PriorOperationID string `json:"prior_operation_id"`
CleanupPlanID string `json:"cleanup_plan_id"`
PriorOperation operation.Record `json:"-"`
}
// RecoverBackupRequest replaces the managed root disk of an existing VM with
// one immutable portable-backup object. Network and domain identity remain
// attached to the existing VM, while the Agent reduces its NoCloud seed to
@ -823,6 +835,7 @@ type Hypervisor interface {
ActivatePeerMigration(context.Context, string, ActivatePeerMigrationRequest) (PeerMigrationStateResult, error)
RollbackPeerMigration(context.Context, string, RollbackPeerMigrationRequest) (PeerMigrationStateResult, error)
CleanupPeerMigration(context.Context, string, CleanupPeerMigrationRequest) (DeleteVMResult, error)
CleanupPeerMigrationTarget(context.Context, string, CleanupPeerMigrationTargetRequest) (DeleteVMResult, error)
DeleteVM(context.Context, string, DeleteVMRequest) (DeleteVMResult, error)
ForceDeleteVM(context.Context, string, ForceDeleteVMRequest) (DeleteVMResult, error)
Power(context.Context, string, PowerRequest) (VMState, error)
@ -911,6 +924,9 @@ func (Unsupported) RollbackPeerMigration(context.Context, string, RollbackPeerMi
func (Unsupported) CleanupPeerMigration(context.Context, string, CleanupPeerMigrationRequest) (DeleteVMResult, error) {
return DeleteVMResult{}, unsupported("peer migration cleanup")
}
func (Unsupported) CleanupPeerMigrationTarget(context.Context, string, CleanupPeerMigrationTargetRequest) (DeleteVMResult, error) {
return DeleteVMResult{}, unsupported("peer migration target cleanup")
}
func (Unsupported) DeleteVM(context.Context, string, DeleteVMRequest) (DeleteVMResult, error) {
return DeleteVMResult{}, unsupported("VM delete")
}

@ -160,8 +160,9 @@ type Record struct {
// they are never exposed by the Agent API.
LockKeys []string `json:"-"`
HeavyIO bool `json:"-"`
// SupersedesOperationID is only valid for an explicitly acknowledged
// force-delete-vm successor of one matching graceful/force_required record.
// SupersedesOperationID is valid only for an explicitly reviewed successor:
// force-delete-vm after force_required, or a plan-bound PeerMigration target
// cleanup after one exact failed receive operation.
SupersedesOperationID string `json:"-"`
// InspectionReceiptRecovered and InspectionEvidenceUnavailable are
// ephemeral read-path markers. They are never persisted or exposed by the
@ -2655,6 +2656,27 @@ func validForceDeleteSupersession(spec Spec, lockKeys []string, predecessor Reco
return lockKeysOverlap(lockKeys, predecessorKeys)
}
func validPeerMigrationTargetCleanupSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
if spec.SupersedesOperationID == "" || spec.SupersedesOperationID == spec.OperationID ||
!cleanupPeerTargetActionPattern.MatchString(spec.Action) ||
predecessor.OperationID != spec.SupersedesOperationID ||
predecessor.ResourceID != spec.ResourceID ||
predecessor.State != StateFailed || !predecessor.EffectMayHaveApplied ||
predecessor.SafeToResubmit || predecessor.ExecutionFenced ||
!receivePeerActionPattern.MatchString(predecessor.Action) ||
strings.TrimPrefix(spec.Action, "cleanup-peer-migration-target:") !=
strings.TrimPrefix(predecessor.Action, "receive-peer-migration:") {
return false
}
predecessorKeys := effectiveLockKeys(predecessor.OperationID, predecessor.ResourceID, predecessor.Action, predecessor.LockKeys)
return lockKeysOverlap(lockKeys, predecessorKeys)
}
func validExplicitSupersession(spec Spec, lockKeys []string, predecessor Record) bool {
return validForceDeleteSupersession(spec, lockKeys, predecessor) ||
validPeerMigrationTargetCleanupSupersession(spec, lockKeys, predecessor)
}
// findResourceBlocker validates any explicit force-delete predecessor and
// reads only the compact blocker index for unresolved resource conflicts.
func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, bool, error) {
@ -2670,7 +2692,7 @@ func findResourceBlocker(tx *bolt.Tx, spec Spec, lockKeys []string) (Record, boo
return Record{}, false, ErrInvalidSupersession
}
predecessor, err := decodeRecord(encoded)
if err != nil || !validForceDeleteSupersession(spec, lockKeys, predecessor) {
if err != nil || !validExplicitSupersession(spec, lockKeys, predecessor) {
return Record{}, false, ErrInvalidSupersession
}
supersededOperationID = predecessor.OperationID
@ -2965,7 +2987,9 @@ func validateSpec(spec Spec) error {
return err
}
if spec.SupersedesOperationID != "" {
if !operationIDPattern.MatchString(spec.SupersedesOperationID) || spec.SupersedesOperationID == spec.OperationID || spec.Action != "force-delete-vm" {
if !operationIDPattern.MatchString(spec.SupersedesOperationID) ||
spec.SupersedesOperationID == spec.OperationID ||
(spec.Action != "force-delete-vm" && !cleanupPeerTargetActionPattern.MatchString(spec.Action)) {
return ErrInvalidSupersession
}
}

@ -1572,6 +1572,7 @@ func TestSystemRecoveryActionAllowlistIsExplicit(t *testing.T) {
"service-suspend",
"prepare-peer-migration:0123456789abcdef0123456789abcdef",
"receive-peer-migration:0123456789abcdef0123456789abcdef",
"cleanup-peer-migration-target:0123456789abcdef0123456789abcdef",
"activate-peer-migration:0123456789abcdef0123456789abcdef",
"rollback-peer-migration:0123456789abcdef0123456789abcdef",
"create-snapshot:0123456789abcdef0123456789abcdef:filesystem",
@ -1682,6 +1683,51 @@ func TestSuccessfulForceSuccessorOnlyRemovesPredecessorBlockerIndex(t *testing.T
}
}
func TestPeerMigrationTargetCleanupMayCrossOnlyItsExactReceiveFence(t *testing.T) {
j := openTestJournal(t)
migrationID := "0123456789abcdef0123456789abcdef"
predecessor := testSpec("operation-peer-receive-residue", "vm-peer-residue")
predecessor.Action = "receive-peer-migration:" + migrationID
predecessor.Fingerprint = Fingerprint("POST", "/v1/vms/"+predecessor.ResourceID+"/peer-migration/receive", []byte(`{}`))
if _, _, err := j.Execute(context.Background(), predecessor, func(context.Context) (json.RawMessage, error) {
return nil, testFailure{code: "result_unconfirmed", message: "receive result unknown", effect: true, safe: false}
}); err == nil {
t.Fatal("peer receive predecessor unexpectedly succeeded")
}
successor := testSpec("operation-peer-target-cleanup", predecessor.ResourceID)
successor.Action = "cleanup-peer-migration-target:" + migrationID
successor.Fingerprint = Fingerprint("POST", "/v1/vms/"+predecessor.ResourceID+"/peer-migration/cleanup-target", []byte(`{}`))
successor.SupersedesOperationID = predecessor.OperationID
if record, _, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"deleted":true,"managed_artifacts_absent":true,"forced":false}`), nil
}); err != nil || record.State != StateSucceeded {
t.Fatalf("peer target cleanup successor = %#v err=%v", record, err)
}
// Cleanup is an evidence-producing successor, not the effect verdict for
// the original receive. Keep that receive fence active until the panel
// performs one fresh absence inspection and resolves it as not applied.
if state, err := j.FenceState(predecessor.OperationID); err != nil || state != FenceStateActive {
t.Fatalf("peer receive fence state=%q err=%v", state, err)
}
followup := testSpec("operation-peer-after-cleanup", predecessor.ResourceID)
if _, _, err := j.Execute(context.Background(), followup, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"accepted":true}`), nil
}); !errors.Is(err, ErrResourceBlocked) {
t.Fatalf("original receive fence was released before reconciliation: %v", err)
}
wrong := testSpec("operation-peer-wrong-cleanup", predecessor.ResourceID)
wrong.Action = "cleanup-peer-migration-target:fedcba9876543210fedcba9876543210"
wrong.SupersedesOperationID = predecessor.OperationID
if _, _, err := j.Execute(context.Background(), wrong, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"deleted":true}`), nil
}); !errors.Is(err, ErrInvalidSupersession) {
t.Fatalf("mismatched migration cleanup supersession err=%v", err)
}
}
func TestInitializeRemovesLegacyStorageBlockerAndPreservesVMFence(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
j, err := Open(path)

@ -14,9 +14,10 @@ import (
const recoveryContractSchemaVersion = 1
var (
recoveryKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
restoreBackupActionPattern = regexp.MustCompile(`^restore-backup:[a-f0-9]{32}$`)
receivePeerActionPattern = regexp.MustCompile(`^receive-peer-migration:[a-f0-9]{32}$`)
recoveryKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
restoreBackupActionPattern = regexp.MustCompile(`^restore-backup:[a-f0-9]{32}$`)
receivePeerActionPattern = regexp.MustCompile(`^receive-peer-migration:[a-f0-9]{32}$`)
cleanupPeerTargetActionPattern = regexp.MustCompile(`^cleanup-peer-migration-target:[a-f0-9]{32}$`)
)
//go:embed recovery_contract.json

@ -112,6 +112,19 @@
"not_applied"
]
},
{
"kind": "instance_migration_target_clean",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [],
"action_patterns": [
"^cleanup-peer-migration-target:[a-f0-9]{32}$"
],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "instance_migration_rollback",
"recovery_mode": "generic",

@ -13,6 +13,7 @@ import (
"strings"
"git.luoli.live/root/virt/agent/auth"
"git.luoli.live/root/virt/agent/operation"
)
const (
@ -38,6 +39,10 @@ type Request struct {
Actor auth.Actor `json:"actor"`
OperationID string `json:"operation_id"`
Fingerprint string `json:"fingerprint"`
// SupersededOperation is a non-secret immutable snapshot injected only by
// the parent Agent after it authorizes an exceptional successor. The child
// uses a separate Journal, so it cannot load the predecessor itself.
SupersededOperation *operation.Record `json:"superseded_operation,omitempty"`
}
// RequestIdentity is the non-secret, immutable subset persisted beside an
@ -119,6 +124,17 @@ func (r Request) Validate() error {
if r.Actor.OperationID == "" || r.Actor.OperationID != r.OperationID {
return errors.New("worker actor operation ID does not match request")
}
if r.SupersededOperation != nil {
prior := r.SupersededOperation
if prior.OperationID == "" || prior.OperationID == r.OperationID ||
!operation.ValidOperationIdentity(prior.Action, prior.ResourceID, prior.Fingerprint) ||
prior.State != operation.StateFailed || !prior.EffectMayHaveApplied ||
prior.SafeToResubmit || prior.Tombstone ||
!(strings.HasSuffix(r.RequestTarget, "/force-delete") ||
strings.HasSuffix(r.RequestTarget, "/peer-migration/cleanup-target")) {
return errors.New("worker superseded operation is invalid")
}
}
if Fingerprint(r.Method, r.RequestTarget, []byte(r.Body)) != r.Fingerprint {
return errors.New("worker request fingerprint does not match target and body")
}

@ -807,6 +807,9 @@ final class CloudAdminOperationRecoveryService
$dto['error'] = self::INSPECTION_ERROR_LABELS[$errorCode]
?? '宿主机操作失败(错误码已记录)';
}
if (($value['cleanup']['available'] ?? false) === true) {
$dto['cleanup'] = $value['cleanup'];
}
if ($includeProjection && $projection !== null) {
$dto['effect_projection'] = $projection;
}
@ -997,7 +1000,8 @@ final class CloudAdminOperationRecoveryService
}
if (in_array($kind,
['delete', 'delete_force', 'expiry_delete',
'instance_migration_cleanup'], true)) {
'instance_migration_cleanup',
'instance_migration_target_clean'], true)) {
return self::DELETE_PROJECTION;
}
if (in_array($kind, [
@ -1098,7 +1102,8 @@ final class CloudAdminOperationRecoveryService
'network_change' => ['uuid', 'network_changed'],
'reinstall' => ['uuid', 'reinstall_committed'],
'delete', 'delete_force', 'expiry_delete',
'instance_migration_cleanup' => [
'instance_migration_cleanup',
'instance_migration_target_clean' => [
'deleted', 'managed_artifacts_absent', 'forced',
],
'instance_migration_rollback', 'instance_migration_activate' =>

@ -920,6 +920,14 @@ final class CloudAutomaticOperationRecoveryService
}
$resolution = (string) ($inspection['effect_resolution'] ?? '');
if ($resolution === 'partial'
&& (string) $job->kind === 'instance_migration_transfer'
&& CloudInstanceMigrationService::ensureTargetResidueCleanup(
$job,
$inspection
)) {
return self::pending('recovery_target_cleanup_pending');
}
if (in_array($resolution, ['partial', 'inconclusive'], true)) {
return self::pending(
$resolution === 'partial'

@ -25,6 +25,156 @@ final class CloudInstanceMigrationService
'attention_required',
];
/**
* Queue one plan-bound cleanup for residue left by an interrupted target
* receive. The original receive remains fenced and is never replayed; its
* ordinary automatic recovery will prove absence and repair every PHP
* placement/migration ledger after this successor succeeds.
*/
public static function ensureTargetResidueCleanup(
CloudJob $receiveJob,
array $inspection
): bool {
$cleanup = $inspection['cleanup'] ?? null;
$planId = is_array($cleanup) ? ($cleanup['plan_id'] ?? null) : null;
$steps = is_array($cleanup) ? ($cleanup['steps'] ?? null) : null;
$allowedSteps = [
'delete_managed_volumes', 'delete_seed_volume',
'undefine_network_filter', 'delete_nvram',
'remove_traffic_state',
];
if ((string) $receiveJob->kind !== 'instance_migration_transfer'
|| ($cleanup['available'] ?? false) !== true
|| !is_string($planId)
|| preg_match('/^[a-f0-9]{64}$/D', strtolower($planId)) !== 1
|| !is_array($steps) || !array_is_list($steps) || $steps === []
|| array_diff($steps, $allowedSteps) !== []
|| count(array_unique($steps)) !== count($steps)) {
return false;
}
return $receiveJob->getConnection()->transaction(
static function () use ($receiveJob, $planId, $steps): bool {
$lockedJob = CloudJob::where(
'id', (int) $receiveJob->id
)->lockForUpdate()->first();
$migration = CloudInstanceMigration::where(
'target_receive_job_id', (int) $receiveJob->id
)->lockForUpdate()->first();
$instance = $migration === null ? null : CloudInstance::where(
'id', (int) $migration->instance_id
)->lockForUpdate()->first();
$restore = $migration === null ? null : CloudBackupRestore::where(
'id', (int) $migration->restore_id
)->lockForUpdate()->first();
if ($lockedJob === null || $migration === null
|| $instance === null || $restore === null
|| (string) $lockedJob->kind
!== 'instance_migration_transfer'
|| (string) $lockedJob->phase !== 'terminal'
|| (string) $lockedJob->outcome !== 'failed'
|| (string) $lockedJob->effect_resolution !== 'unknown'
|| (int) $lockedJob->effect_may_have_applied !== 1
|| (int) $lockedJob->safe_to_resubmit !== 0
|| (int) $lockedJob->host_id
!== (int) $migration->target_host_id
|| (int) $lockedJob->instance_id !== (int) $instance->id
|| (int) $lockedJob->backup_restore_id
!== (int) $restore->id
|| (int) $restore->target_host_id
!== (int) $migration->target_host_id
|| !in_array((string) $restore->state,
['restoring', 'uncertain'], true)
|| !hash_equals((string) $lockedJob->operation_id,
(string) $restore->operation_id)) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标残留清理账本不一致', 409
);
}
$priorCleanups = CloudJob::where(
'parent_job_id', (int) $lockedJob->id
)->where('kind', 'instance_migration_target_clean')
->orderBy('id')->lockForUpdate()->get();
foreach ($priorCleanups as $priorCleanup) {
if ((string) $priorCleanup->phase !== 'terminal'
|| (string) $priorCleanup->outcome === 'succeeded'
|| (string) $priorCleanup->effect_resolution
!== 'not_applied') {
return true;
}
}
if (count($priorCleanups) >= 3) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标残留自动清理已达到安全重试上限',
409
);
}
$generation = count($priorCleanups) + 1;
$target = '/v1/vms/'
. rawurlencode((string) $migration->target_uuid)
. '/peer-migration/cleanup-target';
$body = [
'migration_id' => (string) $migration->migration_key,
'prior_operation_id' =>
(string) $lockedJob->operation_id,
'cleanup_plan_id' => strtolower($planId),
];
[$cleanupJob, $created] = CloudJobService::
reserveStandaloneAgentMutation(
[
'user_id' => (int) $lockedJob->user_id,
'request_idempotency_key' =>
'migration-target-clean-'
. (string) $migration->migration_key
. '-' . $generation,
],
(int) $migration->target_host_id,
'instance_migration_target_clean',
'POST',
$target,
$body,
[
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'maintenance_migration_id' =>
(int) $migration->id,
'prior_job_id' => (int) $lockedJob->id,
'cleanup_plan_id' => strtolower($planId),
'cleanup_steps' => array_values($steps),
],
[
'migration_id' => (int) $migration->id,
'generation' => $generation,
'prior_job_id' => (int) $lockedJob->id,
'prior_operation_id' =>
(string) $lockedJob->operation_id,
'cleanup_plan_id' => strtolower($planId),
'cleanup_steps' => array_values($steps),
],
null,
'/v1/instance-maintenance-migrations/'
. (int) $migration->id
. '/target-residue-cleanup/' . $generation
);
if (!$created) {
return true;
}
$cleanupJob->parent_job_id = (int) $lockedJob->id;
$cleanupJob->instance_id = (int) $instance->id;
$cleanupJob->backup_restore_id =
(int) $migration->restore_id;
$cleanupJob->save();
self::setState(
$migration,
'attention_required',
'target_residue_cleanup_running',
'reconcile_transfer'
);
return true;
},
3
);
}
public static function start(
CloudInstance $instance,
array $actor,

@ -73,6 +73,8 @@ final class CloudJobService
'Agent 回执与宿主机证据暂不一致;资源保持安全锁定,系统仍会继续自动核对。',
'recovery_evidence_partial' =>
'宿主机目前只能确认部分操作结果;资源保持安全锁定,系统仍会继续自动核对。',
'recovery_target_cleanup_pending' =>
'系统正在自动清理维护迁移目标机残留;原迁移不会重放,清理后会继续核对并修复迁移账本。',
'recovery_evidence_inconclusive' =>
'宿主机证据暂不足以确认操作结果;资源保持安全锁定,系统仍会继续自动核对。',
'recovery_evidence_invalid' =>

@ -29,6 +29,7 @@ final class CloudOperationCatalog
'backup_recover' => '从云备份恢复实例',
'instance_migration_prepare' => '准备维护迁移源实例',
'instance_migration_transfer' => '传输维护迁移数据',
'instance_migration_target_clean' => '清理维护迁移目标残留',
'instance_migration_cleanup' => '清理维护迁移源副本',
'instance_migration_rollback' => '恢复维护迁移源实例',
'instance_migration_activate' => '恢复维护迁移目标状态',
@ -77,6 +78,7 @@ final class CloudOperationCatalog
'backup_recover',
'instance_migration_prepare',
'instance_migration_transfer',
'instance_migration_target_clean',
'instance_migration_cleanup',
'instance_migration_rollback',
'instance_migration_activate',

@ -509,6 +509,7 @@ final class CloudOperationErrorCatalog
'backup_recover' => '从云备份恢复实例失败;任务已结束且不会自动重试',
'instance_migration_prepare' => '维护迁移源准备失败;请按迁移恢复出口处理',
'instance_migration_transfer' => '维护迁移传输失败;请按迁移恢复出口处理',
'instance_migration_target_clean' => '维护迁移目标残留清理失败;原迁移仍保持锁定并继续核对',
'instance_migration_cleanup' => '维护迁移源清理失败;目标实例不受影响',
'instance_migration_rollback' => '维护迁移源恢复失败;请按迁移恢复出口处理',
'instance_migration_activate' => '维护迁移目标状态恢复失败;请按迁移恢复出口处理',

@ -63,6 +63,13 @@ final class CloudOperationRecoveryContract
'action_patterns' => ['^receive-peer-migration:[a-f0-9]{32}$'],
'system_resolutions' => ['applied', 'not_applied'],
],
'instance_migration_target_clean' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => [],
'action_patterns' => ['^cleanup-peer-migration-target:[a-f0-9]{32}$'],
'system_resolutions' => ['applied', 'not_applied'],
],
'instance_migration_rollback' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',

@ -263,6 +263,7 @@ final class CloudOperationRecoveryIdentity
$kind = (string) $job->kind;
if (in_array($kind,
['instance_migration_prepare', 'instance_migration_transfer',
'instance_migration_target_clean',
'instance_migration_cleanup', 'instance_migration_rollback',
'instance_migration_activate'], true)
&& isset($payload['maintenance_migration_id'])) {
@ -432,6 +433,48 @@ final class CloudOperationRecoveryIdentity
return ['receive-peer-migration:'
. (string) $migration->migration_key, $uuid];
}
if ((string) $job->kind === 'instance_migration_target_clean') {
$prior = $job->parent_job_id === null ? null : CloudJob::find(
(int) $job->parent_job_id
);
if ($prior === null
|| (int) $migration->target_receive_job_id
!== (int) $prior->id
|| (string) $prior->kind
!== 'instance_migration_transfer'
|| (int) $migration->target_host_id !== (int) $job->host_id
|| preg_match(self::UUID_PATTERN,
strtolower((string) $migration->target_uuid)) !== 1) {
throw new CloudRecoveryInstanceStateConflict(
'PeerMigration 目标残留清理身份不一致', 409
);
}
$uuid = strtolower((string) $migration->target_uuid);
self::expectRequest($method, $target, 'POST',
'/v1/vms/' . rawurlencode($uuid)
. '/peer-migration/cleanup-target');
self::assertAllowedBody($body, [
'operation_id', 'migration_id', 'prior_operation_id',
'cleanup_plan_id',
]);
if (($body['migration_id'] ?? null)
!== (string) $migration->migration_key
|| !hash_equals((string) $prior->operation_id,
(string) ($body['prior_operation_id'] ?? ''))
|| !is_string($body['cleanup_plan_id'] ?? null)
|| preg_match('/^[a-f0-9]{64}$/D',
(string) $body['cleanup_plan_id']) !== 1
|| !hash_equals((string) ($payload['cleanup_plan_id'] ?? ''),
(string) $body['cleanup_plan_id'])
|| (int) ($payload['prior_job_id'] ?? 0)
!== (int) $prior->id) {
self::invalidPayload(
'PeerMigration 目标残留清理确认无效'
);
}
return ['cleanup-peer-migration-target:'
. (string) $migration->migration_key, $uuid];
}
if ((string) $job->kind === 'instance_migration_activate') {
if ((int) $migration->activate_job_id !== (int) $job->id
|| (int) $migration->target_host_id !== (int) $job->host_id

@ -833,6 +833,36 @@ namespace {
null,
9002
);
$migrationTargetClean = $insertJob(
'instance_migration_target_clean',
static fn (int $id, string $operationId): array => [
'method' => 'POST',
'request_target' => '/v1/vms/'
. rawurlencode($migrationTargetUuid)
. '/peer-migration/cleanup-target',
'body' => [
'operation_id' => $operationId,
'migration_id' => $migrationKey,
'prior_operation_id' =>
$migrationTransfer['operation_id'],
'cleanup_plan_id' => str_repeat('c', 64),
],
'owner_id' => $ownerId,
'can_manage' => true,
'maintenance_migration_id' => 1,
'prior_job_id' => $migrationTransfer['id'],
'cleanup_plan_id' => str_repeat('c', 64),
'cleanup_steps' => ['delete_managed_volumes'],
],
$instanceId,
null,
$migrationTransfer['id'],
null,
'agent',
$ownerId,
null,
9002
);
$migrationCleanup = $insertJob(
'instance_migration_cleanup',
static fn (int $id, string $operationId): array => [
@ -1017,6 +1047,14 @@ namespace {
'staged_powered_off' => true,
]
);
$addFixture(
'instance_migration_target_clean',
'target-residue-cleanup',
$migrationTargetClean,
'cleanup-peer-migration-target:' . $migrationKey,
$migrationTargetUuid,
$deleteProjection
);
$addFixture(
'instance_migration_cleanup',
'source-cleanup',
@ -2181,6 +2219,34 @@ namespace {
CloudAdminOperationRecoveryService::class,
'inspection'
);
$transferIdentity = CloudOperationRecoveryIdentity::expectedFromPayload(
$migrationTransfer['job'],
$migrationTransfer['payload']
);
$cleanupInspection = $inspectionMethod->invoke(null, [
'operation_id' => $migrationTransfer['operation_id'],
'action' => $transferIdentity['action'],
'resource_id' => $transferIdentity['resource_id'],
'request_fingerprint' =>
$transferIdentity['request_fingerprint'],
'operation_status' => 'failed',
'checked_at' => time(),
'outcome' => 'uncertain',
'summary_code' => 'cleanup_required',
'effect_resolution' => 'partial',
'fence_state' => 'active',
'fence_resolution_required' => true,
'abort_job' => ['available' => false],
'cleanup' => [
'available' => true,
'plan_id' => str_repeat('c', 64),
'steps' => ['delete_managed_volumes'],
],
], $migrationTransfer['operation_id'],
'instance_migration_transfer', true);
$assert(($cleanupInspection['cleanup']['plan_id'] ?? null)
=== str_repeat('c', 64),
'Validated target cleanup evidence was dropped before recovery.');
$upgradeNotAppliedRejected = false;
try {
$inspectionMethod->invoke(null, [
@ -2727,5 +2793,187 @@ namespace {
)->where('id', '<>', $migrationRollback['id'])->count() === 1,
'Abort prepare created more than one compensating rollback task.');
// A partial target-receive inspection queues one independent, plan-bound
// cleanup generation. Re-entering automatic recovery must reuse that
// durable child rather than replaying the receive or creating duplicates.
$nextJobId = max(
$nextJobId,
(int) Capsule::table('cloud_jobs')->max('id') + 1
);
$residueMigrationKey = str_repeat('9', 32);
$residueTargetUuid = '88888888-8888-4888-8888-888888888888';
$residueTransfer = $insertJob(
'instance_migration_transfer',
static fn (int $id, string $operationId): array => [
'method' => 'POST',
'request_target' => '/v1/vms/'
. rawurlencode($residueTargetUuid)
. '/peer-migration/receive',
'body' => [
'operation_id' => $operationId,
'public_ip_cidr' => '203.0.113.30/24',
],
'owner_id' => $ownerId,
'can_manage' => true,
'maintenance_migration_id' => 2,
],
$instanceId,
null,
null,
null,
'agent',
$ownerId,
null,
9012
);
Capsule::table('cloud_backup_restores')->insert([
'id' => 9012,
'restore_key' => str_repeat('f', 32),
'backup_id' => null,
'target_host_id' => $hostId,
'target_instance_id' => null,
'reservation_id' => 9004,
'job_id' => $residueTransfer['id'],
'operation_id' => $residueTransfer['operation_id'],
'new_uuid' => $residueTargetUuid,
'new_name' => 'residue-target',
'state' => 'uncertain',
'error_code' => 'operation_effect_unknown',
'ready_at' => null,
'created_at' => $now - 120,
'updated_at' => $now,
]);
Capsule::table('cloud_instance_migrations')->insert([
'id' => 2,
'migration_key' => $residueMigrationKey,
'instance_id' => $instanceId,
'restore_id' => 9012,
'source_host_id' => $hostId,
'source_uuid' => $instanceUuid,
'source_reservation_id' => 9003,
'target_host_id' => $hostId,
'target_uuid' => $residueTargetUuid,
'target_reservation_id' => 9004,
'source_prepare_job_id' => null,
'target_receive_job_id' => $residueTransfer['id'],
'firewall_job_id' => null,
'activate_job_id' => null,
'rollback_job_id' => null,
'cleanup_job_id' => null,
'cleanup_mode' => 'delete',
'state' => 'attention_required',
'recovery_action' => 'reconcile_transfer',
'retry_generation' => 0,
'source_endpoint' => '["https://192.0.2.10:9443"]',
'peer_trust_id' => str_repeat('1', 64),
'manifest_digest' => str_repeat('2', 64),
'source_was_running' => true,
'source_autostart_enabled' => true,
'source_service_status' => 'active',
'source_network_status' => 'active',
'error_code' => 'operation_effect_unknown',
'cutover_at' => null,
'completed_at' => null,
'created_at' => $now - 120,
'updated_at' => $now,
]);
Capsule::table('cloud_resource_reservations')->where('id', 9004)
->update([
'backup_restore_id' => 9012,
'operation_id' => $residueTransfer['operation_id'],
'instance_id' => $instanceId,
'state' => 'uncertain',
'updated_at' => $now,
]);
Capsule::table('cloud_public_ips')->where('id', 9007)->update([
'state' => 'uncertain',
'instance_id' => null,
'operation_id' => $residueTransfer['operation_id'],
'updated_at' => $now,
]);
$residueInspection = [
'cleanup' => [
'available' => true,
'plan_id' => str_repeat('3', 64),
'steps' => ['delete_managed_volumes'],
],
];
$assert(CloudInstanceMigrationService::ensureTargetResidueCleanup(
CloudJob::findOrFail($residueTransfer['id']),
$residueInspection
), 'A safe target-residue plan did not queue cleanup.');
$targetCleanup = CloudJob::where(
'parent_job_id', $residueTransfer['id']
)->where('kind', 'instance_migration_target_clean')->first();
$targetCleanupPayload = $targetCleanup === null ? null
: CloudSecrets::decrypt(
(string) $targetCleanup->sealed_payload,
'job-payload'
);
$assert($targetCleanup !== null
&& (int) $targetCleanup->host_id === $hostId
&& (int) $targetCleanup->instance_id === $instanceId
&& ($targetCleanupPayload['body']['prior_operation_id'] ?? null)
=== $residueTransfer['operation_id']
&& ($targetCleanupPayload['body']['cleanup_plan_id'] ?? null)
=== str_repeat('3', 64)
&& str_ends_with(
(string) ($targetCleanupPayload['request_target'] ?? ''),
'/peer-migration/cleanup-target'
), 'Target-residue cleanup did not retain its exact plan and parent.');
$assert(CloudInstanceMigrationService::ensureTargetResidueCleanup(
CloudJob::findOrFail($residueTransfer['id']),
$residueInspection
) && CloudJob::where(
'parent_job_id', $residueTransfer['id']
)->where('kind', 'instance_migration_target_clean')->count() === 1,
'Repeated recovery created duplicate target-residue cleanups.');
// Model the next recovery pass after the independent cleanup succeeds and
// the original receive inspection proves the target is absent. The normal
// transfer finalizer must restore every target ledger to one retryable
// operation identity instead of leaving uncertain capacity or IP claims.
Capsule::table('cloud_jobs')->where('id', $targetCleanup->id)->update([
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'terminal_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_jobs')->where('id', $residueTransfer['id'])
->update([
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'not_applied',
'effect_resolution_reason' => 'system_inspection',
'updated_at' => $now,
]);
CloudInstanceMigrationService::finishJobLocked(
CloudJob::findOrFail($residueTransfer['id']),
'failed',
null,
$residueTransfer['payload']
);
$repairedMigration = CloudInstanceMigration::findOrFail(2);
$repairedRestore = Capsule::table('cloud_backup_restores')
->where('id', 9012)->first();
$repairedReservation = Capsule::table('cloud_resource_reservations')
->where('id', 9004)->first();
$repairedIp = Capsule::table('cloud_public_ips')
->where('id', 9007)->first();
$assert((string) $repairedMigration->state === 'attention_required'
&& (string) $repairedMigration->recovery_action === 'retry_transfer'
&& (string) $repairedMigration->error_code
=== 'peer_transfer_failed'
&& (string) $repairedRestore->state === 'reserved'
&& (string) $repairedRestore->error_code === 'peer_transfer_failed'
&& (string) $repairedReservation->state === 'held'
&& (string) $repairedIp->state === 'reserved'
&& $repairedIp->instance_id === null,
'Target cleanup did not repair migration, restore, capacity, and IP ledgers for a safe retry.');
echo "Cloud operation recovery matrix tests passed\n";
}

@ -56,6 +56,7 @@ namespace {
'instance_migration_cleanup',
'instance_migration_prepare',
'instance_migration_transfer',
'instance_migration_target_clean',
'instance_migration_rollback',
'instance_migration_activate',
'expiry_delete',
@ -81,8 +82,8 @@ namespace {
];
$assert(CloudOperationRecoveryPolicy::kinds() === $expectedKinds,
'The Agent leaf recovery policy changed without updating its contract.');
$assert(count(CloudOperationRecoveryPolicy::kinds()) === 30,
'The recovery policy contract must contain exactly 30 Agent leaf kinds.');
$assert(count(CloudOperationRecoveryPolicy::kinds()) === 31,
'The recovery policy contract must contain exactly 31 Agent leaf kinds.');
foreach ($expectedKinds as $kind) {
$assert(CloudOperationRecoveryPolicy::supports($kind),
'The recovery policy omitted ' . $kind . '.');

@ -634,7 +634,8 @@ namespace {
'instance_upgrade', 'firewall_apply', 'snapshot_create',
'snapshot_restore', 'snapshot_delete', 'backup_create',
'backup_restore', 'backup_recover', 'instance_migration_prepare',
'instance_migration_transfer', 'instance_migration_cleanup',
'instance_migration_transfer', 'instance_migration_target_clean',
'instance_migration_cleanup',
'instance_migration_rollback', 'instance_migration_activate', 'rescue',
], 'The instance operation log does not cover every settings operation.');
$assert(in_array((string) $settingsAfterDelete->id, $operationLogIds, true),

Loading…
Cancel
Save