From 63f365d847a43dc30d43dadcc52e2cc4da336d9b Mon Sep 17 00:00:00 2001 From: codex Date: Sun, 25 Oct 2026 14:09:09 +0800 Subject: [PATCH] fix(cloud): repair portable backup preflight --- agent/control/client.go | 6 +- agent/control/client_test.go | 7 +- agent/hypervisor/libvirt_backup.go | 35 ++++++++-- agent/hypervisor/libvirt_backup_test.go | 65 +++++++++++++++++++ agent/objecttransfer/client.go | 29 +++++++-- agent/objecttransfer/client_test.go | 46 +++++++++++++ agent/operation/journal.go | 23 ++++++- agent/operation/journal_test.go | 27 ++++++++ gateway/agent/completion.go | 4 +- gateway/agent/completion_test.go | 16 ++++- gateway/agent/registry.go | 8 ++- .../Cloud/CloudOperationCompletionService.php | 52 ++++++++++++--- .../Cloud/CloudOperationErrorCatalog.php | 1 + .../CloudOperationCompletionContractTest.php | 29 ++++++++- web/tests/CloudOperationErrorCatalogTest.php | 2 + 15 files changed, 320 insertions(+), 30 deletions(-) diff --git a/agent/control/client.go b/agent/control/client.go index fb7bf255..3a8ce8d5 100644 --- a/agent/control/client.go +++ b/agent/control/client.go @@ -91,6 +91,7 @@ type completionMessage struct { EffectMayHaveApplied bool `json:"effect_may_have_applied"` SafeToResubmit bool `json:"safe_to_resubmit"` ErrorCode string `json:"error_code,omitempty"` + CauseCode string `json:"cause_code,omitempty"` Result json.RawMessage `json:"result"` } @@ -105,8 +106,9 @@ func completionMessageFromReceipt(completion operation.CompletionReceipt) (compl Action: completion.Action, ResourceID: completion.ResourceID, RequestFingerprint: completion.RequestFingerprint, Status: completion.Status, Result: completion.Result, - ErrorCode: completion.ErrorCode, EffectMayHaveApplied: completion.EffectMayHaveApplied, - SafeToResubmit: completion.SafeToResubmit, + ErrorCode: completion.ErrorCode, CauseCode: completion.CauseCode, + EffectMayHaveApplied: completion.EffectMayHaveApplied, + SafeToResubmit: completion.SafeToResubmit, }, nil } diff --git a/agent/control/client_test.go b/agent/control/client_test.go index 481eeac2..ee110785 100644 --- a/agent/control/client_test.go +++ b/agent/control/client_test.go @@ -55,14 +55,17 @@ func TestCompletionMessageCarriesStrictImmutableOperationIdentity(t *testing.T) EventID: "completion_identity_123", OperationID: "operation_identity_123", Action: "delete-vm", ResourceID: "vm-completion-identity", RequestFingerprint: strings.Repeat("a", 64), - Status: "succeeded", Result: json.RawMessage(`{"done":true}`), + Status: "failed", Result: json.RawMessage("null"), + ErrorCode: "operation_failed", + CauseCode: "backup_disk_transition_active", } message, err := completionMessageFromReceipt(receipt) if err != nil { t.Fatal(err) } if message.Action != receipt.Action || message.ResourceID != receipt.ResourceID || - message.RequestFingerprint != receipt.RequestFingerprint { + message.RequestFingerprint != receipt.RequestFingerprint || + message.CauseCode != receipt.CauseCode { t.Fatalf("completion message identity = %#v", message) } diff --git a/agent/hypervisor/libvirt_backup.go b/agent/hypervisor/libvirt_backup.go index 712b135b..7a8d7cc8 100644 --- a/agent/hypervisor/libvirt_backup.go +++ b/agent/hypervisor/libvirt_backup.go @@ -21,9 +21,13 @@ func (l *Libvirt) CreateBackup(ctx context.Context, uuid string, request BackupR if err := ValidateBackupRequest(request); err != nil { return BackupResult{}, operation.MarkNotApplied(err) } + client := objecttransfer.New() + defer client.CloseIdleConnections() active, err := l.validateBackupSource(ctx, uuid, request.BackupKey) if err != nil { - return BackupResult{}, operation.MarkNotApplied(err) + return BackupResult{}, abortRejectedBackupPlan( + ctx, client.AbortMultipart, request.ObjectUpload, err, + ) } snapshotRequest := SnapshotRequest{ @@ -61,8 +65,6 @@ func (l *Libvirt) CreateBackup(ctx context.Context, uuid string, request BackupR )) } - client := objecttransfer.New() - defer client.CloseIdleConnections() upload, err := client.UploadFile(ctx, path, request.ObjectUpload) if err != nil { translated := translateObjectTransferError(err) @@ -103,6 +105,18 @@ func (l *Libvirt) CreateBackup(ctx context.Context, uuid string, request BackupR return result, nil } +func abortRejectedBackupPlan(ctx context.Context, + abort func(context.Context, objecttransfer.MultipartPlan) error, + plan objecttransfer.MultipartPlan, rejection error) error { + if cleanupErr := abort(ctx, plan); cleanupErr != nil { + // The VM and backup object were never mutated. Retain retry-safe + // semantics and the original preflight cause while keeping the bounded + // multipart cleanup failure in the private Agent error chain. + return operation.MarkNotApplied(errors.Join(rejection, cleanupErr)) + } + return operation.MarkNotApplied(rejection) +} + func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey string) (bool, error) { domain, err := l.domainByUUID(ctx, uuid) if err != nil { @@ -118,8 +132,8 @@ func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey stri if metadata.RescueTemplateID != "" { return false, NewError(ErrConflict, "backup_rescue_active", "VM backup is unavailable while rescue media is attached", nil) } - if metadata.PendingCleanupVolume != "" || metadata.DiskTransitionKind != "" { - return false, NewError(ErrConflict, "backup_disk_transition_active", "VM backup is unavailable during a disk transition", nil) + if err := validateBackupTransitionState(metadata); err != nil { + return false, err } for _, stored := range metadata.Snapshots { if stored.Key == backupKey { @@ -166,6 +180,17 @@ func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey stri return active, nil } +func validateBackupTransitionState(metadata *domainMetadataXML) error { + // Reinstall and snapshot restore deliberately retain their transition kind + // and operation ID after the old root has been deleted. Those markers prove + // a completed root switch during reconciliation; only a pending old root + // means that the disk transition is still active. + if metadata != nil && metadata.PendingCleanupVolume != "" { + return NewError(ErrConflict, "backup_disk_transition_active", "VM backup is unavailable during a disk transition", nil) + } + return nil +} + func validateSingleWritableRoot(definition *libvirtxml.Domain) error { if definition == nil || definition.Devices == nil { return NewError(ErrConflict, "backup_disk_layout_unsupported", "VM disk layout is unavailable for portable backup", nil) diff --git a/agent/hypervisor/libvirt_backup_test.go b/agent/hypervisor/libvirt_backup_test.go index 06d6e83e..1a29b5ca 100644 --- a/agent/hypervisor/libvirt_backup_test.go +++ b/agent/hypervisor/libvirt_backup_test.go @@ -3,9 +3,11 @@ package hypervisor import ( + "context" "errors" "testing" + "git.luoli.live/root/virt/agent/objecttransfer" "libvirt.org/go/libvirt" "libvirt.org/go/libvirtxml" ) @@ -99,3 +101,66 @@ func TestValidateStandaloneQCOW2RejectsBackingChain(t *testing.T) { t.Fatalf("backing-chain error = %v", err) } } + +func TestValidateBackupTransitionStateAllowsCompletedDiskTransitions(t *testing.T) { + tests := []struct { + name string + metadata *domainMetadataXML + wantError bool + }{ + { + name: "reinstall cleanup pending", + metadata: &domainMetadataXML{ + PendingCleanupVolume: "123e4567-e89b-42d3-a456-426614174000-root.qcow2", + DiskTransitionKind: "reinstall", + DiskTransitionID: "operation-reinstall-one", + }, + wantError: true, + }, + { + name: "completed reinstall marker", + metadata: &domainMetadataXML{ + DiskTransitionKind: "reinstall", + DiskTransitionID: "operation-reinstall-one", + }, + }, + { + name: "completed snapshot restore marker", + metadata: &domainMetadataXML{ + DiskTransitionKind: "snapshot_restore", + DiskTransitionID: "operation-snapshot-restore-one", + }, + }, + {name: "no transition", metadata: &domainMetadataXML{}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateBackupTransitionState(test.metadata) + if test.wantError && !errors.Is(err, ErrConflict) { + t.Fatalf("pending transition error = %v", err) + } + if !test.wantError && err != nil { + t.Fatalf("completed transition marker rejected: %v", err) + } + }) + } +} + +func TestAbortRejectedBackupPlanPreservesPreflightCause(t *testing.T) { + rejected := NewError(ErrConflict, "backup_disk_transition_active", + "VM backup is unavailable during a disk transition", nil) + called := false + err := abortRejectedBackupPlan( + context.Background(), + func(context.Context, objecttransfer.MultipartPlan) error { + called = true + return nil + }, + objecttransfer.MultipartPlan{}, + rejected, + ) + if !called || !errors.Is(err, rejected) { + t.Fatalf("called=%v err=%v", called, err) + } +} diff --git a/agent/objecttransfer/client.go b/agent/objecttransfer/client.go index 0baba504..1341a189 100644 --- a/agent/objecttransfer/client.go +++ b/agent/objecttransfer/client.go @@ -199,16 +199,18 @@ func (c *Client) UploadFile(ctx context.Context, path string, plan MultipartPlan } size, digest, before, err := hashFile(path) if err != nil { - return UploadResult{}, err + return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL, err) } partsNeeded := int((size + plan.PartSizeBytes - 1) / plan.PartSizeBytes) if size > plan.MaximumSizeBytes || partsNeeded > len(plan.PartURLs) || partsNeeded > MaxMultipartParts { - return UploadResult{}, transferError("backup_source_too_large", "backup source exceeds the multipart upload plan", false) + return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL, + transferError("backup_source_too_large", "backup source exceeds the multipart upload plan", false)) } file, err := os.Open(path) if err != nil { - return UploadResult{}, transferError("backup_source_unavailable", "backup source could not be opened", false) + return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL, + transferError("backup_source_unavailable", "backup source could not be opened", false)) } defer file.Close() @@ -357,6 +359,10 @@ func (c *Client) complete(ctx context.Context, capability string, parts []Comple } func (c *Client) failAndAbort(ctx context.Context, capability, code, message string) error { + return c.failAndAbortError(ctx, capability, transferError(code, message, false)) +} + +func (c *Client) failAndAbortError(ctx context.Context, capability string, failure error) error { // The caller's cancellation often caused the upload failure itself. Give // cleanup one independent, tightly bounded chance; never inherit a long or // already-expired operation deadline. @@ -365,7 +371,22 @@ func (c *Client) failAndAbort(ctx context.Context, capability, code, message str if err := c.abort(abortCtx, capability); err != nil { return transferError("upload_abort_uncertain", "backup upload failed and multipart cleanup is unconfirmed", true) } - return transferError(code, message, false) + return failure +} + +// AbortMultipart consumes only the bounded DELETE capability from a validated +// plan. It is used when higher-level VM preflight rejects the backup before a +// local snapshot or object upload can start. +func (c *Client) AbortMultipart(ctx context.Context, plan MultipartPlan) error { + if c == nil || c.http == nil { + return transferError("transfer_client_unavailable", "object transfer client is unavailable", false) + } + if err := ValidateMultipartPlan(plan); err != nil { + return err + } + abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + return c.abort(abortCtx, plan.AbortURL) } func (c *Client) abort(ctx context.Context, capability string) error { diff --git a/agent/objecttransfer/client_test.go b/agent/objecttransfer/client_test.go index 9dc48ff6..523c0199 100644 --- a/agent/objecttransfer/client_test.go +++ b/agent/objecttransfer/client_test.go @@ -138,6 +138,52 @@ func TestUploadFileAbortsAfterKnownPartFailure(t *testing.T) { } } +func TestUploadFileAbortsBeforeFirstPartWhenSourceIsUnavailable(t *testing.T) { + aborted := false + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Errorf("unexpected method before source validation: %s", r.Method) + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + aborted = true + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + missing := filepath.Join(t.TempDir(), "missing.qcow2") + _, err := testClient(server).UploadFile( + context.Background(), missing, multipartPlan(server.URL, 1, MinPartSizeBytes), + ) + if err == nil || IsAmbiguous(err) || !aborted { + t.Fatalf("err=%v ambiguous=%v aborted=%v", err, IsAmbiguous(err), aborted) + } +} + +func TestAbortMultipartUsesDeleteCapability(t *testing.T) { + aborted := false + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete || r.URL.Query().Get("abort") != "1" { + t.Errorf("unexpected abort request: %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusBadRequest) + return + } + aborted = true + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client := testClient(server) + if err := client.AbortMultipart( + context.Background(), multipartPlan(server.URL, 1, MinPartSizeBytes), + ); err != nil { + t.Fatalf("AbortMultipart: %v", err) + } + if !aborted { + t.Fatal("multipart abort capability was not used") + } +} + func TestUploadFileDoesNotAbortAmbiguousCompletion(t *testing.T) { path, contents := writeSource(t, int(MinPartSizeBytes)) aborted := false diff --git a/agent/operation/journal.go b/agent/operation/journal.go index f3df4afb..493063a7 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -215,6 +215,7 @@ var terminalEvidenceOpenTimeout = 2 * time.Second // state. Delivery timing is kept in a separate outbox record so an ACK can // never rewrite the operation or its receipt. type CompletionReceipt struct { + Version int `json:"version,omitempty"` EventID string `json:"id"` OperationID string `json:"operation_id"` Action string `json:"action"` @@ -223,6 +224,7 @@ type CompletionReceipt struct { Status string `json:"status"` Result json.RawMessage `json:"result"` ErrorCode string `json:"error_code,omitempty"` + CauseCode string `json:"cause_code,omitempty"` EffectMayHaveApplied bool `json:"effect_may_have_applied"` SafeToResubmit bool `json:"safe_to_resubmit"` CreatedAt time.Time `json:"-"` @@ -687,6 +689,7 @@ func completionReceiptMatchesRecordForInspection(eventID, requestedOperationID s receipt.RequestFingerprint != record.Fingerprint || receipt.EffectMayHaveApplied != record.EffectMayHaveApplied || receipt.SafeToResubmit != record.SafeToResubmit || + !completionCauseMatchesRecord(receipt, record) || len(receipt.Result) == 0 || !json.Valid(receipt.Result) { return false } @@ -1364,6 +1367,7 @@ func completionReceiptMatchesRecord(receipt CompletionReceipt, record Record) bo receipt.RequestFingerprint != record.Fingerprint || receipt.EffectMayHaveApplied != record.EffectMayHaveApplied || receipt.SafeToResubmit != record.SafeToResubmit || + !completionCauseMatchesRecord(receipt, record) || !completionResultSafe(receipt.Result) || !completionDeliveryFitsPanelLimit(receipt) { return false } @@ -1397,6 +1401,20 @@ func completionReceiptMatchesRecord(receipt CompletionReceipt, record Record) bo return completionResultsEqual(receipt.Result, record.Result) } +func completionCauseMatchesRecord(receipt CompletionReceipt, record Record) bool { + switch receipt.Version { + case 0: + // Legacy v1 receipts predate cause_code. They remain deliverable after + // an Agent upgrade, but cannot recover the diagnostic that was omitted. + return receipt.CauseCode == "" + case 2: + return (receipt.CauseCode == "" || validCauseCode(receipt.CauseCode)) && + receipt.CauseCode == record.CauseCode + default: + return false + } +} + func completionDeliveryFitsPanelLimit(receipt CompletionReceipt) bool { // The Gateway adds agent_id before POSTing to PHP. Marshal the exact public // envelope with the longest valid agent ID so every claimed receipt is @@ -1413,13 +1431,14 @@ func completionDeliveryFitsPanelLimit(receipt CompletionReceipt) bool { EffectMayHaveApplied bool `json:"effect_may_have_applied"` SafeToResubmit bool `json:"safe_to_resubmit"` ErrorCode string `json:"error_code"` + CauseCode string `json:"cause_code,omitempty"` Result json.RawMessage `json:"result"` }{ Type: "completion", ID: receipt.EventID, AgentID: "9999999999999999999", OperationID: receipt.OperationID, Action: receipt.Action, ResourceID: receipt.ResourceID, RequestFingerprint: receipt.RequestFingerprint, Status: receipt.Status, EffectMayHaveApplied: receipt.EffectMayHaveApplied, SafeToResubmit: receipt.SafeToResubmit, - ErrorCode: receipt.ErrorCode, Result: receipt.Result, + ErrorCode: receipt.ErrorCode, CauseCode: receipt.CauseCode, Result: receipt.Result, }) return err == nil && len(encoded) <= completionPanelMaximumBody } @@ -1689,10 +1708,12 @@ func syncCompletionReceipt(receipts, outbox *bolt.Bucket, previous, current Reco errorCode = current.Error.Code } receipt := CompletionReceipt{ + Version: 2, EventID: eventID, OperationID: current.OperationID, Action: current.Action, ResourceID: current.ResourceID, RequestFingerprint: current.Fingerprint, Status: status, Result: append(json.RawMessage(nil), current.Result...), ErrorCode: errorCode, + CauseCode: current.CauseCode, EffectMayHaveApplied: current.EffectMayHaveApplied, SafeToResubmit: current.SafeToResubmit, CreatedAt: current.UpdatedAt.UTC(), } diff --git a/agent/operation/journal_test.go b/agent/operation/journal_test.go index 176b926c..063b6d58 100644 --- a/agent/operation/journal_test.go +++ b/agent/operation/journal_test.go @@ -669,6 +669,33 @@ func TestCompletionReceiptIsImmutableAndAckOnlyClearsOutbox(t *testing.T) { } } +func TestFailedCompletionReceiptRetainsCauseCode(t *testing.T) { + j := openTestJournal(t) + spec := testSpec("operation-completion-cause", "vm-completion-cause") + if _, _, err := j.Reserve(spec); err != nil { + t.Fatal(err) + } + if _, err := j.finishWithSemantics( + spec.OperationID, + "operation_failed", + "operation failed", + nil, + "backup_disk_transition_active", + false, + true, + ); err != nil { + t.Fatal(err) + } + claimed, err := j.ClaimPendingCompletions(1) + if err != nil || len(claimed) != 1 { + t.Fatalf("completion claim = %#v err=%v", claimed, err) + } + if claimed[0].ErrorCode != "operation_failed" || + claimed[0].CauseCode != "backup_disk_transition_active" { + t.Fatalf("failed completion receipt = %#v", claimed[0]) + } +} + func TestInspectionRecoversAckedCompletionReceiptAfterDetailPrune(t *testing.T) { j, spec, eventID, want := pruneAckedSettingsOperation(t) diff --git a/gateway/agent/completion.go b/gateway/agent/completion.go index 8aba5df0..46c67efa 100644 --- a/gateway/agent/completion.go +++ b/gateway/agent/completion.go @@ -79,6 +79,7 @@ func (s *HTTPCompletionSink) Deliver(ctx context.Context, agentID string, comple EffectMayHaveApplied bool `json:"effect_may_have_applied"` SafeToResubmit bool `json:"safe_to_resubmit"` ErrorCode string `json:"error_code"` + CauseCode string `json:"cause_code,omitempty"` Result json.RawMessage `json:"result"` }{ Type: "completion", ID: completion.ID, AgentID: agentID, OperationID: completion.OperationID, @@ -86,7 +87,8 @@ func (s *HTTPCompletionSink) Deliver(ctx context.Context, agentID string, comple RequestFingerprint: completion.RequestFingerprint, Status: completion.Status, EffectMayHaveApplied: completion.EffectMayHaveApplied, SafeToResubmit: completion.SafeToResubmit, ErrorCode: completion.ErrorCode, - Result: completion.Result, + CauseCode: completion.CauseCode, + Result: completion.Result, }) if err != nil || len(body) > maxControlFrame { return errors.New("encode completion delivery") diff --git a/gateway/agent/completion_test.go b/gateway/agent/completion_test.go index dbf46e90..71131e78 100644 --- a/gateway/agent/completion_test.go +++ b/gateway/agent/completion_test.go @@ -52,7 +52,8 @@ func TestHTTPCompletionSinkUsesExactPanelContractAndGatewayIdentity(t *testing.T Type: "completion", ID: "completion_event_123", OperationID: "operation_terminal_123", Action: "delete-vm", ResourceID: "vm-completion-123", RequestFingerprint: strings.Repeat("a", 64), - Status: "failed", ErrorCode: "result_unconfirmed", Result: json.RawMessage("null"), + Status: "failed", ErrorCode: "result_unconfirmed", + CauseCode: "backup_disk_transition_active", Result: json.RawMessage("null"), EffectMayHaveApplied: true, } if err := sink.Deliver(context.Background(), "42", completion); err != nil { @@ -67,7 +68,7 @@ func TestHTTPCompletionSinkUsesExactPanelContractAndGatewayIdentity(t *testing.T } wantKeys := []string{ "type", "id", "agent_id", "operation_id", "action", "resource_id", "request_fingerprint", - "status", "effect_may_have_applied", "safe_to_resubmit", "error_code", "result", + "status", "effect_may_have_applied", "safe_to_resubmit", "error_code", "cause_code", "result", } if len(payload) != len(wantKeys) { t.Fatalf("completion keys = %#v body=%s", payload, capturedBody) @@ -81,6 +82,7 @@ func TestHTTPCompletionSinkUsesExactPanelContractAndGatewayIdentity(t *testing.T string(payload["agent_id"]) != `"42"` || string(payload["action"]) != `"delete-vm"` || string(payload["resource_id"]) != `"vm-completion-123"` || string(payload["request_fingerprint"]) != `"`+strings.Repeat("a", 64)+`"` || + string(payload["cause_code"]) != `"backup_disk_transition_active"` || string(payload["result"]) != "null" { t.Fatalf("completion payload = %s", capturedBody) } @@ -289,6 +291,16 @@ func TestCompletionValidationRequiresExplicitNullResult(t *testing.T) { if err := validateCompletion(valid); err != nil { t.Fatalf("explicit null result rejected: %v", err) } + withCause := valid + withCause.CauseCode = "backup_disk_transition_active" + if err := validateCompletion(withCause); err != nil { + t.Fatalf("stable cause code rejected: %v", err) + } + invalidCause := valid + invalidCause.CauseCode = "Backup Disk Transition Active" + if err := validateCompletion(invalidCause); err == nil { + t.Fatal("invalid cause code was accepted") + } missing := valid missing.Result = nil if err := validateCompletion(missing); err == nil { diff --git a/gateway/agent/registry.go b/gateway/agent/registry.go index 1fd6d34e..d0b7667b 100644 --- a/gateway/agent/registry.go +++ b/gateway/agent/registry.go @@ -89,6 +89,7 @@ type Completion struct { EffectMayHaveApplied bool `json:"effect_may_have_applied"` SafeToResubmit bool `json:"safe_to_resubmit"` ErrorCode string `json:"error_code,omitempty"` + CauseCode string `json:"cause_code,omitempty"` Result json.RawMessage `json:"result"` } @@ -671,7 +672,8 @@ func validateCompletion(completion Completion) error { return errors.New("invalid completion event") } if completion.Status == "succeeded" { - if completion.ErrorCode != "" || completion.EffectMayHaveApplied || completion.SafeToResubmit { + if completion.ErrorCode != "" || completion.CauseCode != "" || + completion.EffectMayHaveApplied || completion.SafeToResubmit { return errors.New("invalid successful completion") } return nil @@ -679,6 +681,10 @@ func validateCompletion(completion Completion) error { if !stableCodePattern.MatchString(completion.ErrorCode) || len(completion.ErrorCode) > 64 { return errors.New("invalid failed completion") } + if completion.CauseCode != "" && + (!stableCodePattern.MatchString(completion.CauseCode) || len(completion.CauseCode) > 64) { + return errors.New("invalid failed completion cause") + } return nil } diff --git a/web/app/Services/Cloud/CloudOperationCompletionService.php b/web/app/Services/Cloud/CloudOperationCompletionService.php index 8aca8d9e..86f9d347 100644 --- a/web/app/Services/Cloud/CloudOperationCompletionService.php +++ b/web/app/Services/Cloud/CloudOperationCompletionService.php @@ -9,7 +9,8 @@ use App\Models\CloudJob; final class CloudOperationCompletionService { private const RESULT_ENVELOPE = 'virt_cloud_completion_event'; - private const RESULT_ENVELOPE_VERSION = 1; + private const RESULT_ENVELOPE_VERSION = 2; + private const LEGACY_RESULT_ENVELOPE_VERSION = 1; private const IDENTITY_RETRY_SECONDS = 300; public static function accept(array $input): array @@ -87,6 +88,7 @@ final class CloudOperationCompletionService 'request_fingerprint' => $receipt['request_fingerprint'], ], + 'cause_code' => $receipt['cause_code'] ?? null, 'result' => $receipt['result'], ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); @@ -275,7 +277,7 @@ final class CloudOperationCompletionService public static function operation(CloudAgentCompletionEvent $event): array { $envelope = self::resultEnvelope($event); - return [ + $operation = [ 'operation_id' => (string) $event->operation_id, 'status' => (string) $event->outcome, 'effect_may_have_applied' => (bool) $event->effect_may_have_applied, @@ -292,6 +294,10 @@ final class CloudOperationCompletionService $envelope['identity']['request_fingerprint'], 'result' => $envelope['result'], ]; + if ($envelope['cause_code'] !== null) { + $operation['cause_code'] = $envelope['cause_code']; + } + return $operation; } /** @@ -475,6 +481,9 @@ final class CloudOperationCompletionService 'error_code' => $errorCode, 'result' => $result, ]; + if ($envelope['cause_code'] !== null) { + $receipt['cause_code'] = $envelope['cause_code']; + } $calculated = hash('sha256', json_encode( $receipt, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES @@ -594,7 +603,7 @@ final class CloudOperationCompletionService } /** - * @return array{schema:string,version:int,identity:array{action:string,resource_id:string,request_fingerprint:string},result:?array} + * @return array{schema:string,version:int,identity:array{action:string,resource_id:string,request_fingerprint:string},cause_code:?string,result:?array} */ private static function resultEnvelope( CloudAgentCompletionEvent $event @@ -618,23 +627,37 @@ final class CloudOperationCompletionService ); } $identity = is_array($value) ? ($value['identity'] ?? null) : null; + $version = is_array($value) ? ($value['version'] ?? null) : null; + $allowedKeys = $version === self::LEGACY_RESULT_ENVELOPE_VERSION + ? ['schema', 'version', 'identity', 'result'] + : ['schema', 'version', 'identity', 'cause_code', 'result']; + $causeCode = $version === self::LEGACY_RESULT_ENVELOPE_VERSION + ? null : (is_array($value) ? ($value['cause_code'] ?? null) : null); if (!is_array($value) || array_is_list($value) - || array_diff(array_keys($value), [ - 'schema', 'version', 'identity', 'result', - ]) !== [] + || array_diff(array_keys($value), $allowedKeys) !== [] || ($value['schema'] ?? null) !== self::RESULT_ENVELOPE - || ($value['version'] ?? null) !== self::RESULT_ENVELOPE_VERSION + || !in_array($version, [ + self::LEGACY_RESULT_ENVELOPE_VERSION, + self::RESULT_ENVELOPE_VERSION, + ], true) || !is_array($identity) || array_is_list($identity) || array_diff(array_keys($identity), [ 'action', 'resource_id', 'request_fingerprint', ]) !== [] || !array_key_exists('result', $value) + || ($version === self::RESULT_ENVELOPE_VERSION + && !array_key_exists('cause_code', $value)) + || ($causeCode !== null + && (!is_string($causeCode) + || preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $causeCode) + !== 1)) || ($value['result'] !== null && !is_array($value['result']))) { throw new CloudRecoveryOperationIdentityConflict( 'Agent 完成回执身份封装无效', 409 ); } self::assertIdentityShape($identity); + $value['cause_code'] = $causeCode; return $value; } @@ -1179,7 +1202,8 @@ final class CloudOperationCompletionService $allowed = ['type', 'id', 'agent_id', 'operation_id', 'action', 'resource_id', 'request_fingerprint', 'status', 'effect_may_have_applied', 'safe_to_resubmit', 'error_code', - 'result']; + 'cause_code', 'result']; + $causeCode = $input['cause_code'] ?? ''; if (array_is_list($input) || array_diff(array_keys($input), $allowed) !== [] || ($input['type'] ?? null) !== 'completion' @@ -1211,12 +1235,16 @@ final class CloudOperationCompletionService || (($input['error_code'] ?? '') !== '' && preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $input['error_code']) !== 1) + || !is_string($causeCode) + || ($causeCode !== '' + && preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $causeCode) !== 1) || !array_key_exists('result', $input) || ($input['result'] !== null && !is_array($input['result'])) || (($input['status'] ?? null) === 'succeeded' && (($input['effect_may_have_applied'] ?? null) !== false || ($input['safe_to_resubmit'] ?? null) !== false - || ($input['error_code'] ?? null) !== '')) + || ($input['error_code'] ?? null) !== '' + || $causeCode !== '')) || (($input['status'] ?? null) === 'failed' && (($input['error_code'] ?? '') === '' || (($input['effect_may_have_applied'] ?? false) === true @@ -1224,7 +1252,7 @@ final class CloudOperationCompletionService throw new CloudError('Agent 完成回执无效', 400); } self::assertCompletionResultSafe($input['result']); - return [ + $receipt = [ 'event_id' => $input['id'], 'host_id' => (int) $input['agent_id'], 'operation_id' => $input['operation_id'], @@ -1237,5 +1265,9 @@ final class CloudOperationCompletionService 'error_code' => $input['error_code'], 'result' => $input['result'], ]; + if ($causeCode !== '') { + $receipt['cause_code'] = $causeCode; + } + return $receipt; } } diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php index 08e3b238..e41ab479 100644 --- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php +++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php @@ -136,6 +136,7 @@ final class CloudOperationErrorCatalog 'storage_configuration_busy' => '另一存储配置 helper 尚未退出;本次任务已立即失败且不会重试,请在 helper 退出后创建新任务', 'storage_configuration_lock_unavailable' => 'Agent 无法使用存储配置进程锁;请检查 journal_path 所在目录的权限和文件系统锁支持', 'libvirt_unavailable' => 'Agent 无法连接 libvirt;请检查 libvirt 服务和 Agent 权限', + 'backup_disk_transition_active' => '实例仍有旧系统盘等待清理,当前不能创建备份;若重装或快照恢复已完成,请核对宿主机持久配置', ]; /** diff --git a/web/tests/CloudOperationCompletionContractTest.php b/web/tests/CloudOperationCompletionContractTest.php index dafe305b..7491fdb1 100644 --- a/web/tests/CloudOperationCompletionContractTest.php +++ b/web/tests/CloudOperationCompletionContractTest.php @@ -70,11 +70,36 @@ $assert($reject($invalid), $validFailure = $invalid; $validFailure['safe_to_resubmit'] = false; +$validFailure['cause_code'] = 'backup_disk_transition_active'; $validFailure = $validate->invoke(null, $validFailure); $assert(($validFailure['status'] ?? null) === 'failed' && ($validFailure['effect_may_have_applied'] ?? null) === true - && ($validFailure['safe_to_resubmit'] ?? null) === false, - 'A conservative terminal failure receipt was rejected.'); + && ($validFailure['safe_to_resubmit'] ?? null) === false + && ($validFailure['cause_code'] ?? null) + === 'backup_disk_transition_active', + 'A conservative terminal failure receipt lost its Agent cause.'); + +$causeEvent = new CloudAgentCompletionEvent(); +$causeEvent->operation_id = str_repeat('7', 32); +$causeEvent->outcome = 'failed'; +$causeEvent->effect_may_have_applied = 0; +$causeEvent->safe_to_resubmit = 1; +$causeEvent->error_code = 'operation_failed'; +$causeEvent->result_json = json_encode([ + 'schema' => 'virt_cloud_completion_event', + 'version' => 2, + 'identity' => [ + 'action' => 'create-backup:abc:crash', + 'resource_id' => '11111111-1111-4111-8111-111111111111', + 'request_fingerprint' => str_repeat('6', 64), + ], + 'cause_code' => 'backup_disk_transition_active', + 'result' => null, +], JSON_THROW_ON_ERROR); +$causeOperation = CloudOperationCompletionService::operation($causeEvent); +$assert(($causeOperation['cause_code'] ?? null) + === 'backup_disk_transition_active', + 'A durable completion event did not reconstruct its Agent cause.'); // Completion delivery intentionally omits the display-only retryable field. // Reconstructing its terminal Agent operation must restore the canonical diff --git a/web/tests/CloudOperationErrorCatalogTest.php b/web/tests/CloudOperationErrorCatalogTest.php index 31168a9a..6ad77d4d 100644 --- a/web/tests/CloudOperationErrorCatalogTest.php +++ b/web/tests/CloudOperationErrorCatalogTest.php @@ -114,6 +114,8 @@ $assert(str_contains((string) CloudOperationErrorCatalog::agentCauseLabel( 'managed_disk_shared'), '另一台虚拟机') && str_contains((string) CloudOperationErrorCatalog::agentCauseLabel( 'managed_filter_shared'), '过滤器引用') + && str_contains((string) CloudOperationErrorCatalog::agentCauseLabel( + 'backup_disk_transition_active'), '旧系统盘') && CloudOperationErrorCatalog::agentCauseLabel('storage_clone_failed') === null && CloudOperationErrorCatalog::agentCauseLabel('bad/path') === null, 'Reviewed Agent cause labels exposed raw or unknown diagnostic codes.');