fix(cloud): repair portable backup preflight

go
codex 3 weeks ago
parent 4c898a2310
commit 63f365d847

@ -91,6 +91,7 @@ type completionMessage struct {
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
SafeToResubmit bool `json:"safe_to_resubmit"`
ErrorCode string `json:"error_code,omitempty"`
CauseCode string `json:"cause_code,omitempty"`
Result json.RawMessage `json:"result"`
}
@ -105,8 +106,9 @@ func completionMessageFromReceipt(completion operation.CompletionReceipt) (compl
Action: completion.Action, ResourceID: completion.ResourceID,
RequestFingerprint: completion.RequestFingerprint,
Status: completion.Status, Result: completion.Result,
ErrorCode: completion.ErrorCode, EffectMayHaveApplied: completion.EffectMayHaveApplied,
SafeToResubmit: completion.SafeToResubmit,
ErrorCode: completion.ErrorCode, CauseCode: completion.CauseCode,
EffectMayHaveApplied: completion.EffectMayHaveApplied,
SafeToResubmit: completion.SafeToResubmit,
}, nil
}

@ -55,14 +55,17 @@ func TestCompletionMessageCarriesStrictImmutableOperationIdentity(t *testing.T)
EventID: "completion_identity_123", OperationID: "operation_identity_123",
Action: "delete-vm", ResourceID: "vm-completion-identity",
RequestFingerprint: strings.Repeat("a", 64),
Status: "succeeded", Result: json.RawMessage(`{"done":true}`),
Status: "failed", Result: json.RawMessage("null"),
ErrorCode: "operation_failed",
CauseCode: "backup_disk_transition_active",
}
message, err := completionMessageFromReceipt(receipt)
if err != nil {
t.Fatal(err)
}
if message.Action != receipt.Action || message.ResourceID != receipt.ResourceID ||
message.RequestFingerprint != receipt.RequestFingerprint {
message.RequestFingerprint != receipt.RequestFingerprint ||
message.CauseCode != receipt.CauseCode {
t.Fatalf("completion message identity = %#v", message)
}

@ -21,9 +21,13 @@ func (l *Libvirt) CreateBackup(ctx context.Context, uuid string, request BackupR
if err := ValidateBackupRequest(request); err != nil {
return BackupResult{}, operation.MarkNotApplied(err)
}
client := objecttransfer.New()
defer client.CloseIdleConnections()
active, err := l.validateBackupSource(ctx, uuid, request.BackupKey)
if err != nil {
return BackupResult{}, operation.MarkNotApplied(err)
return BackupResult{}, abortRejectedBackupPlan(
ctx, client.AbortMultipart, request.ObjectUpload, err,
)
}
snapshotRequest := SnapshotRequest{
@ -61,8 +65,6 @@ func (l *Libvirt) CreateBackup(ctx context.Context, uuid string, request BackupR
))
}
client := objecttransfer.New()
defer client.CloseIdleConnections()
upload, err := client.UploadFile(ctx, path, request.ObjectUpload)
if err != nil {
translated := translateObjectTransferError(err)
@ -103,6 +105,18 @@ func (l *Libvirt) CreateBackup(ctx context.Context, uuid string, request BackupR
return result, nil
}
func abortRejectedBackupPlan(ctx context.Context,
abort func(context.Context, objecttransfer.MultipartPlan) error,
plan objecttransfer.MultipartPlan, rejection error) error {
if cleanupErr := abort(ctx, plan); cleanupErr != nil {
// The VM and backup object were never mutated. Retain retry-safe
// semantics and the original preflight cause while keeping the bounded
// multipart cleanup failure in the private Agent error chain.
return operation.MarkNotApplied(errors.Join(rejection, cleanupErr))
}
return operation.MarkNotApplied(rejection)
}
func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey string) (bool, error) {
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
@ -118,8 +132,8 @@ func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey stri
if metadata.RescueTemplateID != "" {
return false, NewError(ErrConflict, "backup_rescue_active", "VM backup is unavailable while rescue media is attached", nil)
}
if metadata.PendingCleanupVolume != "" || metadata.DiskTransitionKind != "" {
return false, NewError(ErrConflict, "backup_disk_transition_active", "VM backup is unavailable during a disk transition", nil)
if err := validateBackupTransitionState(metadata); err != nil {
return false, err
}
for _, stored := range metadata.Snapshots {
if stored.Key == backupKey {
@ -166,6 +180,17 @@ func (l *Libvirt) validateBackupSource(ctx context.Context, uuid, backupKey stri
return active, nil
}
func validateBackupTransitionState(metadata *domainMetadataXML) error {
// Reinstall and snapshot restore deliberately retain their transition kind
// and operation ID after the old root has been deleted. Those markers prove
// a completed root switch during reconciliation; only a pending old root
// means that the disk transition is still active.
if metadata != nil && metadata.PendingCleanupVolume != "" {
return NewError(ErrConflict, "backup_disk_transition_active", "VM backup is unavailable during a disk transition", nil)
}
return nil
}
func validateSingleWritableRoot(definition *libvirtxml.Domain) error {
if definition == nil || definition.Devices == nil {
return NewError(ErrConflict, "backup_disk_layout_unsupported", "VM disk layout is unavailable for portable backup", nil)

@ -3,9 +3,11 @@
package hypervisor
import (
"context"
"errors"
"testing"
"git.luoli.live/root/virt/agent/objecttransfer"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
@ -99,3 +101,66 @@ func TestValidateStandaloneQCOW2RejectsBackingChain(t *testing.T) {
t.Fatalf("backing-chain error = %v", err)
}
}
func TestValidateBackupTransitionStateAllowsCompletedDiskTransitions(t *testing.T) {
tests := []struct {
name string
metadata *domainMetadataXML
wantError bool
}{
{
name: "reinstall cleanup pending",
metadata: &domainMetadataXML{
PendingCleanupVolume: "123e4567-e89b-42d3-a456-426614174000-root.qcow2",
DiskTransitionKind: "reinstall",
DiskTransitionID: "operation-reinstall-one",
},
wantError: true,
},
{
name: "completed reinstall marker",
metadata: &domainMetadataXML{
DiskTransitionKind: "reinstall",
DiskTransitionID: "operation-reinstall-one",
},
},
{
name: "completed snapshot restore marker",
metadata: &domainMetadataXML{
DiskTransitionKind: "snapshot_restore",
DiskTransitionID: "operation-snapshot-restore-one",
},
},
{name: "no transition", metadata: &domainMetadataXML{}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
err := validateBackupTransitionState(test.metadata)
if test.wantError && !errors.Is(err, ErrConflict) {
t.Fatalf("pending transition error = %v", err)
}
if !test.wantError && err != nil {
t.Fatalf("completed transition marker rejected: %v", err)
}
})
}
}
func TestAbortRejectedBackupPlanPreservesPreflightCause(t *testing.T) {
rejected := NewError(ErrConflict, "backup_disk_transition_active",
"VM backup is unavailable during a disk transition", nil)
called := false
err := abortRejectedBackupPlan(
context.Background(),
func(context.Context, objecttransfer.MultipartPlan) error {
called = true
return nil
},
objecttransfer.MultipartPlan{},
rejected,
)
if !called || !errors.Is(err, rejected) {
t.Fatalf("called=%v err=%v", called, err)
}
}

@ -199,16 +199,18 @@ func (c *Client) UploadFile(ctx context.Context, path string, plan MultipartPlan
}
size, digest, before, err := hashFile(path)
if err != nil {
return UploadResult{}, err
return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL, err)
}
partsNeeded := int((size + plan.PartSizeBytes - 1) / plan.PartSizeBytes)
if size > plan.MaximumSizeBytes || partsNeeded > len(plan.PartURLs) || partsNeeded > MaxMultipartParts {
return UploadResult{}, transferError("backup_source_too_large", "backup source exceeds the multipart upload plan", false)
return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL,
transferError("backup_source_too_large", "backup source exceeds the multipart upload plan", false))
}
file, err := os.Open(path)
if err != nil {
return UploadResult{}, transferError("backup_source_unavailable", "backup source could not be opened", false)
return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL,
transferError("backup_source_unavailable", "backup source could not be opened", false))
}
defer file.Close()
@ -357,6 +359,10 @@ func (c *Client) complete(ctx context.Context, capability string, parts []Comple
}
func (c *Client) failAndAbort(ctx context.Context, capability, code, message string) error {
return c.failAndAbortError(ctx, capability, transferError(code, message, false))
}
func (c *Client) failAndAbortError(ctx context.Context, capability string, failure error) error {
// The caller's cancellation often caused the upload failure itself. Give
// cleanup one independent, tightly bounded chance; never inherit a long or
// already-expired operation deadline.
@ -365,7 +371,22 @@ func (c *Client) failAndAbort(ctx context.Context, capability, code, message str
if err := c.abort(abortCtx, capability); err != nil {
return transferError("upload_abort_uncertain", "backup upload failed and multipart cleanup is unconfirmed", true)
}
return transferError(code, message, false)
return failure
}
// AbortMultipart consumes only the bounded DELETE capability from a validated
// plan. It is used when higher-level VM preflight rejects the backup before a
// local snapshot or object upload can start.
func (c *Client) AbortMultipart(ctx context.Context, plan MultipartPlan) error {
if c == nil || c.http == nil {
return transferError("transfer_client_unavailable", "object transfer client is unavailable", false)
}
if err := ValidateMultipartPlan(plan); err != nil {
return err
}
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
defer cancel()
return c.abort(abortCtx, plan.AbortURL)
}
func (c *Client) abort(ctx context.Context, capability string) error {

@ -138,6 +138,52 @@ func TestUploadFileAbortsAfterKnownPartFailure(t *testing.T) {
}
}
func TestUploadFileAbortsBeforeFirstPartWhenSourceIsUnavailable(t *testing.T) {
aborted := false
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
t.Errorf("unexpected method before source validation: %s", r.Method)
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
aborted = true
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
missing := filepath.Join(t.TempDir(), "missing.qcow2")
_, err := testClient(server).UploadFile(
context.Background(), missing, multipartPlan(server.URL, 1, MinPartSizeBytes),
)
if err == nil || IsAmbiguous(err) || !aborted {
t.Fatalf("err=%v ambiguous=%v aborted=%v", err, IsAmbiguous(err), aborted)
}
}
func TestAbortMultipartUsesDeleteCapability(t *testing.T) {
aborted := false
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete || r.URL.Query().Get("abort") != "1" {
t.Errorf("unexpected abort request: %s %s", r.Method, r.URL.String())
w.WriteHeader(http.StatusBadRequest)
return
}
aborted = true
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
client := testClient(server)
if err := client.AbortMultipart(
context.Background(), multipartPlan(server.URL, 1, MinPartSizeBytes),
); err != nil {
t.Fatalf("AbortMultipart: %v", err)
}
if !aborted {
t.Fatal("multipart abort capability was not used")
}
}
func TestUploadFileDoesNotAbortAmbiguousCompletion(t *testing.T) {
path, contents := writeSource(t, int(MinPartSizeBytes))
aborted := false

@ -215,6 +215,7 @@ var terminalEvidenceOpenTimeout = 2 * time.Second
// state. Delivery timing is kept in a separate outbox record so an ACK can
// never rewrite the operation or its receipt.
type CompletionReceipt struct {
Version int `json:"version,omitempty"`
EventID string `json:"id"`
OperationID string `json:"operation_id"`
Action string `json:"action"`
@ -223,6 +224,7 @@ type CompletionReceipt struct {
Status string `json:"status"`
Result json.RawMessage `json:"result"`
ErrorCode string `json:"error_code,omitempty"`
CauseCode string `json:"cause_code,omitempty"`
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
SafeToResubmit bool `json:"safe_to_resubmit"`
CreatedAt time.Time `json:"-"`
@ -687,6 +689,7 @@ func completionReceiptMatchesRecordForInspection(eventID, requestedOperationID s
receipt.RequestFingerprint != record.Fingerprint ||
receipt.EffectMayHaveApplied != record.EffectMayHaveApplied ||
receipt.SafeToResubmit != record.SafeToResubmit ||
!completionCauseMatchesRecord(receipt, record) ||
len(receipt.Result) == 0 || !json.Valid(receipt.Result) {
return false
}
@ -1364,6 +1367,7 @@ func completionReceiptMatchesRecord(receipt CompletionReceipt, record Record) bo
receipt.RequestFingerprint != record.Fingerprint ||
receipt.EffectMayHaveApplied != record.EffectMayHaveApplied ||
receipt.SafeToResubmit != record.SafeToResubmit ||
!completionCauseMatchesRecord(receipt, record) ||
!completionResultSafe(receipt.Result) || !completionDeliveryFitsPanelLimit(receipt) {
return false
}
@ -1397,6 +1401,20 @@ func completionReceiptMatchesRecord(receipt CompletionReceipt, record Record) bo
return completionResultsEqual(receipt.Result, record.Result)
}
func completionCauseMatchesRecord(receipt CompletionReceipt, record Record) bool {
switch receipt.Version {
case 0:
// Legacy v1 receipts predate cause_code. They remain deliverable after
// an Agent upgrade, but cannot recover the diagnostic that was omitted.
return receipt.CauseCode == ""
case 2:
return (receipt.CauseCode == "" || validCauseCode(receipt.CauseCode)) &&
receipt.CauseCode == record.CauseCode
default:
return false
}
}
func completionDeliveryFitsPanelLimit(receipt CompletionReceipt) bool {
// The Gateway adds agent_id before POSTing to PHP. Marshal the exact public
// envelope with the longest valid agent ID so every claimed receipt is
@ -1413,13 +1431,14 @@ func completionDeliveryFitsPanelLimit(receipt CompletionReceipt) bool {
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
SafeToResubmit bool `json:"safe_to_resubmit"`
ErrorCode string `json:"error_code"`
CauseCode string `json:"cause_code,omitempty"`
Result json.RawMessage `json:"result"`
}{
Type: "completion", ID: receipt.EventID, AgentID: "9999999999999999999",
OperationID: receipt.OperationID, Action: receipt.Action, ResourceID: receipt.ResourceID,
RequestFingerprint: receipt.RequestFingerprint, Status: receipt.Status,
EffectMayHaveApplied: receipt.EffectMayHaveApplied, SafeToResubmit: receipt.SafeToResubmit,
ErrorCode: receipt.ErrorCode, Result: receipt.Result,
ErrorCode: receipt.ErrorCode, CauseCode: receipt.CauseCode, Result: receipt.Result,
})
return err == nil && len(encoded) <= completionPanelMaximumBody
}
@ -1689,10 +1708,12 @@ func syncCompletionReceipt(receipts, outbox *bolt.Bucket, previous, current Reco
errorCode = current.Error.Code
}
receipt := CompletionReceipt{
Version: 2,
EventID: eventID, OperationID: current.OperationID,
Action: current.Action, ResourceID: current.ResourceID, RequestFingerprint: current.Fingerprint,
Status: status,
Result: append(json.RawMessage(nil), current.Result...), ErrorCode: errorCode,
CauseCode: current.CauseCode,
EffectMayHaveApplied: current.EffectMayHaveApplied,
SafeToResubmit: current.SafeToResubmit, CreatedAt: current.UpdatedAt.UTC(),
}

@ -669,6 +669,33 @@ func TestCompletionReceiptIsImmutableAndAckOnlyClearsOutbox(t *testing.T) {
}
}
func TestFailedCompletionReceiptRetainsCauseCode(t *testing.T) {
j := openTestJournal(t)
spec := testSpec("operation-completion-cause", "vm-completion-cause")
if _, _, err := j.Reserve(spec); err != nil {
t.Fatal(err)
}
if _, err := j.finishWithSemantics(
spec.OperationID,
"operation_failed",
"operation failed",
nil,
"backup_disk_transition_active",
false,
true,
); err != nil {
t.Fatal(err)
}
claimed, err := j.ClaimPendingCompletions(1)
if err != nil || len(claimed) != 1 {
t.Fatalf("completion claim = %#v err=%v", claimed, err)
}
if claimed[0].ErrorCode != "operation_failed" ||
claimed[0].CauseCode != "backup_disk_transition_active" {
t.Fatalf("failed completion receipt = %#v", claimed[0])
}
}
func TestInspectionRecoversAckedCompletionReceiptAfterDetailPrune(t *testing.T) {
j, spec, eventID, want := pruneAckedSettingsOperation(t)

@ -79,6 +79,7 @@ func (s *HTTPCompletionSink) Deliver(ctx context.Context, agentID string, comple
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
SafeToResubmit bool `json:"safe_to_resubmit"`
ErrorCode string `json:"error_code"`
CauseCode string `json:"cause_code,omitempty"`
Result json.RawMessage `json:"result"`
}{
Type: "completion", ID: completion.ID, AgentID: agentID, OperationID: completion.OperationID,
@ -86,7 +87,8 @@ func (s *HTTPCompletionSink) Deliver(ctx context.Context, agentID string, comple
RequestFingerprint: completion.RequestFingerprint,
Status: completion.Status, EffectMayHaveApplied: completion.EffectMayHaveApplied,
SafeToResubmit: completion.SafeToResubmit, ErrorCode: completion.ErrorCode,
Result: completion.Result,
CauseCode: completion.CauseCode,
Result: completion.Result,
})
if err != nil || len(body) > maxControlFrame {
return errors.New("encode completion delivery")

@ -52,7 +52,8 @@ func TestHTTPCompletionSinkUsesExactPanelContractAndGatewayIdentity(t *testing.T
Type: "completion", ID: "completion_event_123", OperationID: "operation_terminal_123",
Action: "delete-vm", ResourceID: "vm-completion-123",
RequestFingerprint: strings.Repeat("a", 64),
Status: "failed", ErrorCode: "result_unconfirmed", Result: json.RawMessage("null"),
Status: "failed", ErrorCode: "result_unconfirmed",
CauseCode: "backup_disk_transition_active", Result: json.RawMessage("null"),
EffectMayHaveApplied: true,
}
if err := sink.Deliver(context.Background(), "42", completion); err != nil {
@ -67,7 +68,7 @@ func TestHTTPCompletionSinkUsesExactPanelContractAndGatewayIdentity(t *testing.T
}
wantKeys := []string{
"type", "id", "agent_id", "operation_id", "action", "resource_id", "request_fingerprint",
"status", "effect_may_have_applied", "safe_to_resubmit", "error_code", "result",
"status", "effect_may_have_applied", "safe_to_resubmit", "error_code", "cause_code", "result",
}
if len(payload) != len(wantKeys) {
t.Fatalf("completion keys = %#v body=%s", payload, capturedBody)
@ -81,6 +82,7 @@ func TestHTTPCompletionSinkUsesExactPanelContractAndGatewayIdentity(t *testing.T
string(payload["agent_id"]) != `"42"` || string(payload["action"]) != `"delete-vm"` ||
string(payload["resource_id"]) != `"vm-completion-123"` ||
string(payload["request_fingerprint"]) != `"`+strings.Repeat("a", 64)+`"` ||
string(payload["cause_code"]) != `"backup_disk_transition_active"` ||
string(payload["result"]) != "null" {
t.Fatalf("completion payload = %s", capturedBody)
}
@ -289,6 +291,16 @@ func TestCompletionValidationRequiresExplicitNullResult(t *testing.T) {
if err := validateCompletion(valid); err != nil {
t.Fatalf("explicit null result rejected: %v", err)
}
withCause := valid
withCause.CauseCode = "backup_disk_transition_active"
if err := validateCompletion(withCause); err != nil {
t.Fatalf("stable cause code rejected: %v", err)
}
invalidCause := valid
invalidCause.CauseCode = "Backup Disk Transition Active"
if err := validateCompletion(invalidCause); err == nil {
t.Fatal("invalid cause code was accepted")
}
missing := valid
missing.Result = nil
if err := validateCompletion(missing); err == nil {

@ -89,6 +89,7 @@ type Completion struct {
EffectMayHaveApplied bool `json:"effect_may_have_applied"`
SafeToResubmit bool `json:"safe_to_resubmit"`
ErrorCode string `json:"error_code,omitempty"`
CauseCode string `json:"cause_code,omitempty"`
Result json.RawMessage `json:"result"`
}
@ -671,7 +672,8 @@ func validateCompletion(completion Completion) error {
return errors.New("invalid completion event")
}
if completion.Status == "succeeded" {
if completion.ErrorCode != "" || completion.EffectMayHaveApplied || completion.SafeToResubmit {
if completion.ErrorCode != "" || completion.CauseCode != "" ||
completion.EffectMayHaveApplied || completion.SafeToResubmit {
return errors.New("invalid successful completion")
}
return nil
@ -679,6 +681,10 @@ func validateCompletion(completion Completion) error {
if !stableCodePattern.MatchString(completion.ErrorCode) || len(completion.ErrorCode) > 64 {
return errors.New("invalid failed completion")
}
if completion.CauseCode != "" &&
(!stableCodePattern.MatchString(completion.CauseCode) || len(completion.CauseCode) > 64) {
return errors.New("invalid failed completion cause")
}
return nil
}

@ -9,7 +9,8 @@ use App\Models\CloudJob;
final class CloudOperationCompletionService
{
private const RESULT_ENVELOPE = 'virt_cloud_completion_event';
private const RESULT_ENVELOPE_VERSION = 1;
private const RESULT_ENVELOPE_VERSION = 2;
private const LEGACY_RESULT_ENVELOPE_VERSION = 1;
private const IDENTITY_RETRY_SECONDS = 300;
public static function accept(array $input): array
@ -87,6 +88,7 @@ final class CloudOperationCompletionService
'request_fingerprint' =>
$receipt['request_fingerprint'],
],
'cause_code' => $receipt['cause_code'] ?? null,
'result' => $receipt['result'],
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE);
@ -275,7 +277,7 @@ final class CloudOperationCompletionService
public static function operation(CloudAgentCompletionEvent $event): array
{
$envelope = self::resultEnvelope($event);
return [
$operation = [
'operation_id' => (string) $event->operation_id,
'status' => (string) $event->outcome,
'effect_may_have_applied' => (bool) $event->effect_may_have_applied,
@ -292,6 +294,10 @@ final class CloudOperationCompletionService
$envelope['identity']['request_fingerprint'],
'result' => $envelope['result'],
];
if ($envelope['cause_code'] !== null) {
$operation['cause_code'] = $envelope['cause_code'];
}
return $operation;
}
/**
@ -475,6 +481,9 @@ final class CloudOperationCompletionService
'error_code' => $errorCode,
'result' => $result,
];
if ($envelope['cause_code'] !== null) {
$receipt['cause_code'] = $envelope['cause_code'];
}
$calculated = hash('sha256', json_encode(
$receipt,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
@ -594,7 +603,7 @@ final class CloudOperationCompletionService
}
/**
* @return array{schema:string,version:int,identity:array{action:string,resource_id:string,request_fingerprint:string},result:?array}
* @return array{schema:string,version:int,identity:array{action:string,resource_id:string,request_fingerprint:string},cause_code:?string,result:?array}
*/
private static function resultEnvelope(
CloudAgentCompletionEvent $event
@ -618,23 +627,37 @@ final class CloudOperationCompletionService
);
}
$identity = is_array($value) ? ($value['identity'] ?? null) : null;
$version = is_array($value) ? ($value['version'] ?? null) : null;
$allowedKeys = $version === self::LEGACY_RESULT_ENVELOPE_VERSION
? ['schema', 'version', 'identity', 'result']
: ['schema', 'version', 'identity', 'cause_code', 'result'];
$causeCode = $version === self::LEGACY_RESULT_ENVELOPE_VERSION
? null : (is_array($value) ? ($value['cause_code'] ?? null) : null);
if (!is_array($value) || array_is_list($value)
|| array_diff(array_keys($value), [
'schema', 'version', 'identity', 'result',
]) !== []
|| array_diff(array_keys($value), $allowedKeys) !== []
|| ($value['schema'] ?? null) !== self::RESULT_ENVELOPE
|| ($value['version'] ?? null) !== self::RESULT_ENVELOPE_VERSION
|| !in_array($version, [
self::LEGACY_RESULT_ENVELOPE_VERSION,
self::RESULT_ENVELOPE_VERSION,
], true)
|| !is_array($identity) || array_is_list($identity)
|| array_diff(array_keys($identity), [
'action', 'resource_id', 'request_fingerprint',
]) !== []
|| !array_key_exists('result', $value)
|| ($version === self::RESULT_ENVELOPE_VERSION
&& !array_key_exists('cause_code', $value))
|| ($causeCode !== null
&& (!is_string($causeCode)
|| preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $causeCode)
!== 1))
|| ($value['result'] !== null && !is_array($value['result']))) {
throw new CloudRecoveryOperationIdentityConflict(
'Agent 完成回执身份封装无效', 409
);
}
self::assertIdentityShape($identity);
$value['cause_code'] = $causeCode;
return $value;
}
@ -1179,7 +1202,8 @@ final class CloudOperationCompletionService
$allowed = ['type', 'id', 'agent_id', 'operation_id', 'action',
'resource_id', 'request_fingerprint', 'status',
'effect_may_have_applied', 'safe_to_resubmit', 'error_code',
'result'];
'cause_code', 'result'];
$causeCode = $input['cause_code'] ?? '';
if (array_is_list($input)
|| array_diff(array_keys($input), $allowed) !== []
|| ($input['type'] ?? null) !== 'completion'
@ -1211,12 +1235,16 @@ final class CloudOperationCompletionService
|| (($input['error_code'] ?? '') !== ''
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D',
$input['error_code']) !== 1)
|| !is_string($causeCode)
|| ($causeCode !== ''
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $causeCode) !== 1)
|| !array_key_exists('result', $input)
|| ($input['result'] !== null && !is_array($input['result']))
|| (($input['status'] ?? null) === 'succeeded'
&& (($input['effect_may_have_applied'] ?? null) !== false
|| ($input['safe_to_resubmit'] ?? null) !== false
|| ($input['error_code'] ?? null) !== ''))
|| ($input['error_code'] ?? null) !== ''
|| $causeCode !== ''))
|| (($input['status'] ?? null) === 'failed'
&& (($input['error_code'] ?? '') === ''
|| (($input['effect_may_have_applied'] ?? false) === true
@ -1224,7 +1252,7 @@ final class CloudOperationCompletionService
throw new CloudError('Agent 完成回执无效', 400);
}
self::assertCompletionResultSafe($input['result']);
return [
$receipt = [
'event_id' => $input['id'],
'host_id' => (int) $input['agent_id'],
'operation_id' => $input['operation_id'],
@ -1237,5 +1265,9 @@ final class CloudOperationCompletionService
'error_code' => $input['error_code'],
'result' => $input['result'],
];
if ($causeCode !== '') {
$receipt['cause_code'] = $causeCode;
}
return $receipt;
}
}

@ -136,6 +136,7 @@ final class CloudOperationErrorCatalog
'storage_configuration_busy' => '另一存储配置 helper 尚未退出;本次任务已立即失败且不会重试,请在 helper 退出后创建新任务',
'storage_configuration_lock_unavailable' => 'Agent 无法使用存储配置进程锁;请检查 journal_path 所在目录的权限和文件系统锁支持',
'libvirt_unavailable' => 'Agent 无法连接 libvirt;请检查 libvirt 服务和 Agent 权限',
'backup_disk_transition_active' => '实例仍有旧系统盘等待清理,当前不能创建备份;若重装或快照恢复已完成,请核对宿主机持久配置',
];
/**

@ -70,11 +70,36 @@ $assert($reject($invalid),
$validFailure = $invalid;
$validFailure['safe_to_resubmit'] = false;
$validFailure['cause_code'] = 'backup_disk_transition_active';
$validFailure = $validate->invoke(null, $validFailure);
$assert(($validFailure['status'] ?? null) === 'failed'
&& ($validFailure['effect_may_have_applied'] ?? null) === true
&& ($validFailure['safe_to_resubmit'] ?? null) === false,
'A conservative terminal failure receipt was rejected.');
&& ($validFailure['safe_to_resubmit'] ?? null) === false
&& ($validFailure['cause_code'] ?? null)
=== 'backup_disk_transition_active',
'A conservative terminal failure receipt lost its Agent cause.');
$causeEvent = new CloudAgentCompletionEvent();
$causeEvent->operation_id = str_repeat('7', 32);
$causeEvent->outcome = 'failed';
$causeEvent->effect_may_have_applied = 0;
$causeEvent->safe_to_resubmit = 1;
$causeEvent->error_code = 'operation_failed';
$causeEvent->result_json = json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 2,
'identity' => [
'action' => 'create-backup:abc:crash',
'resource_id' => '11111111-1111-4111-8111-111111111111',
'request_fingerprint' => str_repeat('6', 64),
],
'cause_code' => 'backup_disk_transition_active',
'result' => null,
], JSON_THROW_ON_ERROR);
$causeOperation = CloudOperationCompletionService::operation($causeEvent);
$assert(($causeOperation['cause_code'] ?? null)
=== 'backup_disk_transition_active',
'A durable completion event did not reconstruct its Agent cause.');
// Completion delivery intentionally omits the display-only retryable field.
// Reconstructing its terminal Agent operation must restore the canonical

@ -114,6 +114,8 @@ $assert(str_contains((string) CloudOperationErrorCatalog::agentCauseLabel(
'managed_disk_shared'), '另一台虚拟机')
&& str_contains((string) CloudOperationErrorCatalog::agentCauseLabel(
'managed_filter_shared'), '过滤器引用')
&& str_contains((string) CloudOperationErrorCatalog::agentCauseLabel(
'backup_disk_transition_active'), '旧系统盘')
&& CloudOperationErrorCatalog::agentCauseLabel('storage_clone_failed') === null
&& CloudOperationErrorCatalog::agentCauseLabel('bad/path') === null,
'Reviewed Agent cause labels exposed raw or unknown diagnostic codes.');

Loading…
Cancel
Save