//go:build libvirt package hypervisor import ( "context" "encoding/json" "errors" "path/filepath" "reflect" "strings" "testing" "time" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/peermigration" "libvirt.org/go/libvirt" ) type cleanupJobDomainFake struct { active bool activeErr error job *libvirt.DomainJobInfo jobErr error document string inactiveXML string xmlErr error blockInfo map[string]*libvirt.DomainBlockJobInfo blockErr map[string]error blockQueries []string } func (f *cleanupJobDomainFake) IsActive() (bool, error) { return f.active, f.activeErr } func (f *cleanupJobDomainFake) GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) { return f.job, f.jobErr } func (f *cleanupJobDomainFake) GetXMLDesc(flags libvirt.DomainXMLFlags) (string, error) { if flags == libvirt.DOMAIN_XML_INACTIVE && f.inactiveXML != "" { return f.inactiveXML, f.xmlErr } return f.document, f.xmlErr } func (f *cleanupJobDomainFake) GetBlockJobInfo(target string, _ libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error) { f.blockQueries = append(f.blockQueries, target) if err := f.blockErr[target]; err != nil { return nil, err } if info, ok := f.blockInfo[target]; ok { return info, nil } return &libvirt.DomainBlockJobInfo{}, nil } func TestCleanupJobEvidenceRequiresIdleDomainAndBlockJobs(t *testing.T) { document := `` idle := &cleanupJobDomainFake{ active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document, } evidence, safe := inspectCleanupJobEvidence(idle) joinedEvidence := strings.Join(evidence, "\n") if !safe || !reflect.DeepEqual(idle.blockQueries, []string{"vda", "vdb"}) || !strings.Contains(joinedEvidence, "domain-runtime:active:job-none") || !strings.Contains(joinedEvidence, "disk-target:vda") || !strings.Contains(joinedEvidence, "disk-target:vdb") || !strings.Contains(joinedEvidence, "persistent-domain-xml:") { t.Fatalf("idle job evidence=%v safe=%v queries=%v", evidence, safe, idle.blockQueries) } inactive := &cleanupJobDomainFake{document: document} evidence, safe = inspectCleanupJobEvidence(inactive) if !safe || len(inactive.blockQueries) != 0 || evidence[0] != "domain-runtime:inactive" { t.Fatalf("inactive job evidence=%v safe=%v queries=%v", evidence, safe, inactive.blockQueries) } for name, domain := range map[string]*cleanupJobDomainFake{ "active domain job": { active: true, job: activeBackupJob(), document: document, }, "active block job": { active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document, blockInfo: map[string]*libvirt.DomainBlockJobInfo{"vda": {Type: libvirt.DOMAIN_BLOCK_JOB_TYPE_COPY, End: 10}}, }, "block state unreadable": { active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document, blockErr: map[string]error{"vda": errors.New("query failed")}, }, "disk evidence incomplete": { active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: ``, }, } { t.Run(name, func(t *testing.T) { if evidence, safe := inspectCleanupJobEvidence(domain); safe || len(evidence) != 0 { t.Fatalf("unsafe job evidence=%v safe=%v", evidence, safe) } }) } } func TestBackupRecoveryEvidenceRequiresRootAndSeedConvergence(t *testing.T) { tests := []struct { name string replacement, disk, diskClaim, seed bool want backupRecoveryEvidence }{ {name: "root and seed committed", replacement: true, disk: true, diskClaim: true, seed: true, want: backupRecoveryEvidenceApplied}, {name: "nothing dispatched", want: backupRecoveryEvidenceNotApplied}, {name: "seed only", seed: true, want: backupRecoveryEvidenceUnknown}, {name: "root only", replacement: true, disk: true, diskClaim: true, want: backupRecoveryEvidenceUnknown}, {name: "replacement residue only", replacement: true, want: backupRecoveryEvidenceUnknown}, {name: "transition claim without volume", diskClaim: true, want: backupRecoveryEvidenceUnknown}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if got := classifyBackupRecoveryEvidence( test.replacement, test.disk, test.diskClaim, test.seed, ); got != test.want { t.Fatalf("evidence = %v, want %v", got, test.want) } }) } } func TestCleanupPlanIdentifierBindsPrivateExactEvidence(t *testing.T) { record := operation.Record{ OperationID: "operation-exact-cleanup-plan", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "delete-vm", Fingerprint: operation.Fingerprint( "DELETE", "/v1/vms/123e4567-e89b-42d3-a456-426614174000", []byte(`{}`), ), State: operation.StateFailed, EffectMayHaveApplied: true, } first := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:first")) second := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:second")) if first.Cleanup.PlanID == second.Cleanup.PlanID { t.Fatal("cleanup plan id did not change with the exact resource set") } encoded, err := json.Marshal(first) if err != nil { t.Fatal(err) } if string(encoded) == "" || containsAny(string(encoded), "managed-volume:first") || !strings.Contains(string(encoded), record.Fingerprint) { t.Fatalf("private cleanup evidence leaked: %s", encoded) } } func TestForceRequiredOperationsAlwaysUseFreshReadOnlyInspection(t *testing.T) { for _, action := range []string{"delete-vm", "service-suspend"} { record := operation.Record{ OperationID: "operation-fresh-" + strings.ReplaceAll(action, "-", ""), ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: action, State: operation.StateFailed, Error: &operation.PublicError{Code: "force_required", Message: "explicit force required"}, } if storedOperationInspectionAvailable(record) { t.Fatalf("%s force-required record reused cached inspection", action) } } ordinaryFailure := operation.Record{ Action: "power-start", State: operation.StateFailed, Error: &operation.PublicError{Code: "operation_failed"}, } if !storedOperationInspectionAvailable(ordinaryFailure) { t.Fatal("ordinary terminal failure unexpectedly required host inspection") } ordinaryFailure.EffectMayHaveApplied = true if storedOperationInspectionAvailable(ordinaryFailure) { t.Fatal("effectful terminal failure reused its stored failure instead of live host inspection") } } func TestOperationInspectionPublishesBoundedEffectEvidence(t *testing.T) { record := operation.Record{ OperationID: "operation-settings-effect-evidence", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "update-settings", Fingerprint: operation.Fingerprint("POST", "/v1/vms/123e4567-e89b-42d3-a456-426614174000/settings", []byte(`{}`)), State: operation.StateFailed, EffectMayHaveApplied: true, } resolution := operation.Resolution{ Resolved: true, Succeeded: true, Result: []byte(`{ "uuid":"123e4567-e89b-42d3-a456-426614174000", "name":"vm-123e4567", "status":"running", "vcpu":2, "memory_mib":2048, "hostname":"safe.example", "settings_changed":true, "root_password":"must-not-leak", "credential":"must-not-leak", "private_key":"must-not-leak", "public_keys":["must-not-leak"], "seed":"must-not-leak", "user-data":"must-not-leak" }`), } inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}) if inspection.Action != record.Action || inspection.ResourceID != record.ResourceID || inspection.RequestFingerprint != record.Fingerprint { t.Fatalf("inspection omitted immutable operation identity: %#v", inspection) } if inspection.EffectResolution != InspectionEffectApplied { t.Fatalf("effect resolution = %q, want %q", inspection.EffectResolution, InspectionEffectApplied) } if len(inspection.EffectProjection) == 0 { t.Fatal("applied settings inspection omitted its safe effect projection") } encoded, err := json.Marshal(inspection) if err != nil { t.Fatal(err) } for _, forbidden := range []string{ "must-not-leak", "root_password", "credential", "private_key", "public_keys", "seed", "user-data", } { if strings.Contains(string(encoded), forbidden) { t.Fatalf("inspection leaked forbidden effect data %q: %s", forbidden, encoded) } } if !strings.Contains(string(inspection.EffectProjection), `"settings_changed":true`) || !strings.Contains(string(inspection.EffectProjection), `"hostname":"safe.example"`) { t.Fatalf("settings projection omitted safe convergence evidence: %s", inspection.EffectProjection) } failed := buildOperationInspection(record, operation.Resolution{ Resolved: true, Succeeded: false, ErrorCode: "settings_not_committed", }, OperationAbortInspection{}, operationCleanupInspection{}) if failed.EffectResolution != InspectionEffectNotApplied || len(failed.EffectProjection) != 0 { t.Fatalf("failed effect inspection = %#v", failed) } exactNoEffect := inspectionFromRecord(operation.Record{ OperationID: "operation-settings-exact-no-effect", ResourceID: record.ResourceID, Action: "update-settings", State: operation.StateFailed, Error: &operation.PublicError{Code: "cloudinit_validation_failed"}, }) if exactNoEffect.EffectResolution != InspectionEffectNotApplied || exactNoEffect.FenceResolutionRequired || exactNoEffect.ErrorCode != "cloudinit_validation_failed" { t.Fatalf("exact no-effect Journal inspection = %#v", exactNoEffect) } partial := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, newOperationCleanupInspection([]string{CleanupStepPersistCleanupMetadata}, "private-host-evidence")) if partial.EffectResolution != InspectionEffectPartial || len(partial.EffectProjection) != 0 { t.Fatalf("partial effect inspection = %#v", partial) } inconclusive := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, operationCleanupInspection{}) if inconclusive.EffectResolution != InspectionEffectInconclusive || len(inconclusive.EffectProjection) != 0 { t.Fatalf("inconclusive effect inspection = %#v", inconclusive) } } func TestReceiptBackedTombstoneInspectionRequiresProjectableEvidence(t *testing.T) { journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = journal.Close() }) now := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC) journal.SetClockForTesting(func() time.Time { return now }) resourceID := "123e4567-e89b-42d3-a456-426614174000" spec := operation.Spec{ OperationID: "operation-pruned-settings-receipt", ResourceID: resourceID, Action: "update-settings", Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+resourceID+"/settings", []byte(`{}`)), Destructive: true, } result := json.RawMessage(`{ "uuid":"123e4567-e89b-42d3-a456-426614174000", "name":"vm-safe", "status":"running", "hostname":"safe.example", "settings_changed":true, "internal_note":"must-not-leak" }`) if _, _, err := journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return result, nil }); err != nil { t.Fatal(err) } claimed, err := journal.ClaimPendingCompletions(1) if err != nil || len(claimed) != 1 { t.Fatalf("completion claim = %#v err=%v", claimed, err) } if err := journal.AcknowledgeCompletion(claimed[0].EventID); err != nil { t.Fatal(err) } now = now.Add(91 * 24 * time.Hour) if accepted, err := journal.UseNonce("nonce-project-pruned-receipt", now, now.Add(time.Minute)); err != nil || !accepted { t.Fatalf("trigger detail pruning accepted=%v err=%v", accepted, err) } record, err := journal.GetForInspection(spec.OperationID) if err != nil || !record.Tombstone || !record.InspectionReceiptRecovered || record.InspectionEvidenceUnavailable { t.Fatalf("receipt-backed record = %#v err=%v", record, err) } inspection, err := (&Libvirt{}).InspectOperation(context.Background(), record) if err != nil || inspection.Outcome != InspectionOutcomeSucceeded || inspection.EffectResolution != InspectionEffectApplied || len(inspection.EffectProjection) == 0 { t.Fatalf("receipt-backed tombstone inspection = %#v err=%v", inspection, err) } encoded, err := json.Marshal(inspection) if err != nil { t.Fatal(err) } if strings.Contains(string(encoded), "must-not-leak") || strings.Contains(string(encoded), "internal_note") || !strings.Contains(string(inspection.EffectProjection), `"settings_changed":true`) { t.Fatalf("unsafe or incomplete receipt projection: %s", encoded) } t.Run("tampered receipt marker", func(t *testing.T) { tampered := record tampered.Result = nil tampered.InspectionReceiptRecovered = false tampered.InspectionEvidenceUnavailable = true inspection, err := (&Libvirt{}).InspectOperation(context.Background(), tampered) if err != nil || inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete || inspection.EffectResolution != InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 { t.Fatalf("tampered receipt inspection = %#v err=%v", inspection, err) } }) t.Run("unprojectable recovered result", func(t *testing.T) { unprojectable := record unprojectable.Result = json.RawMessage(`{"uuid":"123e4567-e89b-42d3-a456-426614174999","settings_changed":true}`) inspection, err := (&Libvirt{}).InspectOperation(context.Background(), unprojectable) if err != nil || inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete || inspection.EffectResolution != InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 { t.Fatalf("unprojectable receipt inspection = %#v err=%v", inspection, err) } }) } func TestTemplateOperationProjectionPreservesFinalizerIdentity(t *testing.T) { raw := []byte(`{ "id":"template-safe", "name":"Debian 13", "os_family":"linux", "version":"13", "architecture":"amd64", "size_bytes":1048576, "virtual_size_bytes":2147483648, "sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "revision":7, "generation_token":"0123456789abcdef0123456789abcdef", "created_at":"2026-10-20T01:00:00Z", "source_token":"must-not-leak" }`) for _, action := range []string{"import-template", "update-template"} { t.Run(action, func(t *testing.T) { inspection := inspectionFromRecord(operation.Record{ OperationID: "operation-template-projection-" + strings.ReplaceAll(action, "-", ""), ResourceID: "template:template-safe", Action: action, State: operation.StateSucceeded, Result: raw, }) var projection struct { ID string `json:"id"` Name string `json:"name"` Revision uint64 `json:"revision"` GenerationToken string `json:"generation_token"` } if err := json.Unmarshal(inspection.EffectProjection, &projection); err != nil { t.Fatalf("decode template projection: %v (%s)", err, inspection.EffectProjection) } if projection.ID != "template-safe" || projection.Name != "Debian 13" || projection.Revision != 7 || projection.GenerationToken != "0123456789abcdef0123456789abcdef" { t.Fatalf("template projection omitted finalizer identity: %#v", projection) } if strings.Contains(string(inspection.EffectProjection), "must-not-leak") || strings.Contains(string(inspection.EffectProjection), "source_token") { t.Fatalf("template projection leaked source credential: %s", inspection.EffectProjection) } }) } mismatched := operationEffectProjection(operation.Record{ ResourceID: "template:different", Action: "import-template", }, raw) if len(mismatched) != 0 { t.Fatalf("template identity mismatch produced projection: %s", mismatched) } } func TestRescueOperationProjectionPreservesMountedGeneration(t *testing.T) { raw := []byte(`{ "active":true, "staged":true, "template_id":"0123456789abcdef0123456789abcdef", "revision":7, "generation_token":"abcdef0123456789abcdef0123456789", "network_suspended":false, "warning":"must-not-leak" }`) projection := operationEffectProjection(operation.Record{ ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "enter-rescue", }, raw) var decoded RescueState if err := json.Unmarshal(projection, &decoded); err != nil { t.Fatalf("decode rescue projection: %v (%s)", err, projection) } if !decoded.Active || !decoded.Staged || decoded.Revision != 7 || decoded.TemplateID != "0123456789abcdef0123456789abcdef" || decoded.GenerationToken != "abcdef0123456789abcdef0123456789" { t.Fatalf("rescue projection omitted mounted generation: %#v", decoded) } if strings.Contains(string(projection), "must-not-leak") || strings.Contains(string(projection), "warning") { t.Fatalf("rescue projection leaked warning text: %s", projection) } } func TestVPCPortOperationProjectionBindsCompleteIdentity(t *testing.T) { resourceID := "123e4567-e89b-42d3-a456-426614174000" port := validVPCPortMetadata() expected := &operation.ExpectedState{ Kind: operation.ExpectedVPCPort, VPCPresent: true, VPCPortKey: port.PortKey, VPCKey: port.VPCKey, VPCPrivateIP: port.PrivateIP, VPCCIDR: port.CIDR, VPCGateway: port.Gateway, VPCMAC: port.MAC, VPCBridge: port.Bridge, VPCRevision: port.HostRevision, VPCDigest: port.HostDigest, } result := VPCPortResult{ UUID: resourceID, PortKey: port.PortKey, VPCKey: port.VPCKey, VPCPort: &port, Status: VMStateRunning, Attached: true, } raw, err := json.Marshal(result) if err != nil { t.Fatal(err) } record := operation.Record{ ResourceID: resourceID, Action: "apply-vpc-port", ExpectedState: expected, } projection := operationEffectProjection(record, raw) if len(projection) == 0 { t.Fatal("exact VPC port result did not produce a finalizer projection") } for name, mutate := range map[string]func(*VPCPortResult){ "resource": func(value *VPCPortResult) { value.UUID = "123e4567-e89b-42d3-a456-426614174999" }, "port key": func(value *VPCPortResult) { value.PortKey = "ffffffffffffffffffffffffffffffff" }, "private ip": func(value *VPCPortResult) { value.VPCPort.PrivateIP = "10.60.0.10" }, "host revision": func(value *VPCPortResult) { value.VPCPort.HostRevision++ }, "effect flags": func(value *VPCPortResult) { value.Removed = true }, } { t.Run(name, func(t *testing.T) { candidate := result candidatePort := port candidate.VPCPort = &candidatePort mutate(&candidate) encoded, marshalErr := json.Marshal(candidate) if marshalErr != nil { t.Fatal(marshalErr) } if projected := operationEffectProjection(record, encoded); len(projected) != 0 { t.Fatalf("mismatched VPC port produced projection: %s", projected) } }) } removeExpected := *expected removeExpected.VPCPresent = false removed := result removed.VPCPort = nil removed.Attached = false removed.Removed = true removeRaw, err := json.Marshal(removed) if err != nil { t.Fatal(err) } if projected := operationEffectProjection(operation.Record{ ResourceID: resourceID, Action: "remove-vpc-port", ExpectedState: &removeExpected, }, removeRaw); len(projected) == 0 { t.Fatal("exact VPC port removal did not produce a finalizer projection") } } func TestPortableBackupOperationProjectionPreservesFinalizerIdentity(t *testing.T) { backupKey := "0123456789abcdef0123456789abcdef" digest := strings.Repeat("a", 64) resourceID := "123e4567-e89b-42d3-a456-426614174000" backup := operationEffectProjection(operation.Record{ ResourceID: resourceID, Action: "create-backup:" + backupKey + ":filesystem", }, []byte(`{ "backup_key":"`+backupKey+`", "size_bytes":1048576, "virtual_size_bytes":2147483648, "sha256":"`+digest+`", "object_etag":"\"backup-etag\"", "object_version_id":"version/7+opaque", "consistency":"filesystem", "source_contract":{"contract":1,"architecture":"x86_64","machine":"q35","firmware":"uefi","secure_boot":false,"disk_format":"qcow2","system_disk_only":true,"public_network_contract":1,"vpc_inherited":false}, "source_url":"must-not-leak" }`)) var backupProjection BackupResult if err := json.Unmarshal(backup, &backupProjection); err != nil { t.Fatalf("decode backup projection: %v (%s)", err, backup) } if backupProjection.BackupKey != backupKey || backupProjection.SizeBytes != 1048576 || backupProjection.VirtualSizeBytes != 2147483648 || backupProjection.SHA256 != digest || backupProjection.ObjectETag != `"backup-etag"` || backupProjection.ObjectVersionID != "version/7+opaque" || backupProjection.Consistency != "filesystem" || backupProjection.SourceContract == nil || *backupProjection.SourceContract != fixedPortableRestoreContract() { t.Fatalf("backup projection omitted finalizer identity: %#v", backupProjection) } if strings.Contains(string(backup), "must-not-leak") || strings.Contains(string(backup), "source_url") { t.Fatalf("backup projection leaked transfer credential: %s", backup) } downgraded := operationEffectProjection(operation.Record{ ResourceID: resourceID, Action: "create-backup:" + backupKey + ":filesystem", }, []byte(`{"backup_key":"`+backupKey+`","size_bytes":1,"virtual_size_bytes":1,"sha256":"`+digest+`","consistency":"crash","source_contract":{"contract":1,"architecture":"x86_64","machine":"q35","firmware":"uefi","secure_boot":false,"disk_format":"qcow2","system_disk_only":true,"public_network_contract":1,"vpc_inherited":false}}`)) if len(downgraded) == 0 { t.Fatal("filesystem backup downgraded to crash consistency lost its finalizer projection") } missingContract := operationEffectProjection(operation.Record{ ResourceID: resourceID, Action: "create-backup:" + backupKey + ":filesystem", }, []byte(`{"backup_key":"`+backupKey+`","size_bytes":1,"virtual_size_bytes":1,"sha256":"`+digest+`","consistency":"filesystem"}`)) if len(missingContract) != 0 { t.Fatalf("backup without source contract produced projection: %s", missingContract) } wrongContract := fixedPortableRestoreContract() wrongContract.Machine = "pc-i440fx" wrongResult := BackupResult{ BackupKey: backupKey, SizeBytes: 1, VirtualSizeBytes: 1, SHA256: digest, Consistency: "filesystem", SourceContract: &wrongContract, } wrongRaw, err := json.Marshal(wrongResult) if err != nil { t.Fatal(err) } if projected := operationEffectProjection(operation.Record{ ResourceID: resourceID, Action: "create-backup:" + backupKey + ":filesystem", }, wrongRaw); len(projected) != 0 { t.Fatalf("backup with wrong source contract produced projection: %s", projected) } restore := operationEffectProjection(operation.Record{ ResourceID: resourceID, Action: "restore-backup:" + backupKey, }, []byte(`{ "uuid":"`+resourceID+`", "name":"restored-vm", "status":"running", "backup_key":"`+backupKey+`", "sha256":"`+digest+`", "source_url":"must-not-leak" }`)) var restoreProjection RestoreBackupResult if err := json.Unmarshal(restore, &restoreProjection); err != nil { t.Fatalf("decode restore projection: %v (%s)", err, restore) } if restoreProjection.UUID != resourceID || restoreProjection.Name != "restored-vm" || restoreProjection.BackupKey != backupKey || restoreProjection.SHA256 != digest { t.Fatalf("restore projection omitted finalizer identity: %#v", restoreProjection) } if strings.Contains(string(restore), "must-not-leak") || strings.Contains(string(restore), "source_url") { t.Fatalf("restore projection leaked transfer credential: %s", restore) } for name, test := range map[string]struct { record operation.Record raw []byte }{ "backup key mismatch": { record: operation.Record{ResourceID: resourceID, Action: "create-backup:" + backupKey + ":filesystem"}, raw: []byte(`{"backup_key":"ffffffffffffffffffffffffffffffff","size_bytes":1,"virtual_size_bytes":1,"sha256":"` + digest + `","consistency":"filesystem"}`), }, "backup consistency upgrade": { record: operation.Record{ResourceID: resourceID, Action: "create-backup:" + backupKey + ":crash"}, raw: []byte(`{"backup_key":"` + backupKey + `","size_bytes":1,"virtual_size_bytes":1,"sha256":"` + digest + `","consistency":"filesystem"}`), }, "restore uuid mismatch": { record: operation.Record{ResourceID: resourceID, Action: "restore-backup:" + backupKey}, raw: []byte(`{"uuid":"123e4567-e89b-42d3-a456-426614174999","backup_key":"` + backupKey + `","sha256":"` + digest + `"}`), }, "restore digest malformed": { record: operation.Record{ResourceID: resourceID, Action: "restore-backup:" + backupKey}, raw: []byte(`{"uuid":"` + resourceID + `","backup_key":"` + backupKey + `","sha256":"ABC"}`), }, } { t.Run(name, func(t *testing.T) { if projected := operationEffectProjection(test.record, test.raw); len(projected) != 0 { t.Fatalf("identity mismatch produced projection: %s", projected) } }) } } func TestPeerMigrationStateProjectionProvesPowerAndAutostart(t *testing.T) { resourceID := "123e4567-e89b-42d3-a456-426614174000" migrationID := strings.Repeat("a", 32) record := operation.Record{ ResourceID: resourceID, Action: "activate-peer-migration:" + migrationID, ExpectedState: &operation.ExpectedState{ Kind: operation.ExpectedPeerMigrationState, Enabled: true, StartIfSuspended: false, }, } projected := operationEffectProjection(record, []byte(`{ "uuid":"`+resourceID+`", "state":"shutoff", "running":false, "autostart_enabled":true, "name":"must-not-leak" }`)) var proof struct { UUID string `json:"uuid"` State VMStateName `json:"state"` Running bool `json:"running"` AutostartEnabled bool `json:"autostart_enabled"` } if err := json.Unmarshal(projected, &proof); err != nil { t.Fatalf("decode peer state projection: %v (%s)", err, projected) } if proof.UUID != resourceID || proof.State != VMStateShutoff || proof.Running || !proof.AutostartEnabled { t.Fatalf("peer state proof = %#v", proof) } if strings.Contains(string(projected), "must-not-leak") { t.Fatalf("peer state projection leaked non-contract data: %s", projected) } for name, raw := range map[string][]byte{ "wrong runtime": []byte(`{"uuid":"` + resourceID + `","state":"running","running":true,"autostart_enabled":true}`), "wrong autostart": []byte(`{"uuid":"` + resourceID + `","state":"shutoff","running":false,"autostart_enabled":false}`), "missing proof": []byte(`{"uuid":"` + resourceID + `","state":"shutoff"}`), } { t.Run(name, func(t *testing.T) { if value := operationEffectProjection(record, raw); len(value) != 0 { t.Fatalf("mismatched peer state produced projection: %s", value) } }) } } func TestPeerMigrationReceiveProjectionPreservesCanonicalNetworkAndVPC(t *testing.T) { resourceID := "123e4567-e89b-42d3-a456-426614174000" migrationID := strings.Repeat("a", 32) artifact := peermigration.Artifact{ ID: strings.Repeat("b", 32), Role: "root", Consistency: "crash", SizeBytes: 1, VirtualSizeBytes: 1, SHA256: strings.Repeat("c", 64), CreatedAt: time.Now().Unix(), } definition := validDomainDefinition() for _, withVPC := range []bool{false, true} { var port *VPCPortMetadata if withVPC { value := validVPCPortMetadata() port = &value } result := ReceivePeerMigrationResult{ VMMetadata: VMMetadata{ UUID: resourceID, Name: "migration-target", Status: VMStateShutoff, VCPUs: 2, MemoryMiB: 2048, DiskGiB: 20, BandwidthMbps: 100, PublicNetwork: definition.PublicNetwork, Machine: "q35", Firmware: "uefi", VPCPort: port, }, MigrationID: migrationID, ManifestDigest: strings.Repeat("d", 64), Artifacts: []peermigration.Artifact{artifact}, StagedPoweredOff: true, } raw, err := json.Marshal(result) if err != nil { t.Fatal(err) } projected := operationEffectProjection(operation.Record{ ResourceID: resourceID, Action: "receive-peer-migration:" + migrationID, }, raw) var proof struct { operationVMEffectProjection MigrationID string `json:"migration_id"` ManifestDigest string `json:"manifest_digest"` Artifacts []peermigration.Artifact `json:"artifacts"` StagedPoweredOff bool `json:"staged_powered_off"` } if err := json.Unmarshal(projected, &proof); err != nil { t.Fatalf("decode receive projection (VPC=%v): %v (%s)", withVPC, err, projected) } if proof.UUID != resourceID || proof.Status != VMStateShutoff || proof.MigrationID != migrationID || !proof.StagedPoweredOff || !reflect.DeepEqual(proof.PublicNetwork, definition.PublicNetwork) || !reflect.DeepEqual(proof.VPCPort, port) { t.Fatalf("receive projection lost canonical identity (VPC=%v): %#v", withVPC, proof) } } } func TestSuccessfulBackupCleanupPendingRetainsImmutableObjectProjection(t *testing.T) { key := "0123456789abcdef0123456789abcdef" record := operation.Record{ OperationID: "operation-backup-cleanup-pending", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "create-backup:" + key + ":crash", State: operation.StateSucceeded, Result: json.RawMessage(`{"backup_key":"` + key + `","size_bytes":1048576,"virtual_size_bytes":2147483648,"sha256":"` + strings.Repeat("a", 64) + `","consistency":"crash","cleanup_pending":true,` + `"source_contract":{"contract":1,"architecture":"x86_64","machine":"q35",` + `"firmware":"uefi","secure_boot":false,"disk_format":"qcow2",` + `"system_disk_only":true,"public_network_contract":1,"vpc_inherited":false}}`), } if !storedOperationInspectionAvailable(record) { t.Fatal("successful backup with local cleanup residue lost its immutable result") } inspection := inspectionFromRecord(record) if inspection.Outcome != InspectionOutcomeSucceeded || inspection.EffectResolution != InspectionEffectApplied || !strings.Contains(string(inspection.EffectProjection), `"cleanup_pending":true`) { t.Fatalf("backup cleanup-pending projection = %#v", inspection) } if !successfulBackupCleanupPending(record) { t.Fatal("valid successful backup cleanup marker was not recognized") } stored := &domainSnapshotMetadataXML{ Key: key, Consistency: "crash", SizeBytes: 2 << 30, } if !backupResidueCleanupAllowed(record, stored) { t.Fatal("successful operation-owned backup residue was not eligible for cleanup inspection") } cleanup := (&Libvirt{}).backupResidueCleanupInspection( &cleanupJobDomainFake{document: ``}, nil, snapshotVolumeName(record.ResourceID, key), key, stored, ) inspection = appliedBackupInspectionWithCleanup(record, inspection, cleanup) if inspection.Outcome != InspectionOutcomeSucceeded || inspection.EffectResolution != InspectionEffectApplied || !strings.Contains(string(inspection.EffectProjection), `"cleanup_pending":true`) || inspection.SummaryCode != InspectionSummaryCleanupRequired || !inspection.Cleanup.Available || !reflect.DeepEqual(inspection.Cleanup.Steps, []string{CleanupStepRemoveSnapshotMetadata}) { t.Fatalf("backup applied cleanup inspection = %#v", inspection) } withoutMarker := record withoutMarker.Result = json.RawMessage(strings.ReplaceAll(string(record.Result), `,"cleanup_pending":true`, "")) if successfulBackupCleanupPending(withoutMarker) || backupResidueCleanupAllowed(withoutMarker, stored) { t.Fatal("successful backup without cleanup marker exposed a cleanup plan") } } func TestUncertainTemplateUpdateDoesNotUseCurrentCatalogStateAsOperationEvidence(t *testing.T) { resolution, handled, err := (&Libvirt{}).reconcileCatalogOperation(context.Background(), operation.Record{ OperationID: "operation-template-update-inspect", ResourceID: "template:template-safe", Action: "update-template", State: operation.StateFailed, EffectMayHaveApplied: true, }) if err != nil || !handled || resolution.Resolved { t.Fatalf("template update inspection = %#v handled=%v err=%v", resolution, handled, err) } } func TestOperationInspectionDoesNotPromoteTargetStateToExactEffect(t *testing.T) { for name, record := range map[string]operation.Record{ "power target state": { OperationID: "operation-power-target-state", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "power-start", State: operation.StateFailed, EffectMayHaveApplied: true, }, "network link target state": { OperationID: "operation-network-target-state", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "network-suspend", State: operation.StateFailed, EffectMayHaveApplied: true, }, "storage partially configured": { OperationID: "operation-storage-target-state", ResourceID: "host-storage:7", Action: "configure-storage", State: operation.StateFailed, EffectMayHaveApplied: true, }, } { t.Run(name, func(t *testing.T) { for _, resolution := range []operation.Resolution{ {Resolved: true, Succeeded: true, Result: []byte(`{"network_enabled":false}`)}, {Resolved: true, Succeeded: false, ErrorCode: "network_state_not_applied"}, } { inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}) if inspection.EffectResolution != InspectionEffectInconclusive || inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete { t.Fatalf("target-state observation became exact effect verdict: %#v", inspection) } } }) } stored := operation.Record{ OperationID: "operation-power-journal-success", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "power-start", State: operation.StateSucceeded, } if inspection := inspectionFromRecord(stored); inspection.EffectResolution != InspectionEffectApplied { t.Fatalf("immutable Journal success lost its exact effect verdict: %#v", inspection) } else if inspection.FenceResolutionRequired { t.Fatalf("immutable Journal success incorrectly requires a fence resolution: %#v", inspection) } } func TestOperationInspectionNeverProjectsUnknownOrMalformedResults(t *testing.T) { for name, record := range map[string]operation.Record{ "unknown action": { OperationID: "operation-unknown-projection", ResourceID: "resource-unknown", Action: "future-secret-operation", State: operation.StateSucceeded, Result: []byte(`{"password":"must-not-leak"}`), }, "malformed known result": { OperationID: "operation-malformed-projection", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "update-settings", State: operation.StateSucceeded, Result: []byte(`{"hostname":"truncated"`), }, } { t.Run(name, func(t *testing.T) { inspection := inspectionFromRecord(record) if inspection.EffectResolution != InspectionEffectApplied { t.Fatalf("effect resolution = %q", inspection.EffectResolution) } if len(inspection.EffectProjection) != 0 { t.Fatalf("unsafe projection escaped: %s", inspection.EffectProjection) } encoded, err := json.Marshal(inspection) if err != nil { t.Fatal(err) } if strings.Contains(string(encoded), "must-not-leak") || strings.Contains(string(encoded), "truncated") { t.Fatalf("stored result leaked through inspection: %s", encoded) } }) } } func TestDeleteInspectionClassifiesActiveInactiveAndAbsentWithoutForceReplay(t *testing.T) { record := operation.Record{ OperationID: "operation-delete-inspection-states", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "delete-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed, Error: &operation.PublicError{Code: "force_required"}, } activeGraceful, handled := inspectActiveDeleteResolution(record, true) if !handled || !activeGraceful.Resolved || activeGraceful.Succeeded || activeGraceful.ErrorCode != "force_required" { t.Fatalf("active graceful inspection = %#v handled=%v", activeGraceful, handled) } record.State = operation.StateFailed record.EffectMayHaveApplied = true record.Error = nil crashWindow, handled := inspectActiveDeleteResolution(record, true) if !handled || !crashWindow.Resolved || crashWindow.Succeeded || crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("active graceful crash-window inspection did not require a safe retry = %#v handled=%v", crashWindow, handled) } record.State = operation.StateDispatching liveDispatch, handled := inspectActiveDeleteResolution(record, true) if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" { t.Fatalf("live graceful dispatch was closed before the native call returned = %#v handled=%v", liveDispatch, handled) } record.CauseCode = "result_uncertain" knownCause, handled := inspectActiveDeleteResolution(record, true) if !handled || knownCause.Resolved || knownCause.ErrorCode != "" { t.Fatalf("known-cause graceful operation was closed as a restart crash = %#v handled=%v", knownCause, handled) } record.CauseCode = "" if _, handled := inspectActiveDeleteResolution(record, false); handled { t.Fatal("inactive graceful delete was treated as an active force boundary") } record.CauseCode = "graceful_shutdown_ambiguous" ambiguousGraceful, handled := inspectActiveDeleteResolution(record, true) if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded || ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("ambiguous graceful delete did not close safely = %#v handled=%v", ambiguousGraceful, handled) } record.CauseCode = "" record.Action = "force-delete-vm" record.State = operation.StateFailed activeForce, handled := inspectActiveDeleteResolution(record, true) activeForceInspection := buildOperationInspection(record, activeForce, OperationAbortInspection{}, operationCleanupInspection{}) if !handled || !activeForce.Resolved || activeForce.Succeeded || activeForce.ErrorCode != "force_delete_unconfirmed_active" || activeForceInspection.Cleanup.Available || activeForceInspection.Outcome != InspectionOutcomeUncertain || activeForceInspection.EffectResolution != InspectionEffectInconclusive || activeForceInspection.SummaryCode != InspectionSummaryIncomplete { t.Fatalf("active force inspection = %#v resolution=%#v handled=%v", activeForceInspection, activeForce, handled) } record.State = operation.StateDispatching liveForce, handled := inspectActiveDeleteResolution(record, true) if !handled || liveForce.Resolved { t.Fatalf("live force dispatch was closed before its native call returned = %#v handled=%v", liveForce, handled) } record.State = operation.StateFailed record.EffectMayHaveApplied = true inactivePlan := newOperationCleanupInspection([]string{CleanupStepUndefineDomain}, "domain:inactive") inactiveInspection := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, inactivePlan) if !inactiveInspection.Cleanup.Available || inactiveInspection.Cleanup.Steps[0] != CleanupStepUndefineDomain || inactiveInspection.SummaryCode != InspectionSummaryCleanupRequired { t.Fatalf("inactive delete inspection = %#v", inactiveInspection) } absent := inspectAbsentDeleteResolution(operationCleanupInspection{}, true) absentInspection := buildOperationInspection(record, absent, OperationAbortInspection{}, operationCleanupInspection{}) var absentResult DeleteVMResult if err := json.Unmarshal(absent.Result, &absentResult); err != nil { t.Fatalf("decode force delete inspection result: %v", err) } if !absent.Resolved || !absent.Succeeded || absentInspection.Outcome != InspectionOutcomeUncertain || absentInspection.EffectResolution != InspectionEffectInconclusive || absentInspection.Cleanup.Available || !absentResult.Deleted || !absentResult.ManagedArtifactsAbsent || !absentResult.Forced { t.Fatalf("absent delete inspection = %#v resolution=%#v", absentInspection, absent) } normalAbsent := inspectAbsentDeleteResolution(operationCleanupInspection{}, false) var normalResult DeleteVMResult if err := json.Unmarshal(normalAbsent.Result, &normalResult); err != nil || !normalResult.Deleted || !normalResult.ManagedArtifactsAbsent || normalResult.Forced { t.Fatalf("normal delete inspection result=%#v err=%v", normalResult, err) } absentResidue := newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:one") residueResolution := inspectAbsentDeleteResolution(absentResidue, true) residueInspection := buildOperationInspection(record, residueResolution, OperationAbortInspection{}, absentResidue) if residueResolution.Resolved || !residueInspection.Cleanup.Available || residueInspection.SummaryCode != InspectionSummaryCleanupRequired { t.Fatalf("absent delete residue inspection = %#v resolution=%#v", residueInspection, residueResolution) } } func TestServiceSuspendInspectionDoesNotInventForceAuthorizationAfterDispatchCrash(t *testing.T) { record := operation.Record{ OperationID: "operation-service-suspend-crash-window", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "service-suspend", State: operation.StateFailed, EffectMayHaveApplied: true, ExpectedState: &operation.ExpectedState{ Kind: operation.ExpectedServiceState, }, } crashWindow, handled := inspectActiveServiceSuspension(record, true) if !handled || !crashWindow.Resolved || crashWindow.Succeeded || crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("dispatch crash did not require a new graceful suspension = %#v handled=%v", crashWindow, handled) } record.State = operation.StateDispatching liveDispatch, handled := inspectActiveServiceSuspension(record, true) if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" { t.Fatalf("live graceful suspension was closed before the native call returned = %#v handled=%v", liveDispatch, handled) } record.State = operation.StateFailed record.EffectMayHaveApplied = true record.CauseCode = "graceful_shutdown_ambiguous" ambiguousGraceful, handled := inspectActiveServiceSuspension(record, true) if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded || ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("ambiguous graceful suspension did not close safely = %#v handled=%v", ambiguousGraceful, handled) } record.CauseCode = "" record.State = operation.StateFailed record.Error = &operation.PublicError{Code: "force_required", Message: "explicit force required"} authoritative, handled := inspectActiveServiceSuspension(record, true) if !handled || !authoritative.Resolved || authoritative.Succeeded || authoritative.ErrorCode != "force_required" { t.Fatalf("authoritative graceful timeout was lost = %#v handled=%v", authoritative, handled) } } func containsAny(value string, candidates ...string) bool { for _, candidate := range candidates { if len(candidate) != 0 && len(value) >= len(candidate) && strings.Contains(value, candidate) { return true } } return false } func TestAbsentCreateTrafficResidueRequiresExplicitCleanup(t *testing.T) { record := operation.Record{ OperationID: "operation-create-traffic-residue", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "create-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed, EffectMayHaveApplied: true, } resolution, steps := inspectAbsentCreateVM(record.ResourceID, true, true) inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, steps) if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryCleanupRequired || !inspection.Cleanup.Available || len(inspection.Cleanup.Steps) != 1 || inspection.Cleanup.Steps[0] != CleanupStepRemoveTrafficState || !lowerHexDigest(inspection.Cleanup.PlanID) { t.Fatalf("create traffic residue inspection = %#v", inspection) } resolution, steps = inspectAbsentCreateVM(record.ResourceID, true, false) inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps) if inspection.Outcome != InspectionOutcomeFailed || inspection.SummaryCode != InspectionSummaryFailed || inspection.ErrorCode != "creation_absent" || inspection.EffectResolution != InspectionEffectNotApplied || inspection.Cleanup.Available { t.Fatalf("clean absent create inspection = %#v", inspection) } var absentProjection struct { CreationAbsent bool `json:"creation_absent"` ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"` } if len(inspection.EffectProjection) == 0 || json.Unmarshal(inspection.EffectProjection, &absentProjection) != nil || !absentProjection.CreationAbsent || !absentProjection.ManagedArtifactsAbsent { t.Fatalf("clean absent create projection = %s", inspection.EffectProjection) } resolution, steps = inspectAbsentCreateVM(record.ResourceID, false, true) inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps) if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete || inspection.Cleanup.Available { t.Fatalf("unproven create absence inspection = %#v", inspection) } } func TestFailedCreateAlwaysRequiresLiveInspection(t *testing.T) { for name, record := range map[string]operation.Record{ "ordinary failure without effect flag": { OperationID: "operation-create-ordinary-failure", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "create-vm", State: operation.StateFailed, Error: &operation.PublicError{Code: "operation_failed", Message: "create failed"}, }, "unknown effect failure": { OperationID: "operation-create-unknown-effect", ResourceID: "123e4567-e89b-42d3-a456-426614174001", Action: "create-vm", State: operation.StateFailed, EffectMayHaveApplied: true, Error: &operation.PublicError{Code: "result_unconfirmed", Message: "create result was not confirmed"}, }, } { t.Run(name, func(t *testing.T) { if storedOperationInspectionAvailable(record) { t.Fatal("failed create reused its original Journal error instead of requiring live host inspection") } }) } } func TestBackupPushMatchesOnlyExactOwnedTarget(t *testing.T) { target := "/var/lib/virt-cloud/managed/snapshot.qcow2" matching := `` if !backupPushMatchesTarget(matching, target) { t.Fatal("exact push backup target was not recognized") } for name, document := range map[string]string{ "foreign target": ``, "wrong disk": ``, "pull backup": ``, "multiple disks": ``, } { t.Run(name, func(t *testing.T) { if backupPushMatchesTarget(document, target) { t.Fatal("non-matching backup was accepted") } }) } } func TestBackupResidueCleanupRequiresTerminalOwnedSnapshot(t *testing.T) { key := "0123456789abcdef0123456789abcdef" domain := &cleanupJobDomainFake{ document: ``, } for name, consistency := range map[string]struct { requested string actual string }{ "crash": {requested: "crash", actual: "crash"}, "filesystem fallback": {requested: "filesystem", actual: "crash"}, } { t.Run(name, func(t *testing.T) { record := operation.Record{ OperationID: "operation-backup-residue-" + strings.ReplaceAll(name, " ", "-"), ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "create-backup:" + key + ":" + consistency.requested, State: operation.StateFailed, EffectMayHaveApplied: true, } stored := &domainSnapshotMetadataXML{ Key: key, Consistency: consistency.actual, SizeBytes: 1 << 30, } if !backupResidueCleanupAllowed(record, stored) { t.Fatal("terminal operation-owned residue was not eligible for cleanup") } plan := (&Libvirt{}).backupResidueCleanupInspection( domain, nil, snapshotVolumeName(record.ResourceID, key), key, stored, ) inspection := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, plan) if !inspection.Cleanup.Available || !reflect.DeepEqual(inspection.Cleanup.Steps, []string{CleanupStepRemoveSnapshotMetadata}) || inspection.EffectResolution != InspectionEffectPartial || inspection.Outcome != InspectionOutcomeUncertain { t.Fatalf("backup residue inspection = %#v", inspection) } }) } active := operation.Record{ OperationID: "operation-backup-residue-active", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "create-backup:" + key + ":filesystem", State: operation.StateDispatching, } stored := &domainSnapshotMetadataXML{Key: key, Consistency: "crash", SizeBytes: 1 << 30} if backupResidueCleanupAllowed(active, stored) { t.Fatal("active backup worker exposed a local residue cleanup plan") } fenced := active fenced.State = operation.StateFailed fenced.EffectMayHaveApplied = true fenced.ExecutionFenced = true if backupResidueCleanupAllowed(fenced, stored) { t.Fatal("execution-fenced backup worker exposed a local residue cleanup plan") } failed := active failed.State = operation.StateFailed failed.EffectMayHaveApplied = true busyDomain := &cleanupJobDomainFake{ active: true, job: activeBackupJob(), document: domain.document, } if plan := (&Libvirt{}).backupResidueCleanupInspection( busyDomain, nil, snapshotVolumeName(failed.ResourceID, key), key, stored, ); len(plan.steps) != 0 { t.Fatalf("active libvirt backup job exposed cleanup steps: %#v", plan.steps) } }