//go:build libvirt package hypervisor import ( "encoding/json" "errors" "reflect" "strings" "testing" "git.luoli.live/root/virt/agent/operation" "libvirt.org/go/libvirt" ) type cleanupJobDomainFake struct { active bool activeErr error job *libvirt.DomainJobInfo jobErr error document string inactiveXML string xmlErr error blockInfo map[string]*libvirt.DomainBlockJobInfo blockErr map[string]error blockQueries []string } func (f *cleanupJobDomainFake) IsActive() (bool, error) { return f.active, f.activeErr } func (f *cleanupJobDomainFake) GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) { return f.job, f.jobErr } func (f *cleanupJobDomainFake) GetXMLDesc(flags libvirt.DomainXMLFlags) (string, error) { if flags == libvirt.DOMAIN_XML_INACTIVE && f.inactiveXML != "" { return f.inactiveXML, f.xmlErr } return f.document, f.xmlErr } func (f *cleanupJobDomainFake) GetBlockJobInfo(target string, _ libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error) { f.blockQueries = append(f.blockQueries, target) if err := f.blockErr[target]; err != nil { return nil, err } if info, ok := f.blockInfo[target]; ok { return info, nil } return &libvirt.DomainBlockJobInfo{}, nil } func TestCleanupJobEvidenceRequiresIdleDomainAndBlockJobs(t *testing.T) { document := `` idle := &cleanupJobDomainFake{ active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document, } evidence, safe := inspectCleanupJobEvidence(idle) joinedEvidence := strings.Join(evidence, "\n") if !safe || !reflect.DeepEqual(idle.blockQueries, []string{"vda", "vdb"}) || !strings.Contains(joinedEvidence, "domain-runtime:active:job-none") || !strings.Contains(joinedEvidence, "disk-target:vda") || !strings.Contains(joinedEvidence, "disk-target:vdb") || !strings.Contains(joinedEvidence, "persistent-domain-xml:") { t.Fatalf("idle job evidence=%v safe=%v queries=%v", evidence, safe, idle.blockQueries) } inactive := &cleanupJobDomainFake{document: document} evidence, safe = inspectCleanupJobEvidence(inactive) if !safe || len(inactive.blockQueries) != 0 || evidence[0] != "domain-runtime:inactive" { t.Fatalf("inactive job evidence=%v safe=%v queries=%v", evidence, safe, inactive.blockQueries) } for name, domain := range map[string]*cleanupJobDomainFake{ "active domain job": { active: true, job: activeBackupJob(), document: document, }, "active block job": { active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document, blockInfo: map[string]*libvirt.DomainBlockJobInfo{"vda": {Type: libvirt.DOMAIN_BLOCK_JOB_TYPE_COPY, End: 10}}, }, "block state unreadable": { active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document, blockErr: map[string]error{"vda": errors.New("query failed")}, }, "disk evidence incomplete": { active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: ``, }, } { t.Run(name, func(t *testing.T) { if evidence, safe := inspectCleanupJobEvidence(domain); safe || len(evidence) != 0 { t.Fatalf("unsafe job evidence=%v safe=%v", evidence, safe) } }) } } func TestCleanupPlanIdentifierBindsPrivateExactEvidence(t *testing.T) { record := operation.Record{ OperationID: "operation-exact-cleanup-plan", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "delete-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed, EffectMayHaveApplied: true, } first := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:first")) second := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:second")) if first.Cleanup.PlanID == second.Cleanup.PlanID { t.Fatal("cleanup plan id did not change with the exact resource set") } encoded, err := json.Marshal(first) if err != nil { t.Fatal(err) } if string(encoded) == "" || containsAny(string(encoded), "managed-volume:first", "request-fingerprint") { t.Fatalf("private cleanup evidence leaked: %s", encoded) } } func TestForceRequiredOperationsAlwaysUseFreshReadOnlyInspection(t *testing.T) { for _, action := range []string{"delete-vm", "service-suspend"} { record := operation.Record{ OperationID: "operation-fresh-" + strings.ReplaceAll(action, "-", ""), ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: action, State: operation.StateFailed, Error: &operation.PublicError{Code: "force_required", Message: "explicit force required"}, } if storedOperationInspectionAvailable(record) { t.Fatalf("%s force-required record reused cached inspection", action) } } ordinaryFailure := operation.Record{ Action: "power-start", State: operation.StateFailed, Error: &operation.PublicError{Code: "operation_failed"}, } if !storedOperationInspectionAvailable(ordinaryFailure) { t.Fatal("ordinary terminal failure unexpectedly required host inspection") } } func TestDeleteInspectionClassifiesActiveInactiveAndAbsentWithoutForceReplay(t *testing.T) { record := operation.Record{ OperationID: "operation-delete-inspection-states", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "delete-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed, Error: &operation.PublicError{Code: "force_required"}, } activeGraceful, handled := inspectActiveDeleteResolution(record, true) if !handled || !activeGraceful.Resolved || activeGraceful.Succeeded || activeGraceful.ErrorCode != "force_required" { t.Fatalf("active graceful inspection = %#v handled=%v", activeGraceful, handled) } record.State = operation.StateFailed record.EffectMayHaveApplied = true record.Error = nil crashWindow, handled := inspectActiveDeleteResolution(record, true) if !handled || !crashWindow.Resolved || crashWindow.Succeeded || crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("active graceful crash-window inspection did not require a safe retry = %#v handled=%v", crashWindow, handled) } record.State = operation.StateDispatching liveDispatch, handled := inspectActiveDeleteResolution(record, true) if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" { t.Fatalf("live graceful dispatch was closed before the native call returned = %#v handled=%v", liveDispatch, handled) } record.CauseCode = "result_uncertain" knownCause, handled := inspectActiveDeleteResolution(record, true) if !handled || knownCause.Resolved || knownCause.ErrorCode != "" { t.Fatalf("known-cause graceful operation was closed as a restart crash = %#v handled=%v", knownCause, handled) } record.CauseCode = "" if _, handled := inspectActiveDeleteResolution(record, false); handled { t.Fatal("inactive graceful delete was treated as an active force boundary") } record.CauseCode = "graceful_shutdown_ambiguous" ambiguousGraceful, handled := inspectActiveDeleteResolution(record, true) if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded || ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("ambiguous graceful delete did not close safely = %#v handled=%v", ambiguousGraceful, handled) } record.CauseCode = "" record.Action = "force-delete-vm" record.State = operation.StateFailed activeForce, handled := inspectActiveDeleteResolution(record, true) activeForceInspection := buildOperationInspection(record, activeForce, OperationAbortInspection{}, operationCleanupInspection{}) if !handled || !activeForce.Resolved || activeForce.Succeeded || activeForce.ErrorCode != "force_delete_unconfirmed_active" || activeForceInspection.Cleanup.Available || activeForceInspection.Outcome != InspectionOutcomeFailed || activeForceInspection.SummaryCode != InspectionSummaryFailed { t.Fatalf("active force inspection = %#v resolution=%#v handled=%v", activeForceInspection, activeForce, handled) } record.State = operation.StateDispatching liveForce, handled := inspectActiveDeleteResolution(record, true) if !handled || liveForce.Resolved { t.Fatalf("live force dispatch was closed before its native call returned = %#v handled=%v", liveForce, handled) } record.State = operation.StateFailed record.EffectMayHaveApplied = true inactivePlan := newOperationCleanupInspection([]string{CleanupStepUndefineDomain}, "domain:inactive") inactiveInspection := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, inactivePlan) if !inactiveInspection.Cleanup.Available || inactiveInspection.Cleanup.Steps[0] != CleanupStepUndefineDomain || inactiveInspection.SummaryCode != InspectionSummaryCleanupRequired { t.Fatalf("inactive delete inspection = %#v", inactiveInspection) } absent := inspectAbsentDeleteResolution(operationCleanupInspection{}, true) absentInspection := buildOperationInspection(record, absent, OperationAbortInspection{}, operationCleanupInspection{}) var absentResult DeleteVMResult if err := json.Unmarshal(absent.Result, &absentResult); err != nil { t.Fatalf("decode force delete inspection result: %v", err) } if !absent.Resolved || !absent.Succeeded || absentInspection.Outcome != InspectionOutcomeSucceeded || absentInspection.Cleanup.Available || !absentResult.Deleted || !absentResult.ManagedArtifactsAbsent || !absentResult.Forced { t.Fatalf("absent delete inspection = %#v resolution=%#v", absentInspection, absent) } normalAbsent := inspectAbsentDeleteResolution(operationCleanupInspection{}, false) var normalResult DeleteVMResult if err := json.Unmarshal(normalAbsent.Result, &normalResult); err != nil || !normalResult.Deleted || !normalResult.ManagedArtifactsAbsent || normalResult.Forced { t.Fatalf("normal delete inspection result=%#v err=%v", normalResult, err) } absentResidue := newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:one") residueResolution := inspectAbsentDeleteResolution(absentResidue, true) residueInspection := buildOperationInspection(record, residueResolution, OperationAbortInspection{}, absentResidue) if residueResolution.Resolved || !residueInspection.Cleanup.Available || residueInspection.SummaryCode != InspectionSummaryCleanupRequired { t.Fatalf("absent delete residue inspection = %#v resolution=%#v", residueInspection, residueResolution) } } func TestServiceSuspendInspectionDoesNotInventForceAuthorizationAfterDispatchCrash(t *testing.T) { record := operation.Record{ OperationID: "operation-service-suspend-crash-window", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "service-suspend", State: operation.StateFailed, EffectMayHaveApplied: true, ExpectedState: &operation.ExpectedState{ Kind: operation.ExpectedServiceState, }, } crashWindow, handled := inspectActiveServiceSuspension(record, true) if !handled || !crashWindow.Resolved || crashWindow.Succeeded || crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("dispatch crash did not require a new graceful suspension = %#v handled=%v", crashWindow, handled) } record.State = operation.StateDispatching liveDispatch, handled := inspectActiveServiceSuspension(record, true) if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" { t.Fatalf("live graceful suspension was closed before the native call returned = %#v handled=%v", liveDispatch, handled) } record.State = operation.StateFailed record.EffectMayHaveApplied = true record.CauseCode = "graceful_shutdown_ambiguous" ambiguousGraceful, handled := inspectActiveServiceSuspension(record, true) if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded || ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("ambiguous graceful suspension did not close safely = %#v handled=%v", ambiguousGraceful, handled) } record.CauseCode = "" record.State = operation.StateFailed record.Error = &operation.PublicError{Code: "force_required", Message: "explicit force required"} authoritative, handled := inspectActiveServiceSuspension(record, true) if !handled || !authoritative.Resolved || authoritative.Succeeded || authoritative.ErrorCode != "force_required" { t.Fatalf("authoritative graceful timeout was lost = %#v handled=%v", authoritative, handled) } } func containsAny(value string, candidates ...string) bool { for _, candidate := range candidates { if len(candidate) != 0 && len(value) >= len(candidate) && strings.Contains(value, candidate) { return true } } return false } func TestAbsentCreateTrafficResidueRequiresExplicitCleanup(t *testing.T) { record := operation.Record{ OperationID: "operation-create-traffic-residue", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "create-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed, EffectMayHaveApplied: true, } resolution, steps := inspectAbsentCreateVM(record.ResourceID, true, true) inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, steps) if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryCleanupRequired || !inspection.Cleanup.Available || len(inspection.Cleanup.Steps) != 1 || inspection.Cleanup.Steps[0] != CleanupStepRemoveTrafficState || !lowerHexDigest(inspection.Cleanup.PlanID) { t.Fatalf("create traffic residue inspection = %#v", inspection) } resolution, steps = inspectAbsentCreateVM(record.ResourceID, true, false) inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps) if inspection.Outcome != InspectionOutcomeFailed || inspection.SummaryCode != InspectionSummaryFailed || inspection.ErrorCode != "creation_absent" || inspection.Cleanup.Available { t.Fatalf("clean absent create inspection = %#v", inspection) } resolution, steps = inspectAbsentCreateVM(record.ResourceID, false, true) inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps) if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete || inspection.Cleanup.Available { t.Fatalf("unproven create absence inspection = %#v", inspection) } } func TestBackupPushMatchesOnlyExactOwnedTarget(t *testing.T) { target := "/var/lib/virt-cloud/managed/snapshot.qcow2" matching := `` if !backupPushMatchesTarget(matching, target) { t.Fatal("exact push backup target was not recognized") } for name, document := range map[string]string{ "foreign target": ``, "wrong disk": ``, "pull backup": ``, "multiple disks": ``, } { t.Run(name, func(t *testing.T) { if backupPushMatchesTarget(document, target) { t.Fatal("non-matching backup was accepted") } }) } }