//go:build libvirt package hypervisor import ( "context" "encoding/json" "errors" "path/filepath" "reflect" "strings" "testing" "time" "git.luoli.live/root/virt/agent/operation" "libvirt.org/go/libvirt" ) type cleanupJobDomainFake struct { active bool activeErr error job *libvirt.DomainJobInfo jobErr error document string inactiveXML string xmlErr error blockInfo map[string]*libvirt.DomainBlockJobInfo blockErr map[string]error blockQueries []string } func (f *cleanupJobDomainFake) IsActive() (bool, error) { return f.active, f.activeErr } func (f *cleanupJobDomainFake) GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) { return f.job, f.jobErr } func (f *cleanupJobDomainFake) GetXMLDesc(flags libvirt.DomainXMLFlags) (string, error) { if flags == libvirt.DOMAIN_XML_INACTIVE && f.inactiveXML != "" { return f.inactiveXML, f.xmlErr } return f.document, f.xmlErr } func (f *cleanupJobDomainFake) GetBlockJobInfo(target string, _ libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error) { f.blockQueries = append(f.blockQueries, target) if err := f.blockErr[target]; err != nil { return nil, err } if info, ok := f.blockInfo[target]; ok { return info, nil } return &libvirt.DomainBlockJobInfo{}, nil } func TestCleanupJobEvidenceRequiresIdleDomainAndBlockJobs(t *testing.T) { document := `` idle := &cleanupJobDomainFake{ active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document, } evidence, safe := inspectCleanupJobEvidence(idle) joinedEvidence := strings.Join(evidence, "\n") if !safe || !reflect.DeepEqual(idle.blockQueries, []string{"vda", "vdb"}) || !strings.Contains(joinedEvidence, "domain-runtime:active:job-none") || !strings.Contains(joinedEvidence, "disk-target:vda") || !strings.Contains(joinedEvidence, "disk-target:vdb") || !strings.Contains(joinedEvidence, "persistent-domain-xml:") { t.Fatalf("idle job evidence=%v safe=%v queries=%v", evidence, safe, idle.blockQueries) } inactive := &cleanupJobDomainFake{document: document} evidence, safe = inspectCleanupJobEvidence(inactive) if !safe || len(inactive.blockQueries) != 0 || evidence[0] != "domain-runtime:inactive" { t.Fatalf("inactive job evidence=%v safe=%v queries=%v", evidence, safe, inactive.blockQueries) } for name, domain := range map[string]*cleanupJobDomainFake{ "active domain job": { active: true, job: activeBackupJob(), document: document, }, "active block job": { active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document, blockInfo: map[string]*libvirt.DomainBlockJobInfo{"vda": {Type: libvirt.DOMAIN_BLOCK_JOB_TYPE_COPY, End: 10}}, }, "block state unreadable": { active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document, blockErr: map[string]error{"vda": errors.New("query failed")}, }, "disk evidence incomplete": { active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: ``, }, } { t.Run(name, func(t *testing.T) { if evidence, safe := inspectCleanupJobEvidence(domain); safe || len(evidence) != 0 { t.Fatalf("unsafe job evidence=%v safe=%v", evidence, safe) } }) } } func TestCleanupPlanIdentifierBindsPrivateExactEvidence(t *testing.T) { record := operation.Record{ OperationID: "operation-exact-cleanup-plan", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "delete-vm", Fingerprint: operation.Fingerprint( "DELETE", "/v1/vms/123e4567-e89b-42d3-a456-426614174000", []byte(`{}`), ), State: operation.StateFailed, EffectMayHaveApplied: true, } first := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:first")) second := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:second")) if first.Cleanup.PlanID == second.Cleanup.PlanID { t.Fatal("cleanup plan id did not change with the exact resource set") } encoded, err := json.Marshal(first) if err != nil { t.Fatal(err) } if string(encoded) == "" || containsAny(string(encoded), "managed-volume:first") || !strings.Contains(string(encoded), record.Fingerprint) { t.Fatalf("private cleanup evidence leaked: %s", encoded) } } func TestForceRequiredOperationsAlwaysUseFreshReadOnlyInspection(t *testing.T) { for _, action := range []string{"delete-vm", "service-suspend"} { record := operation.Record{ OperationID: "operation-fresh-" + strings.ReplaceAll(action, "-", ""), ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: action, State: operation.StateFailed, Error: &operation.PublicError{Code: "force_required", Message: "explicit force required"}, } if storedOperationInspectionAvailable(record) { t.Fatalf("%s force-required record reused cached inspection", action) } } ordinaryFailure := operation.Record{ Action: "power-start", State: operation.StateFailed, Error: &operation.PublicError{Code: "operation_failed"}, } if !storedOperationInspectionAvailable(ordinaryFailure) { t.Fatal("ordinary terminal failure unexpectedly required host inspection") } ordinaryFailure.EffectMayHaveApplied = true if storedOperationInspectionAvailable(ordinaryFailure) { t.Fatal("effectful terminal failure reused its stored failure instead of live host inspection") } } func TestOperationInspectionPublishesBoundedEffectEvidence(t *testing.T) { record := operation.Record{ OperationID: "operation-settings-effect-evidence", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "update-settings", Fingerprint: operation.Fingerprint("POST", "/v1/vms/123e4567-e89b-42d3-a456-426614174000/settings", []byte(`{}`)), State: operation.StateFailed, EffectMayHaveApplied: true, } resolution := operation.Resolution{ Resolved: true, Succeeded: true, Result: []byte(`{ "uuid":"123e4567-e89b-42d3-a456-426614174000", "name":"vm-123e4567", "status":"running", "vcpu":2, "memory_mib":2048, "hostname":"safe.example", "settings_changed":true, "root_password":"must-not-leak", "credential":"must-not-leak", "private_key":"must-not-leak", "public_keys":["must-not-leak"], "seed":"must-not-leak", "user-data":"must-not-leak" }`), } inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}) if inspection.Action != record.Action || inspection.ResourceID != record.ResourceID || inspection.RequestFingerprint != record.Fingerprint { t.Fatalf("inspection omitted immutable operation identity: %#v", inspection) } if inspection.EffectResolution != InspectionEffectApplied { t.Fatalf("effect resolution = %q, want %q", inspection.EffectResolution, InspectionEffectApplied) } if len(inspection.EffectProjection) == 0 { t.Fatal("applied settings inspection omitted its safe effect projection") } encoded, err := json.Marshal(inspection) if err != nil { t.Fatal(err) } for _, forbidden := range []string{ "must-not-leak", "root_password", "credential", "private_key", "public_keys", "seed", "user-data", } { if strings.Contains(string(encoded), forbidden) { t.Fatalf("inspection leaked forbidden effect data %q: %s", forbidden, encoded) } } if !strings.Contains(string(inspection.EffectProjection), `"settings_changed":true`) || !strings.Contains(string(inspection.EffectProjection), `"hostname":"safe.example"`) { t.Fatalf("settings projection omitted safe convergence evidence: %s", inspection.EffectProjection) } failed := buildOperationInspection(record, operation.Resolution{ Resolved: true, Succeeded: false, ErrorCode: "settings_not_committed", }, OperationAbortInspection{}, operationCleanupInspection{}) if failed.EffectResolution != InspectionEffectNotApplied || len(failed.EffectProjection) != 0 { t.Fatalf("failed effect inspection = %#v", failed) } exactNoEffect := inspectionFromRecord(operation.Record{ OperationID: "operation-settings-exact-no-effect", ResourceID: record.ResourceID, Action: "update-settings", State: operation.StateFailed, Error: &operation.PublicError{Code: "cloudinit_validation_failed"}, }) if exactNoEffect.EffectResolution != InspectionEffectNotApplied || exactNoEffect.FenceResolutionRequired || exactNoEffect.ErrorCode != "cloudinit_validation_failed" { t.Fatalf("exact no-effect Journal inspection = %#v", exactNoEffect) } partial := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, newOperationCleanupInspection([]string{CleanupStepPersistCleanupMetadata}, "private-host-evidence")) if partial.EffectResolution != InspectionEffectPartial || len(partial.EffectProjection) != 0 { t.Fatalf("partial effect inspection = %#v", partial) } inconclusive := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, operationCleanupInspection{}) if inconclusive.EffectResolution != InspectionEffectInconclusive || len(inconclusive.EffectProjection) != 0 { t.Fatalf("inconclusive effect inspection = %#v", inconclusive) } } func TestReceiptBackedTombstoneInspectionRequiresProjectableEvidence(t *testing.T) { journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = journal.Close() }) now := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC) journal.SetClockForTesting(func() time.Time { return now }) resourceID := "123e4567-e89b-42d3-a456-426614174000" spec := operation.Spec{ OperationID: "operation-pruned-settings-receipt", ResourceID: resourceID, Action: "update-settings", Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+resourceID+"/settings", []byte(`{}`)), Destructive: true, } result := json.RawMessage(`{ "uuid":"123e4567-e89b-42d3-a456-426614174000", "name":"vm-safe", "status":"running", "hostname":"safe.example", "settings_changed":true, "internal_note":"must-not-leak" }`) if _, _, err := journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { return result, nil }); err != nil { t.Fatal(err) } claimed, err := journal.ClaimPendingCompletions(1) if err != nil || len(claimed) != 1 { t.Fatalf("completion claim = %#v err=%v", claimed, err) } if err := journal.AcknowledgeCompletion(claimed[0].EventID); err != nil { t.Fatal(err) } now = now.Add(91 * 24 * time.Hour) if accepted, err := journal.UseNonce("nonce-project-pruned-receipt", now, now.Add(time.Minute)); err != nil || !accepted { t.Fatalf("trigger detail pruning accepted=%v err=%v", accepted, err) } record, err := journal.GetForInspection(spec.OperationID) if err != nil || !record.Tombstone || !record.InspectionReceiptRecovered || record.InspectionEvidenceUnavailable { t.Fatalf("receipt-backed record = %#v err=%v", record, err) } inspection, err := (&Libvirt{}).InspectOperation(context.Background(), record) if err != nil || inspection.Outcome != InspectionOutcomeSucceeded || inspection.EffectResolution != InspectionEffectApplied || len(inspection.EffectProjection) == 0 { t.Fatalf("receipt-backed tombstone inspection = %#v err=%v", inspection, err) } encoded, err := json.Marshal(inspection) if err != nil { t.Fatal(err) } if strings.Contains(string(encoded), "must-not-leak") || strings.Contains(string(encoded), "internal_note") || !strings.Contains(string(inspection.EffectProjection), `"settings_changed":true`) { t.Fatalf("unsafe or incomplete receipt projection: %s", encoded) } t.Run("tampered receipt marker", func(t *testing.T) { tampered := record tampered.Result = nil tampered.InspectionReceiptRecovered = false tampered.InspectionEvidenceUnavailable = true inspection, err := (&Libvirt{}).InspectOperation(context.Background(), tampered) if err != nil || inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete || inspection.EffectResolution != InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 { t.Fatalf("tampered receipt inspection = %#v err=%v", inspection, err) } }) t.Run("unprojectable recovered result", func(t *testing.T) { unprojectable := record unprojectable.Result = json.RawMessage(`{"uuid":"123e4567-e89b-42d3-a456-426614174999","settings_changed":true}`) inspection, err := (&Libvirt{}).InspectOperation(context.Background(), unprojectable) if err != nil || inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete || inspection.EffectResolution != InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 { t.Fatalf("unprojectable receipt inspection = %#v err=%v", inspection, err) } }) } func TestTemplateOperationProjectionPreservesFinalizerIdentity(t *testing.T) { raw := []byte(`{ "id":"template-safe", "name":"Debian 13", "os_family":"linux", "version":"13", "architecture":"amd64", "size_bytes":1048576, "virtual_size_bytes":2147483648, "sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "revision":7, "generation_token":"0123456789abcdef0123456789abcdef", "created_at":"2026-10-20T01:00:00Z", "source_token":"must-not-leak" }`) for _, action := range []string{"import-template", "update-template"} { t.Run(action, func(t *testing.T) { inspection := inspectionFromRecord(operation.Record{ OperationID: "operation-template-projection-" + strings.ReplaceAll(action, "-", ""), ResourceID: "template:template-safe", Action: action, State: operation.StateSucceeded, Result: raw, }) var projection struct { ID string `json:"id"` Name string `json:"name"` Revision uint64 `json:"revision"` GenerationToken string `json:"generation_token"` } if err := json.Unmarshal(inspection.EffectProjection, &projection); err != nil { t.Fatalf("decode template projection: %v (%s)", err, inspection.EffectProjection) } if projection.ID != "template-safe" || projection.Name != "Debian 13" || projection.Revision != 7 || projection.GenerationToken != "0123456789abcdef0123456789abcdef" { t.Fatalf("template projection omitted finalizer identity: %#v", projection) } if strings.Contains(string(inspection.EffectProjection), "must-not-leak") || strings.Contains(string(inspection.EffectProjection), "source_token") { t.Fatalf("template projection leaked source credential: %s", inspection.EffectProjection) } }) } mismatched := operationEffectProjection(operation.Record{ ResourceID: "template:different", Action: "import-template", }, raw) if len(mismatched) != 0 { t.Fatalf("template identity mismatch produced projection: %s", mismatched) } } func TestUncertainTemplateUpdateDoesNotUseCurrentCatalogStateAsOperationEvidence(t *testing.T) { resolution, handled, err := (&Libvirt{}).reconcileCatalogOperation(context.Background(), operation.Record{ OperationID: "operation-template-update-inspect", ResourceID: "template:template-safe", Action: "update-template", State: operation.StateFailed, EffectMayHaveApplied: true, }) if err != nil || !handled || resolution.Resolved { t.Fatalf("template update inspection = %#v handled=%v err=%v", resolution, handled, err) } } func TestOperationInspectionDoesNotPromoteTargetStateToExactEffect(t *testing.T) { for name, record := range map[string]operation.Record{ "power target state": { OperationID: "operation-power-target-state", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "power-start", State: operation.StateFailed, EffectMayHaveApplied: true, }, "network link target state": { OperationID: "operation-network-target-state", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "network-suspend", State: operation.StateFailed, EffectMayHaveApplied: true, }, "storage partially configured": { OperationID: "operation-storage-target-state", ResourceID: "host-storage:7", Action: "configure-storage", State: operation.StateFailed, EffectMayHaveApplied: true, }, } { t.Run(name, func(t *testing.T) { for _, resolution := range []operation.Resolution{ {Resolved: true, Succeeded: true, Result: []byte(`{"network_enabled":false}`)}, {Resolved: true, Succeeded: false, ErrorCode: "network_state_not_applied"}, } { inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}) if inspection.EffectResolution != InspectionEffectInconclusive || inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete { t.Fatalf("target-state observation became exact effect verdict: %#v", inspection) } } }) } stored := operation.Record{ OperationID: "operation-power-journal-success", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "power-start", State: operation.StateSucceeded, } if inspection := inspectionFromRecord(stored); inspection.EffectResolution != InspectionEffectApplied { t.Fatalf("immutable Journal success lost its exact effect verdict: %#v", inspection) } else if inspection.FenceResolutionRequired { t.Fatalf("immutable Journal success incorrectly requires a fence resolution: %#v", inspection) } } func TestOperationInspectionNeverProjectsUnknownOrMalformedResults(t *testing.T) { for name, record := range map[string]operation.Record{ "unknown action": { OperationID: "operation-unknown-projection", ResourceID: "resource-unknown", Action: "future-secret-operation", State: operation.StateSucceeded, Result: []byte(`{"password":"must-not-leak"}`), }, "malformed known result": { OperationID: "operation-malformed-projection", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "update-settings", State: operation.StateSucceeded, Result: []byte(`{"hostname":"truncated"`), }, } { t.Run(name, func(t *testing.T) { inspection := inspectionFromRecord(record) if inspection.EffectResolution != InspectionEffectApplied { t.Fatalf("effect resolution = %q", inspection.EffectResolution) } if len(inspection.EffectProjection) != 0 { t.Fatalf("unsafe projection escaped: %s", inspection.EffectProjection) } encoded, err := json.Marshal(inspection) if err != nil { t.Fatal(err) } if strings.Contains(string(encoded), "must-not-leak") || strings.Contains(string(encoded), "truncated") { t.Fatalf("stored result leaked through inspection: %s", encoded) } }) } } func TestDeleteInspectionClassifiesActiveInactiveAndAbsentWithoutForceReplay(t *testing.T) { record := operation.Record{ OperationID: "operation-delete-inspection-states", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "delete-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed, Error: &operation.PublicError{Code: "force_required"}, } activeGraceful, handled := inspectActiveDeleteResolution(record, true) if !handled || !activeGraceful.Resolved || activeGraceful.Succeeded || activeGraceful.ErrorCode != "force_required" { t.Fatalf("active graceful inspection = %#v handled=%v", activeGraceful, handled) } record.State = operation.StateFailed record.EffectMayHaveApplied = true record.Error = nil crashWindow, handled := inspectActiveDeleteResolution(record, true) if !handled || !crashWindow.Resolved || crashWindow.Succeeded || crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("active graceful crash-window inspection did not require a safe retry = %#v handled=%v", crashWindow, handled) } record.State = operation.StateDispatching liveDispatch, handled := inspectActiveDeleteResolution(record, true) if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" { t.Fatalf("live graceful dispatch was closed before the native call returned = %#v handled=%v", liveDispatch, handled) } record.CauseCode = "result_uncertain" knownCause, handled := inspectActiveDeleteResolution(record, true) if !handled || knownCause.Resolved || knownCause.ErrorCode != "" { t.Fatalf("known-cause graceful operation was closed as a restart crash = %#v handled=%v", knownCause, handled) } record.CauseCode = "" if _, handled := inspectActiveDeleteResolution(record, false); handled { t.Fatal("inactive graceful delete was treated as an active force boundary") } record.CauseCode = "graceful_shutdown_ambiguous" ambiguousGraceful, handled := inspectActiveDeleteResolution(record, true) if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded || ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("ambiguous graceful delete did not close safely = %#v handled=%v", ambiguousGraceful, handled) } record.CauseCode = "" record.Action = "force-delete-vm" record.State = operation.StateFailed activeForce, handled := inspectActiveDeleteResolution(record, true) activeForceInspection := buildOperationInspection(record, activeForce, OperationAbortInspection{}, operationCleanupInspection{}) if !handled || !activeForce.Resolved || activeForce.Succeeded || activeForce.ErrorCode != "force_delete_unconfirmed_active" || activeForceInspection.Cleanup.Available || activeForceInspection.Outcome != InspectionOutcomeUncertain || activeForceInspection.EffectResolution != InspectionEffectInconclusive || activeForceInspection.SummaryCode != InspectionSummaryIncomplete { t.Fatalf("active force inspection = %#v resolution=%#v handled=%v", activeForceInspection, activeForce, handled) } record.State = operation.StateDispatching liveForce, handled := inspectActiveDeleteResolution(record, true) if !handled || liveForce.Resolved { t.Fatalf("live force dispatch was closed before its native call returned = %#v handled=%v", liveForce, handled) } record.State = operation.StateFailed record.EffectMayHaveApplied = true inactivePlan := newOperationCleanupInspection([]string{CleanupStepUndefineDomain}, "domain:inactive") inactiveInspection := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, inactivePlan) if !inactiveInspection.Cleanup.Available || inactiveInspection.Cleanup.Steps[0] != CleanupStepUndefineDomain || inactiveInspection.SummaryCode != InspectionSummaryCleanupRequired { t.Fatalf("inactive delete inspection = %#v", inactiveInspection) } absent := inspectAbsentDeleteResolution(operationCleanupInspection{}, true) absentInspection := buildOperationInspection(record, absent, OperationAbortInspection{}, operationCleanupInspection{}) var absentResult DeleteVMResult if err := json.Unmarshal(absent.Result, &absentResult); err != nil { t.Fatalf("decode force delete inspection result: %v", err) } if !absent.Resolved || !absent.Succeeded || absentInspection.Outcome != InspectionOutcomeUncertain || absentInspection.EffectResolution != InspectionEffectInconclusive || absentInspection.Cleanup.Available || !absentResult.Deleted || !absentResult.ManagedArtifactsAbsent || !absentResult.Forced { t.Fatalf("absent delete inspection = %#v resolution=%#v", absentInspection, absent) } normalAbsent := inspectAbsentDeleteResolution(operationCleanupInspection{}, false) var normalResult DeleteVMResult if err := json.Unmarshal(normalAbsent.Result, &normalResult); err != nil || !normalResult.Deleted || !normalResult.ManagedArtifactsAbsent || normalResult.Forced { t.Fatalf("normal delete inspection result=%#v err=%v", normalResult, err) } absentResidue := newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:one") residueResolution := inspectAbsentDeleteResolution(absentResidue, true) residueInspection := buildOperationInspection(record, residueResolution, OperationAbortInspection{}, absentResidue) if residueResolution.Resolved || !residueInspection.Cleanup.Available || residueInspection.SummaryCode != InspectionSummaryCleanupRequired { t.Fatalf("absent delete residue inspection = %#v resolution=%#v", residueInspection, residueResolution) } } func TestServiceSuspendInspectionDoesNotInventForceAuthorizationAfterDispatchCrash(t *testing.T) { record := operation.Record{ OperationID: "operation-service-suspend-crash-window", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "service-suspend", State: operation.StateFailed, EffectMayHaveApplied: true, ExpectedState: &operation.ExpectedState{ Kind: operation.ExpectedServiceState, }, } crashWindow, handled := inspectActiveServiceSuspension(record, true) if !handled || !crashWindow.Resolved || crashWindow.Succeeded || crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("dispatch crash did not require a new graceful suspension = %#v handled=%v", crashWindow, handled) } record.State = operation.StateDispatching liveDispatch, handled := inspectActiveServiceSuspension(record, true) if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" { t.Fatalf("live graceful suspension was closed before the native call returned = %#v handled=%v", liveDispatch, handled) } record.State = operation.StateFailed record.EffectMayHaveApplied = true record.CauseCode = "graceful_shutdown_ambiguous" ambiguousGraceful, handled := inspectActiveServiceSuspension(record, true) if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded || ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" { t.Fatalf("ambiguous graceful suspension did not close safely = %#v handled=%v", ambiguousGraceful, handled) } record.CauseCode = "" record.State = operation.StateFailed record.Error = &operation.PublicError{Code: "force_required", Message: "explicit force required"} authoritative, handled := inspectActiveServiceSuspension(record, true) if !handled || !authoritative.Resolved || authoritative.Succeeded || authoritative.ErrorCode != "force_required" { t.Fatalf("authoritative graceful timeout was lost = %#v handled=%v", authoritative, handled) } } func containsAny(value string, candidates ...string) bool { for _, candidate := range candidates { if len(candidate) != 0 && len(value) >= len(candidate) && strings.Contains(value, candidate) { return true } } return false } func TestAbsentCreateTrafficResidueRequiresExplicitCleanup(t *testing.T) { record := operation.Record{ OperationID: "operation-create-traffic-residue", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "create-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed, EffectMayHaveApplied: true, } resolution, steps := inspectAbsentCreateVM(record.ResourceID, true, true) inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, steps) if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryCleanupRequired || !inspection.Cleanup.Available || len(inspection.Cleanup.Steps) != 1 || inspection.Cleanup.Steps[0] != CleanupStepRemoveTrafficState || !lowerHexDigest(inspection.Cleanup.PlanID) { t.Fatalf("create traffic residue inspection = %#v", inspection) } resolution, steps = inspectAbsentCreateVM(record.ResourceID, true, false) inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps) if inspection.Outcome != InspectionOutcomeFailed || inspection.SummaryCode != InspectionSummaryFailed || inspection.ErrorCode != "creation_absent" || inspection.EffectResolution != InspectionEffectNotApplied || inspection.Cleanup.Available { t.Fatalf("clean absent create inspection = %#v", inspection) } var absentProjection struct { CreationAbsent bool `json:"creation_absent"` ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"` } if len(inspection.EffectProjection) == 0 || json.Unmarshal(inspection.EffectProjection, &absentProjection) != nil || !absentProjection.CreationAbsent || !absentProjection.ManagedArtifactsAbsent { t.Fatalf("clean absent create projection = %s", inspection.EffectProjection) } resolution, steps = inspectAbsentCreateVM(record.ResourceID, false, true) inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps) if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete || inspection.Cleanup.Available { t.Fatalf("unproven create absence inspection = %#v", inspection) } } func TestFailedCreateAlwaysRequiresLiveInspection(t *testing.T) { for name, record := range map[string]operation.Record{ "ordinary failure without effect flag": { OperationID: "operation-create-ordinary-failure", ResourceID: "123e4567-e89b-42d3-a456-426614174000", Action: "create-vm", State: operation.StateFailed, Error: &operation.PublicError{Code: "operation_failed", Message: "create failed"}, }, "unknown effect failure": { OperationID: "operation-create-unknown-effect", ResourceID: "123e4567-e89b-42d3-a456-426614174001", Action: "create-vm", State: operation.StateFailed, EffectMayHaveApplied: true, Error: &operation.PublicError{Code: "result_unconfirmed", Message: "create result was not confirmed"}, }, } { t.Run(name, func(t *testing.T) { if storedOperationInspectionAvailable(record) { t.Fatal("failed create reused its original Journal error instead of requiring live host inspection") } }) } } func TestBackupPushMatchesOnlyExactOwnedTarget(t *testing.T) { target := "/var/lib/virt-cloud/managed/snapshot.qcow2" matching := `` if !backupPushMatchesTarget(matching, target) { t.Fatal("exact push backup target was not recognized") } for name, document := range map[string]string{ "foreign target": ``, "wrong disk": ``, "pull backup": ``, "multiple disks": ``, } { t.Run(name, func(t *testing.T) { if backupPushMatchesTarget(document, target) { t.Fatal("non-matching backup was accepted") } }) } }