//go:build libvirt package hypervisor import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "time" "git.luoli.live/root/virt/agent/operation" "libvirt.org/go/libvirt" "libvirt.org/go/libvirtxml" ) var inspectionErrorCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`) // operationCleanupInspection carries the exact, private host evidence used to // bind a cleanup plan. Only the stable step codes are serialized; resource // names, XML digests, and file attributes stay inside the Agent and feed the // one-way plan identifier. type operationCleanupInspection struct { steps []string evidence []string } func newOperationCleanupInspection(steps []string, evidence ...string) operationCleanupInspection { if len(evidence) == 0 { return operationCleanupInspection{} } normalizedEvidence := append([]string(nil), evidence...) sort.Strings(normalizedEvidence) return operationCleanupInspection{ steps: append([]string(nil), steps...), evidence: normalizedEvidence, } } func (inspection operationCleanupInspection) tokenEvidence() string { encoded, _ := json.Marshal(struct { Steps []string `json:"steps"` Evidence []string `json:"evidence"` }{Steps: inspection.steps, Evidence: inspection.evidence}) return string(encoded) } // InspectOperation deliberately bypasses Journal admission. A dispatched // libvirt call retains its resource lease until the real call returns, but an // administrator must still be able to read its state and identify an owned // cancellable backup job while that call is stuck. Every operation below is a // libvirt/filesystem read. It may return a cleanup plan, but inspection never // executes that plan or changes a host resource. func (l *Libvirt) InspectOperation(ctx context.Context, record operation.Record) (OperationInspection, error) { if err := ctx.Err(); err != nil { return OperationInspection{}, err } if storedOperationInspectionAvailable(record) { if record.InspectionEvidenceUnavailable { return inconclusiveInspectionFromRecord(record), nil } inspection := inspectionFromRecord(record) // A recovered successful receipt is useful for convergence only when its // action-specific result can be reduced to the bounded public projection. // Receipt status alone must not turn an unprojectable or damaged result // into an "applied" verdict after Journal detail retention. if record.InspectionReceiptRecovered && record.State == operation.StateSucceeded && len(inspection.EffectProjection) == 0 { return inconclusiveInspectionFromRecord(record), nil } return inspection, nil } if resolution, handled, err := l.reconcileStorageConfiguration(record); handled { return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err } if resolution, handled, err := l.reconcileCatalogOperation(ctx, record); handled { return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err } domain, err := l.domainByUUID(ctx, record.ResourceID) if errors.Is(err, ErrNotFound) { return l.inspectAbsentDomainOperation(ctx, record) } if err != nil { return OperationInspection{}, err } defer domain.Free() abort := l.inspectMatchingBackupJob(record, domain) resolution, cleanup, inspectErr := l.inspectExistingDomainOperation(ctx, record, domain) if abort.Available { // No cleanup may be approved while the matching backup is still live. cleanup = operationCleanupInspection{} } return buildOperationInspection(record, resolution, abort, cleanup), inspectErr } func storedOperationInspectionAvailable(record operation.Record) bool { if record.Tombstone { return true } if record.State == operation.StateSucceeded && !recordHasCleanupPending(record) { return true } if record.State == operation.StateFailed && record.EffectMayHaveApplied { // A failed Journal row describes the response observed by the original // caller, not necessarily the durable host effect. Reuse would make the // operation look definitively failed forever and bypass the read-only // expected-state reconcilers below. return false } return record.State == operation.StateFailed && record.Action != "create-vm" && !operation.RequiresExplicitForceDelete(record) } func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record operation.Record) (OperationInspection, error) { var resolution operation.Resolution var cleanup operationCleanupInspection switch record.Action { case "create-vm": absent, err := l.deleteArtifactsAbsent(record.ResourceID) if err != nil { return OperationInspection{}, err } resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, absent, l.traffic.contains(record.ResourceID)) case "delete-vm", "force-delete-vm": plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil) if err != nil { return OperationInspection{}, err } cleanup = plan resolution = inspectAbsentDeleteResolution(plan, record.Action == "force-delete-vm") default: if record.ExpectedState != nil { resolution = expectedStateResourceAbsent(record) } } return buildOperationInspection(record, resolution, OperationAbortInspection{}, cleanup), nil } func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePresent bool) (operation.Resolution, operationCleanupInspection) { if !managedArtifactsAbsent { return operation.Resolution{}, operationCleanupInspection{} } resolution := operation.Resolution{ Resolved: true, Succeeded: false, Result: []byte(`{"creation_absent":true,"managed_artifacts_absent":true}`), ErrorCode: "creation_absent", ErrorMessage: "host evidence confirms the VM and managed artifacts are absent", } if trafficStatePresent { return resolution, newOperationCleanupInspection( []string{CleanupStepRemoveTrafficState}, "traffic-state:"+uuid, ) } return resolution, operationCleanupInspection{} } func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection, error) { if record.Action == "delete-vm" || record.Action == "force-delete-vm" { _, managed, err := requireManagedDomain(domain) if err != nil { return operation.Resolution{}, operationCleanupInspection{}, err } if err := l.validateDeleteVMArtifacts(ctx, record.ResourceID, managed); err != nil { return operation.Resolution{}, operationCleanupInspection{}, err } active, err := domain.IsActive() if err != nil { return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err) } if resolution, handled := inspectActiveDeleteResolution(record, active); handled { return resolution, operationCleanupInspection{}, nil } plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, domain, managed) return operation.Resolution{}, plan, err } if record.ExpectedState != nil { switch record.ExpectedState.Kind { case operation.ExpectedUpdateSettings, operation.ExpectedChangeNetwork, operation.ExpectedNetworkState, operation.ExpectedServiceState, operation.ExpectedFirewall: resolution, _, err := l.inspectExpectedStateOperation(ctx, record, domain) return resolution, operationCleanupInspection{}, err } } if record.Action == "reinstall-vm" { resolution, steps := l.inspectReinstallOperation(record, domain) return resolution, steps, nil } if strings.HasPrefix(record.Action, "create-snapshot:") || strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:") { resolution, steps := l.inspectSnapshotOperation(record, domain) return resolution, steps, nil } if record.Action == "enter-rescue" || record.Action == "force-start-rescue" || strings.HasPrefix(record.Action, "exit-rescue:") { return l.reconcileRescue(ctx, record, domain), operationCleanupInspection{}, nil } state, _, err := domain.GetState() if err != nil { return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect operation domain state", err) } mapped := mapDomainState(state) switch record.Action { case "create-vm": resolution, err := l.reconcileCreatedVM(domain) return resolution, operationCleanupInspection{}, err case "power-start": if mapped == VMStateRunning || mapped == VMStateBlocked { return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil } case "power-shutdown", "power-destroy": if mapped == VMStateShutoff { return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil } } return operation.Resolution{}, operationCleanupInspection{}, nil } func inspectActiveDeleteResolution(record operation.Record, active bool) (operation.Resolution, bool) { if !active { return operation.Resolution{}, false } if record.Action == "delete-vm" && operation.RequiresExplicitForceDelete(record) { // Preserve an authoritative force_required outcome written by the // original graceful DeleteVM call after its bounded wait. return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "force_required", ErrorMessage: "guest remains active and requires an explicitly authorized force delete"}, true } if record.Action == "delete-vm" && (record.CauseCode == "graceful_shutdown_ambiguous" || recoveredGracefulAttemptRequiresRetry(record)) { // The prior ShutdownFlags response was lost and activity could not be // observed conclusively, or the Agent restarted at the dispatch // boundary before it persisted any cause. A later active observation // can classify the already-terminal operation as failed, but cannot grant // force authorization or dispatch another mutation. return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "graceful_shutdown_unconfirmed", ErrorMessage: "graceful shutdown result was not confirmed; the resource remains blocked for manual remediation"}, true } if record.Action == "force-delete-vm" && forceDeleteCanBeConfirmedForRetry(record) { // Never replay DestroyFlags automatically. Once no native call can still // hold Journal admission, a token-confirmed administrator inspection may // close an old force attempt which still observes an active Domain. The // original graceful force_required record remains the authorization // anchor, so a retry still needs a new operation ID and data-loss ack. return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "force_delete_unconfirmed_active", ErrorMessage: "force delete did not reach an inactive domain; inspect and explicitly authorize a new force operation"}, true } // A force operation may already have crossed Destroy before its result was // lost. Read-only inspection must not turn that ambiguity into a cleanup // plan which would replay Destroy. return operation.Resolution{}, true } func forceDeleteCanBeConfirmedForRetry(record operation.Record) bool { return record.State == operation.StateFailed && record.EffectMayHaveApplied } func recoveredGracefulAttemptRequiresRetry(record operation.Record) bool { if record.CauseCode != "" { return false } return record.State == operation.StateFailed && record.EffectMayHaveApplied } func inspectAbsentDeleteResolution(cleanup operationCleanupInspection, forced bool) operation.Resolution { if len(cleanup.steps) != 0 { return operation.Resolution{} } return resolvedDeleteSuccess(DeleteVMResult{ Deleted: true, ManagedArtifactsAbsent: true, Forced: forced, }) } func (l *Libvirt) inspectDeleteCleanupSteps(ctx context.Context, uuid string, domain *libvirt.Domain, managed *domainMetadataXML) (operationCleanupInspection, error) { steps := make([]string, 0, 7) evidence := make([]string, 0, 8) if domain != nil { jobEvidence, safe := inspectCleanupJobEvidence(domain) if !safe { return operationCleanupInspection{}, nil } evidence = append(evidence, jobEvidence...) deleteEvidence, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain) if err != nil { return operationCleanupInspection{}, err } evidence = append(evidence, deleteEvidence...) active, err := domain.IsActive() if err != nil { return operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err) } if active { return operationCleanupInspection{}, nil } evidence = append(evidence, "domain:"+uuid+":inactive") steps = append(steps, CleanupStepUndefineDomain) } pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged)) if err != nil { return operationCleanupInspection{}, translateLibvirtError("inspect managed storage pool", err) } inventory, err := pool.ListStorageVolumes() if err != nil { pool.Free() return operationCleanupInspection{}, translateLibvirtError("inspect managed volumes", err) } names, err := collectVMManagedVolumeNames(uuid, managed, inventory) if err != nil { pool.Free() return operationCleanupInspection{}, err } managedVolumesExist := false for _, name := range names { if err := ctx.Err(); err != nil { pool.Free() return operationCleanupInspection{}, err } volume, lookupErr := pool.LookupStorageVolByName(name) if isNoStorageVolume(lookupErr) { continue } if lookupErr != nil { pool.Free() return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", lookupErr) } managedVolumesExist = true if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil { volume.Free() pool.Free() return operationCleanupInspection{}, validateErr } volumeDigest, digestErr := storageVolumeInspectionDigest(volume) if digestErr != nil { volume.Free() pool.Free() return operationCleanupInspection{}, digestErr } path, pathErr := volume.GetPath() volume.Free() if pathErr != nil { pool.Free() return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", pathErr) } var referenced bool if domain != nil { referenced, err = l.otherDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), name, uuid, ) } else { referenced, err = l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), name, ) } if err != nil { pool.Free() return operationCleanupInspection{}, err } if referenced { pool.Free() return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed cleanup volume is referenced by another domain", nil) } evidence = append(evidence, "managed-volume:"+name+":"+volumeDigest) } pool.Free() if managedVolumesExist { steps = append(steps, CleanupStepDeleteManagedVolumes) } seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid)) if err != nil { return operationCleanupInspection{}, err } if seed != nil { seedDigest, digestErr := storageVolumeInspectionDigest(seed) if digestErr != nil { seed.Free() return operationCleanupInspection{}, digestErr } path, pathErr := seed.GetPath() seed.Free() if pathErr != nil { return operationCleanupInspection{}, translateLibvirtError("inspect managed seed cleanup volume", pathErr) } referenced, referenceErr := l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), ) if domain != nil { referenced, referenceErr = l.otherDomainUsesVolume( path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid, ) } if referenceErr != nil { return operationCleanupInspection{}, referenceErr } if referenced { return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed seed cleanup volume is referenced by another domain", nil) } steps = append(steps, CleanupStepDeleteSeedVolume) evidence = append(evidence, "seed-volume:"+seedVolumeName(uuid)+":"+seedDigest) } filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid)) if err != nil { return operationCleanupInspection{}, err } if filter != nil { document, xmlErr := filter.GetXMLDesc(0) filter.Free() if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) { return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_invalid", "network filter cannot be confirmed as panel managed", xmlErr) } excludedUUID := "" if domain != nil { excludedUUID = uuid } referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), excludedUUID) if referenceErr != nil { return operationCleanupInspection{}, referenceErr } if referenced { return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil) } steps = append(steps, CleanupStepUndefineNetworkFilter) digest := sha256.Sum256([]byte(document)) evidence = append(evidence, "network-filter:"+networkFilterName(uuid)+":"+hex.EncodeToString(digest[:])) } nvramExists := false for _, name := range managedNVRAMArtifactNames(uuid) { path := filepath.Join(l.local.NVRAMDirectory, name) info, statErr := os.Lstat(path) if errors.Is(statErr, os.ErrNotExist) { continue } if statErr != nil { return operationCleanupInspection{}, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr) } if !info.Mode().IsRegular() { return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil) } referenced, referenceErr := l.anyDomainUsesPath(path) if domain != nil { referenced, referenceErr = l.otherDomainUsesPath(path, uuid) } if referenceErr != nil { return operationCleanupInspection{}, referenceErr } if referenced { return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil) } nvramExists = true evidence = append(evidence, "nvram:"+name+":"+strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10)) } if nvramExists { steps = append(steps, CleanupStepDeleteNVRAM) } trafficPresent := l.traffic.contains(uuid) if trafficPresent || len(steps) != 0 { // cleanupDeletedVMArtifacts durably rewrites traffic state even if an // earlier failed fsync left only an on-disk entry. Make that mutation // explicit in every delete cleanup plan. steps = append(steps, CleanupStepRemoveTrafficState) if trafficPresent { evidence = append(evidence, "traffic-state:"+uuid+":present") } else { evidence = append(evidence, "traffic-state:"+uuid+":rewrite") } } return newOperationCleanupInspection(steps, evidence...), nil } func managedFilterDocument(document, expectedName string) bool { var filter libvirtxml.NWFilter if filter.Unmarshal(document) != nil || filter.Name != expectedName || filter.Chain != "root" { return false } baselineReferences := 0 for _, entry := range filter.Entries { if entry.Ref == nil { continue } if entry.Ref.Filter != "clean-traffic" { return false } baselineReferences++ } return baselineReferences == 1 } func storageVolumeInspectionDigest(volume *libvirt.StorageVol) (string, error) { document, err := volume.GetXMLDesc(0) if err != nil { return "", translateLibvirtError("read cleanup volume XML", err) } digest := sha256.Sum256([]byte(document)) return hex.EncodeToString(digest[:]), nil } func (l *Libvirt) inspectReinstallOperation(record operation.Record, domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) { document, metadata, err := requireManagedDomain(domain) if err != nil { return operation.Resolution{}, operationCleanupInspection{} } replacementName := reinstallVolumeName(record.ResourceID, record.OperationID) replacement, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName) if err != nil { return operation.Resolution{}, operationCleanupInspection{} } replacementExists := replacement != nil if replacement != nil { replacement.Free() } if diskTransitionCommitted(metadata, replacementName, "reinstall", record.OperationID) && replacementExists { state, _, stateErr := domain.GetState() if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "reinstall", record.OperationID) { return operation.Resolution{}, operationCleanupInspection{} } if metadata.PendingCleanupVolume != "" { return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata) } result, parseErr := metadataFromXML(document) if parseErr != nil { return operation.Resolution{}, operationCleanupInspection{} } result.Status = mapDomainState(state) result.ReinstallCommitted = true result.CleanupPending = false encoded, _ := json.Marshal(result) return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{} } if !replacementExists && metadata.DiskTransitionID != record.OperationID { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "reinstall_not_committed", ErrorMessage: "host evidence confirms the reinstall did not switch the root volume"}, operationCleanupInspection{} } return operation.Resolution{}, operationCleanupInspection{} } func (l *Libvirt) inspectSnapshotOperation(record operation.Record, domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) { parts := strings.Split(record.Action, ":") if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) { return operation.Resolution{}, operationCleanupInspection{} } key := parts[1] document, metadata, err := requireManagedDomain(domain) if err != nil { return operation.Resolution{}, operationCleanupInspection{} } var stored *domainSnapshotMetadataXML for index := range metadata.Snapshots { if metadata.Snapshots[index].Key == key { stored = &metadata.Snapshots[index] break } } volume, err := l.lookupOptionalManagedVolume( l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, key), ) if err != nil { return operation.Resolution{}, operationCleanupInspection{} } volumeExists := volume != nil if volume != nil { defer volume.Free() } switch parts[0] { case "create-snapshot": if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") { return operation.Resolution{}, operationCleanupInspection{} } if stored != nil && volumeExists && snapshotConsistencyMatchesRequest(parts[2], stored.Consistency) { info, validateErr := validateQCOW2Volume(volume, stored.SizeBytes) if validateErr != nil || info.Capacity != stored.SizeBytes { return operation.Resolution{}, operationCleanupInspection{} } result := Snapshot{ID: key, SnapshotKey: key, Name: stored.Name, Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(), State: "ready", Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, Created: true} encoded, _ := json.Marshal(result) return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{} } if stored == nil && !volumeExists { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_created", ErrorMessage: "host evidence confirms the snapshot is absent"}, operationCleanupInspection{} } if stored == nil && volumeExists { return operation.Resolution{}, l.snapshotVolumeCleanupInspection( domain, volume, snapshotVolumeName(record.ResourceID, key), CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key), ) } if stored != nil && !volumeExists { return operation.Resolution{}, cleanupInspectionWithIdleDomain( domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key, ) } case "delete-snapshot": if len(parts) != 2 { return operation.Resolution{}, operationCleanupInspection{} } if stored == nil && !volumeExists { result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true}) return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{} } if stored != nil && volumeExists { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_deleted", ErrorMessage: "host evidence confirms the snapshot remains present"}, operationCleanupInspection{} } if stored != nil && !volumeExists { return operation.Resolution{}, cleanupInspectionWithIdleDomain( domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key, ) } if stored == nil && volumeExists { return operation.Resolution{}, l.snapshotVolumeCleanupInspection( domain, volume, snapshotVolumeName(record.ResourceID, key), CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key), ) } case "restore-snapshot": if len(parts) != 2 || stored == nil || !volumeExists { return operation.Resolution{}, operationCleanupInspection{} } replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID) replacement, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName) if lookupErr != nil { return operation.Resolution{}, operationCleanupInspection{} } replacementExists := replacement != nil if replacement != nil { replacement.Free() } if diskTransitionCommitted(metadata, replacementName, "snapshot_restore", record.OperationID) && replacementExists { state, _, stateErr := domain.GetState() if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "snapshot_restore", record.OperationID) { return operation.Resolution{}, operationCleanupInspection{} } if metadata.PendingCleanupVolume != "" { return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata) } result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Restored: true, PowerState: mapDomainState(state), ActiveVolumeName: replacementName, CleanupPending: false}) return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{} } if !replacementExists && metadata.DiskTransitionID != record.OperationID { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_restored", ErrorMessage: "host evidence confirms the root volume was not switched"}, operationCleanupInspection{} } } return operation.Resolution{}, operationCleanupInspection{} } func diskTransitionPowerStateMatches(metadata *domainMetadataXML, state libvirt.DomainState, kind, operationID string) bool { if !diskTransitionMarkerMatches(metadata, kind, operationID) { return true } if metadata.DiskTransitionRunning { return state == libvirt.DOMAIN_RUNNING || state == libvirt.DOMAIN_BLOCKED } return state == libvirt.DOMAIN_SHUTOFF || state == libvirt.DOMAIN_SHUTDOWN } func (l *Libvirt) inspectDiskTransitionCleanupSteps(domain *libvirt.Domain, document string, metadata *domainMetadataXML) operationCleanupInspection { pending := metadata.PendingCleanupVolume if pending == "" || !isVMOwnedRootVolumeName(metadataUUID(document), pending) || pending == metadata.RootVolumeName { return operationCleanupInspection{} } jobEvidence, safe := inspectCleanupJobEvidence(domain) if !safe { return operationCleanupInspection{} } steps := make([]string, 0, 2) evidence := append([]string(nil), jobEvidence...) evidence = append(evidence, "pending-root:"+pending, "active-root:"+metadata.RootVolumeName) old, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), pending) if err != nil { return operationCleanupInspection{} } if old != nil { if _, validateErr := validateQCOW2Volume(old, 0); validateErr != nil { old.Free() return operationCleanupInspection{} } oldDigest, digestErr := storageVolumeInspectionDigest(old) if digestErr != nil { old.Free() return operationCleanupInspection{} } path, pathErr := old.GetPath() old.Free() if pathErr != nil { return operationCleanupInspection{} } used, referenceErr := l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), pending, ) if referenceErr != nil || used { return operationCleanupInspection{} } steps = append(steps, CleanupStepDeletePreviousRoot) evidence = append(evidence, "pending-root-state:present:"+oldDigest) } else { evidence = append(evidence, "pending-root-state:absent") } steps = append(steps, CleanupStepPersistCleanupMetadata) return newOperationCleanupInspection(steps, evidence...) } type cleanupJobDomain interface { IsActive() (bool, error) GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) GetXMLDesc(libvirt.DomainXMLFlags) (string, error) GetBlockJobInfo(string, libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error) } func inspectCleanupJobEvidence(domain cleanupJobDomain) ([]string, bool) { active, err := domain.IsActive() if err != nil { return nil, false } flags := libvirt.DOMAIN_XML_INACTIVE evidence := []string{"domain-runtime:inactive"} if active { flags = 0 job, jobErr := domain.GetJobStats(0) if jobErr != nil || job == nil || job.Type != libvirt.DOMAIN_JOB_NONE { return nil, false } evidence[0] = "domain-runtime:active:job-none" } document, err := domain.GetXMLDesc(flags) if err != nil { return nil, false } documentDigest := sha256.Sum256([]byte(document)) evidence = append(evidence, "domain-xml:"+hex.EncodeToString(documentDigest[:])) if active { persistentDocument, persistentErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if persistentErr != nil { return nil, false } persistentDigest := sha256.Sum256([]byte(persistentDocument)) evidence = append(evidence, "persistent-domain-xml:"+hex.EncodeToString(persistentDigest[:])) } targets, err := cleanupDiskTargets(document) if err != nil { return nil, false } for _, target := range targets { evidence = append(evidence, "disk-target:"+target) if !active { continue } info, infoErr := domain.GetBlockJobInfo(target, 0) if infoErr != nil || !blockJobInfoAbsent(info) { return nil, false } } return evidence, true } func cleanupDiskTargets(document string) ([]string, error) { var definition libvirtxml.Domain if err := definition.Unmarshal(document); err != nil || definition.Devices == nil { return nil, errors.New("domain disk evidence is invalid") } seen := make(map[string]struct{}, len(definition.Devices.Disks)) for _, disk := range definition.Devices.Disks { if disk.Target == nil || disk.Target.Dev == "" { return nil, errors.New("domain disk target evidence is incomplete") } seen[disk.Target.Dev] = struct{}{} } targets := make([]string, 0, len(seen)) for target := range seen { targets = append(targets, target) } sort.Strings(targets) return targets, nil } func blockJobInfoAbsent(info *libvirt.DomainBlockJobInfo) bool { // The Go binding does not expose virDomainGetBlockJobInfo's 0/1 return // value. A 0 (no job) is represented by the zero-valued info struct. return info != nil && info.Type == libvirt.DOMAIN_BLOCK_JOB_TYPE_UNKNOWN && info.Bandwidth == 0 && info.Cur == 0 && info.End == 0 } func cleanupInspectionWithIdleDomain(domain cleanupJobDomain, steps []string, evidence ...string) operationCleanupInspection { jobEvidence, safe := inspectCleanupJobEvidence(domain) if !safe { return operationCleanupInspection{} } evidence = append(evidence, jobEvidence...) return newOperationCleanupInspection(steps, evidence...) } func ensureNoCleanupBlockingJobs(domain cleanupJobDomain) error { if _, safe := inspectCleanupJobEvidence(domain); !safe { return NewError(ErrConflict, "cleanup_job_not_idle", "domain cleanup is blocked by active or unreadable job state", nil) } return nil } func (l *Libvirt) snapshotVolumeCleanupInspection(domain cleanupJobDomain, volume *libvirt.StorageVol, volumeName, step, targetEvidence string) operationCleanupInspection { if volume == nil { return operationCleanupInspection{} } jobEvidence, safe := inspectCleanupJobEvidence(domain) if !safe { return operationCleanupInspection{} } if _, err := validateQCOW2Volume(volume, 0); err != nil { return operationCleanupInspection{} } volumeDigest, err := storageVolumeInspectionDigest(volume) if err != nil { return operationCleanupInspection{} } path, err := volume.GetPath() if err != nil { return operationCleanupInspection{} } referenced, err := l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), volumeName, ) if err != nil || referenced { return operationCleanupInspection{} } evidence := append([]string{targetEvidence + ":" + volumeDigest}, jobEvidence...) return newOperationCleanupInspection([]string{step}, evidence...) } func (l *Libvirt) inspectMatchingBackupJob(record operation.Record, domain *libvirt.Domain) OperationAbortInspection { if record.State != operation.StateDispatching && (record.State != operation.StateFailed || !record.EffectMayHaveApplied) { return OperationAbortInspection{} } parts := strings.Split(record.Action, ":") if len(parts) != 3 || parts[0] != "create-snapshot" || !catalogIDPattern.MatchString(parts[1]) { return OperationAbortInspection{} } job, err := domain.GetJobStats(0) if err != nil || job == nil || (job.Type != libvirt.DOMAIN_JOB_BOUNDED && job.Type != libvirt.DOMAIN_JOB_UNBOUNDED) || !job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP { return OperationAbortInspection{} } backupXML, err := domain.BackupGetXMLDesc(0) if err != nil { return OperationAbortInspection{} } volume, err := l.lookupOptionalManagedVolume( l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, parts[1]), ) if err != nil || volume == nil { return OperationAbortInspection{} } if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil { volume.Free() return OperationAbortInspection{} } targetPath, pathErr := volume.GetPath() volume.Free() if pathErr != nil || !backupPushMatchesTarget(backupXML, targetPath) { return OperationAbortInspection{} } return OperationAbortInspection{ Available: true, Kind: "backup", InspectionToken: operationInspectionToken(record, "backup", backupXML), } } func backupPushMatchesTarget(backupXML, targetPath string) bool { var backup libvirtxml.DomainBackup if backup.Unmarshal(backupXML) != nil || backup.Pull != nil || backup.Push == nil || backup.Push.Disks == nil || len(backup.Push.Disks.Disks) != 1 { return false } disk := backup.Push.Disks.Disks[0] return disk.Name == "vda" && disk.Backup == "yes" && disk.Target != nil && disk.Target.File != nil && filepath.Clean(disk.Target.File.File) == filepath.Clean(targetPath) } func buildOperationInspection(record operation.Record, resolution operation.Resolution, abort OperationAbortInspection, cleanup operationCleanupInspection) OperationInspection { result := OperationInspection{ OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID, RequestFingerprint: record.Fingerprint, OperationStatus: record.State, CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain, SummaryCode: InspectionSummaryIncomplete, EffectResolution: InspectionEffectInconclusive, AbortJob: abort, Cleanup: OperationCleanupPlan{}, } if abort.Available { result.SummaryCode = InspectionSummaryAbortableJob return result } if len(cleanup.steps) != 0 && len(cleanup.evidence) != 0 { result.SummaryCode = InspectionSummaryCleanupRequired result.EffectResolution = InspectionEffectPartial result.Cleanup = OperationCleanupPlan{ Available: true, PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()), Steps: append([]string(nil), cleanup.steps...), } return result } if !resolution.Resolved { return result } if resolution.Succeeded { result.Outcome = InspectionOutcomeSucceeded result.SummaryCode = InspectionSummarySucceeded projection := operationEffectProjection(record, resolution.Result) result.EffectResolution = inspectedEffectResolution(record, resolution, projection) if result.EffectResolution == InspectionEffectApplied { result.EffectProjection = projection } else { result.Outcome = InspectionOutcomeUncertain result.SummaryCode = InspectionSummaryIncomplete } } else { result.Outcome = InspectionOutcomeFailed result.SummaryCode = InspectionSummaryFailed projection := failedOperationEffectProjection(record, resolution) result.EffectResolution = inspectedEffectResolution(record, resolution, projection) if result.EffectResolution == InspectionEffectNotApplied { result.EffectProjection = projection } else { result.Outcome = InspectionOutcomeUncertain result.SummaryCode = InspectionSummaryIncomplete } if inspectionErrorCodePattern.MatchString(resolution.ErrorCode) { result.ErrorCode = resolution.ErrorCode } } return result } // inspectedEffectResolution separates immutable operation evidence from a // read-only observation that merely happens to match the requested target. // Power, service, network-link, catalog and storage state may be changed out // of band, so those observations remain inconclusive even when the target is // currently reached. Only a Journal terminal record or a mutation-specific // durable marker may authorize automated ledger convergence. func inspectedEffectResolution(record operation.Record, resolution operation.Resolution, projection json.RawMessage) string { if record.State == operation.StateSucceeded { return InspectionEffectApplied } if record.State == operation.StateFailed && !record.EffectMayHaveApplied { return InspectionEffectNotApplied } if record.State != operation.StateFailed || !record.EffectMayHaveApplied || !resolution.Resolved { return InspectionEffectInconclusive } if resolution.Succeeded { if len(projection) == 0 { return InspectionEffectInconclusive } switch { case record.Action == "update-settings", record.Action == "change-network", record.Action == "reinstall-vm", record.Action == "apply-firewall", strings.HasPrefix(record.Action, "restore-snapshot:"): return InspectionEffectApplied default: return InspectionEffectInconclusive } } switch { case record.Action == "create-vm" && resolution.ErrorCode == "creation_absent" && len(projection) != 0, record.Action == "update-settings" && resolution.ErrorCode == "settings_not_committed", record.Action == "change-network" && resolution.ErrorCode == "network_not_committed", record.Action == "reinstall-vm" && resolution.ErrorCode == "reinstall_not_committed", record.Action == "apply-firewall" && resolution.ErrorCode == "firewall_not_committed", strings.HasPrefix(record.Action, "restore-snapshot:") && resolution.ErrorCode == "snapshot_not_restored": return InspectionEffectNotApplied default: return InspectionEffectInconclusive } } func inspectionFromRecord(record operation.Record) OperationInspection { switch record.State { case operation.StateSucceeded: return buildOperationInspection(record, operation.Resolution{ Resolved: true, Succeeded: true, Result: append([]byte(nil), record.Result...), }, OperationAbortInspection{}, operationCleanupInspection{}) case operation.StateFailed: resolution := operation.Resolution{Resolved: true, Succeeded: false, Result: append([]byte(nil), record.Result...)} if record.Error != nil { resolution.ErrorCode = record.Error.Code resolution.ErrorMessage = record.Error.Message } return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}) } return inconclusiveInspectionFromRecord(record) } func inconclusiveInspectionFromRecord(record operation.Record) OperationInspection { return OperationInspection{ OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID, RequestFingerprint: record.Fingerprint, OperationStatus: record.State, CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain, SummaryCode: InspectionSummaryIncomplete, EffectResolution: InspectionEffectInconclusive, } } func failedOperationEffectProjection(record operation.Record, resolution operation.Resolution) json.RawMessage { if record.Action != "create-vm" || resolution.ErrorCode != "creation_absent" { return nil } var evidence struct { CreationAbsent bool `json:"creation_absent"` ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"` } if !decodeOperationEffectResult(resolution.Result, &evidence) || !evidence.CreationAbsent || !evidence.ManagedArtifactsAbsent { return nil } encoded, err := json.Marshal(evidence) if err != nil { return nil } return encoded } const maxOperationEffectProjectionBytes = 64 << 10 // operationEffectProjection never forwards Journal bytes. It first decodes a // result into the exact response type allowed for the recorded action and then // rebuilds a deliberately smaller projection. Unknown actions, malformed or // oversized results, and identity mismatches simply omit the optional // projection while keeping the independently derived effect resolution. func operationEffectProjection(record operation.Record, raw []byte) json.RawMessage { if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes || !json.Valid(raw) { return nil } var projection any switch { case record.Action == "configure-storage": var value StorageConfigurationResult if !decodeOperationEffectResult(raw, &value) { return nil } projection = struct { Configured bool `json:"configured"` }{Configured: value.Configured} case record.Action == "create-vm" || record.Action == "update-settings" || record.Action == "change-network" || record.Action == "reinstall-vm": var value VMMetadata if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID { return nil } projection = operationVMEffectProjection{ UUID: value.UUID, Name: value.Name, Status: value.Status, VCPUs: value.VCPUs, MemoryMiB: value.MemoryMiB, DiskGiB: value.DiskGiB, BandwidthMbps: value.BandwidthMbps, Hostname: value.Hostname, PublicIPCIDR: value.PublicIPCIDR, PublicPoolKey: value.PublicPoolKey, PublicPoolCIDR: value.PublicPoolCIDR, Gateway: value.Gateway, DNSServers: append([]string(nil), value.DNSServers...), NetworkSelector: value.NetworkSelector, TemplateID: value.TemplateID, Machine: value.Machine, Firmware: value.Firmware, SecureBoot: value.SecureBoot, SettingsChanged: value.SettingsChanged, NetworkChanged: value.NetworkChanged, ReinstallCommitted: value.ReinstallCommitted, CleanupPending: value.CleanupPending, } case record.Action == "delete-vm" || record.Action == "force-delete-vm": var value DeleteVMResult if !decodeOperationEffectResult(raw, &value) { return nil } projection = value case strings.HasPrefix(record.Action, "power-") || strings.HasPrefix(record.Action, "service-"): var value VMState if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID { return nil } projection = struct { UUID string `json:"uuid"` State VMStateName `json:"state"` }{UUID: value.UUID, State: value.State} case record.Action == "network-suspend" || record.Action == "network-resume": var value struct { NetworkEnabled bool `json:"network_enabled"` } if !decodeOperationEffectResult(raw, &value) { return nil } projection = value case strings.HasPrefix(record.Action, "create-snapshot:"): var value Snapshot if !decodeOperationEffectResult(raw, &value) { return nil } projection = operationSnapshotEffectProjection{ ID: value.ID, SnapshotKey: value.SnapshotKey, Name: value.Name, CreatedAt: value.CreatedAt, State: value.State, Consistency: value.Consistency, SizeBytes: value.SizeBytes, Created: value.Created, } case strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:"): var value SnapshotMutationResult if !decodeOperationEffectResult(raw, &value) { return nil } projection = struct { SnapshotKey string `json:"snapshot_key"` Restored bool `json:"restored,omitempty"` Deleted bool `json:"deleted,omitempty"` PowerState VMStateName `json:"power_state,omitempty"` CleanupPending bool `json:"cleanup_pending,omitempty"` }{value.SnapshotKey, value.Restored, value.Deleted, value.PowerState, value.CleanupPending} case record.Action == "apply-firewall": var value FirewallPolicy if !decodeOperationEffectResult(raw, &value) { return nil } projection = struct { Revision uint64 `json:"revision"` DefaultIn string `json:"default_in"` DefaultOut string `json:"default_out"` PolicyDigest string `json:"policy_digest,omitempty"` Rules []FirewallRule `json:"rules"` }{value.Revision, value.DefaultIn, value.DefaultOut, value.PolicyDigest, append([]FirewallRule(nil), value.Rules...)} case record.Action == "enter-rescue" || record.Action == "force-start-rescue" || strings.HasPrefix(record.Action, "exit-rescue:"): var value RescueState if !decodeOperationEffectResult(raw, &value) { return nil } projection = struct { Active bool `json:"active"` Staged bool `json:"staged"` PendingManualRestart bool `json:"pending_manual_restart,omitempty"` TemplateID string `json:"template_id,omitempty"` NetworkSuspended bool `json:"network_suspended"` }{value.Active, value.Staged, value.PendingManualRestart, value.TemplateID, value.NetworkSuspended} case record.Action == "import-template" || record.Action == "update-template": var value ImageTemplate if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "template:"+value.ID { return nil } projection = operationCatalogImportProjection{ ID: value.ID, Name: value.Name, OSFamily: value.OSFamily, Version: value.Version, Architecture: value.Architecture, SizeBytes: value.SizeBytes, VirtualSizeBytes: value.VirtualSizeBytes, SHA256: value.SHA256, Revision: value.Revision, GenerationToken: value.GenerationToken, CreatedAt: value.CreatedAt, } case record.Action == "import-rescue-template": var value RescueTemplate if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "rescue-template:"+value.ID { return nil } projection = operationCatalogImportProjection{ ID: value.ID, Name: value.Name, SizeBytes: value.SizeBytes, SHA256: value.SHA256, CreatedAt: value.CreatedAt, } case record.Action == "import-user-iso": var value UserISO if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "user-iso:"+strconv.FormatInt(value.OwnerID, 10)+":"+value.ID { return nil } projection = operationCatalogImportProjection{ ID: value.ID, OwnerID: value.OwnerID, Name: value.Name, SizeBytes: value.SizeBytes, SHA256: value.SHA256, CreatedAt: value.CreatedAt, } case record.Action == "delete-template" || record.Action == "delete-rescue-template" || record.Action == "delete-user-iso": var value CatalogDeleteResult if !decodeOperationEffectResult(raw, &value) { return nil } projection = value default: return nil } encoded, err := json.Marshal(projection) if err != nil || len(encoded) == 0 || len(encoded) > maxOperationEffectProjectionBytes { return nil } return encoded } type operationVMEffectProjection struct { UUID string `json:"uuid"` Name string `json:"name,omitempty"` Status VMStateName `json:"status,omitempty"` VCPUs uint `json:"vcpu,omitempty"` MemoryMiB uint64 `json:"memory_mib,omitempty"` DiskGiB uint64 `json:"disk_gib,omitempty"` BandwidthMbps uint64 `json:"bandwidth_mbps,omitempty"` Hostname string `json:"hostname,omitempty"` PublicIPCIDR string `json:"public_ip_cidr,omitempty"` PublicPoolKey string `json:"public_pool_key,omitempty"` PublicPoolCIDR string `json:"public_pool_cidr,omitempty"` Gateway string `json:"gateway,omitempty"` DNSServers []string `json:"dns_servers,omitempty"` NetworkSelector string `json:"network,omitempty"` TemplateID string `json:"template_id,omitempty"` Machine string `json:"machine,omitempty"` Firmware string `json:"firmware,omitempty"` SecureBoot bool `json:"secure_boot,omitempty"` SettingsChanged bool `json:"settings_changed,omitempty"` NetworkChanged bool `json:"network_changed,omitempty"` ReinstallCommitted bool `json:"reinstall_committed,omitempty"` CleanupPending bool `json:"cleanup_pending,omitempty"` } type operationSnapshotEffectProjection struct { ID string `json:"id"` SnapshotKey string `json:"snapshot_key"` Name string `json:"name,omitempty"` CreatedAt time.Time `json:"created_at"` State string `json:"state"` Consistency string `json:"consistency"` SizeBytes uint64 `json:"size_bytes,omitempty"` Created bool `json:"created,omitempty"` } type operationCatalogImportProjection struct { ID string `json:"id"` OwnerID int64 `json:"owner_id,omitempty"` Name string `json:"name,omitempty"` OSFamily string `json:"os_family,omitempty"` Version string `json:"version,omitempty"` Architecture string `json:"architecture,omitempty"` SizeBytes uint64 `json:"size_bytes,omitempty"` VirtualSizeBytes uint64 `json:"virtual_size_bytes,omitempty"` SHA256 string `json:"sha256,omitempty"` Revision uint64 `json:"revision,omitempty"` GenerationToken string `json:"generation_token,omitempty"` CreatedAt time.Time `json:"created_at"` } func decodeOperationEffectResult(raw []byte, target any) bool { if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes { return false } trimmed := strings.TrimSpace(string(raw)) return len(trimmed) >= 2 && trimmed[0] == '{' && trimmed[len(trimmed)-1] == '}' && json.Unmarshal(raw, target) == nil } func recordHasCleanupPending(record operation.Record) bool { if len(record.Result) == 0 { return false } var result struct { CleanupPending bool `json:"cleanup_pending"` } return json.Unmarshal(record.Result, &result) == nil && result.CleanupPending } func operationInspectionToken(record operation.Record, kind, evidence string) string { digest := sha256.New() for _, part := range []string{ "virt-cloud-operation-inspection-v1", record.OperationID, record.ResourceID, record.Action, record.Fingerprint, kind, evidence, } { digest.Write([]byte(part)) digest.Write([]byte{0}) } return hex.EncodeToString(digest.Sum(nil)) } func lowerHexDigest(value string) bool { if len(value) != sha256.Size*2 { return false } for _, current := range value { if (current < '0' || current > '9') && (current < 'a' || current > 'f') { return false } } return true }