//go:build libvirt package hypervisor import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "time" "git.luoli.live/root/virt/agent/operation" "git.luoli.live/root/virt/agent/peermigration" "git.luoli.live/root/virt/agent/publicnetwork" "git.luoli.live/root/virt/agent/vpcrouting" "libvirt.org/go/libvirt" "libvirt.org/go/libvirtxml" ) var inspectionErrorCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`) // operationCleanupInspection carries the exact, private host evidence used to // bind a cleanup plan. Only the stable step codes are serialized; resource // names, XML digests, and file attributes stay inside the Agent and feed the // one-way plan identifier. type operationCleanupInspection struct { steps []string evidence []string } func newOperationCleanupInspection(steps []string, evidence ...string) operationCleanupInspection { if len(evidence) == 0 { return operationCleanupInspection{} } normalizedEvidence := append([]string(nil), evidence...) sort.Strings(normalizedEvidence) return operationCleanupInspection{ steps: append([]string(nil), steps...), evidence: normalizedEvidence, } } func (inspection operationCleanupInspection) tokenEvidence() string { encoded, _ := json.Marshal(struct { Steps []string `json:"steps"` Evidence []string `json:"evidence"` }{Steps: inspection.steps, Evidence: inspection.evidence}) return string(encoded) } // InspectOperation deliberately bypasses Journal admission. A dispatched // libvirt call retains its resource lease until the real call returns, but an // administrator must still be able to read its state and identify an owned // cancellable backup job while that call is stuck. Every operation below is a // libvirt/filesystem read. It may return a cleanup plan, but inspection never // executes that plan or changes a host resource. func (l *Libvirt) InspectOperation(ctx context.Context, record operation.Record) (OperationInspection, error) { if err := ctx.Err(); err != nil { return OperationInspection{}, err } if successfulBackupCleanupPending(record) { if record.InspectionEvidenceUnavailable { return inconclusiveInspectionFromRecord(record), nil } return l.inspectSuccessfulBackupCleanupPending(ctx, record), nil } if storedOperationInspectionAvailable(record) { if record.InspectionEvidenceUnavailable { return inconclusiveInspectionFromRecord(record), nil } inspection := inspectionFromRecord(record) // A recovered successful receipt is useful for convergence only when its // action-specific result can be reduced to the bounded public projection. // Receipt status alone must not turn an unprojectable or damaged result // into an "applied" verdict after Journal detail retention. if record.InspectionReceiptRecovered && record.State == operation.StateSucceeded && len(inspection.EffectProjection) == 0 { return inconclusiveInspectionFromRecord(record), nil } return inspection, nil } if resolution, handled, err := l.reconcileStorageConfiguration(record); handled { return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err } if resolution, handled, err := l.reconcileCatalogOperation(ctx, record); handled { return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err } if resolution, handled, err := l.reconcileVPCNetwork(record); handled { return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err } domain, err := l.domainByUUID(ctx, record.ResourceID) if errors.Is(err, ErrNotFound) { return l.inspectAbsentDomainOperation(ctx, record) } if err != nil { return OperationInspection{}, err } defer domain.Free() abort := l.inspectMatchingBackupJob(record, domain) resolution, cleanup, inspectErr := l.inspectExistingDomainOperation(ctx, record, domain) if abort.Available { // No cleanup may be approved while the matching backup is still live. cleanup = operationCleanupInspection{} } return buildOperationInspection(record, resolution, abort, cleanup), inspectErr } func storedOperationInspectionAvailable(record operation.Record) bool { if record.Tombstone { return true } if record.State == operation.StateSucceeded && strings.HasPrefix(record.Action, "create-backup:") { // A successful backup result is immutable proof that the remote object // committed, even when its optional cleanup_pending marker reports only // local residue. Never discard the sole object SHA/size projection by // replacing it with a weaker current-host observation. return true } if record.State == operation.StateSucceeded && !recordHasCleanupPending(record) { return true } if record.State == operation.StateFailed && record.EffectMayHaveApplied { // A failed Journal row describes the response observed by the original // caller, not necessarily the durable host effect. Reuse would make the // operation look definitively failed forever and bypass the read-only // expected-state reconcilers below. return false } return record.State == operation.StateFailed && !operation.IsCreateLikeAction(record.Action) && !operation.RequiresExplicitForceDelete(record) } // inspectSuccessfulBackupCleanupPending preserves the immutable applied // object projection while adding an optional, read-only cleanup plan for an // exact operation-owned local workspace residue. Host read failures never // downgrade the already verified remote object identity. func (l *Libvirt) inspectSuccessfulBackupCleanupPending(ctx context.Context, record operation.Record) OperationInspection { inspection := inspectionFromRecord(record) domain, err := l.domainByUUID(ctx, record.ResourceID) if errors.Is(err, ErrNotFound) { parts := strings.Split(record.Action, ":") if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) { return inspection } volumeName, volume, _, residueErr := l.backupWorkspaceResidue( record.ResourceID, parts[1], ) if residueErr != nil || volume == nil { return inspection } defer volume.Free() cleanup := l.absentDomainBackupWorkspaceCleanupInspection( volume, volumeName, ) return appliedBackupInspectionWithCleanup( record, inspection, cleanup, ) } if err != nil { return inspection } defer domain.Free() _, cleanup := l.inspectSnapshotOperation(ctx, record, domain) return appliedBackupInspectionWithCleanup(record, inspection, cleanup) } func appliedBackupInspectionWithCleanup(record operation.Record, inspection OperationInspection, cleanup operationCleanupInspection) OperationInspection { if len(cleanup.steps) == 0 || len(cleanup.evidence) == 0 { return inspection } inspection.SummaryCode = InspectionSummaryCleanupRequired inspection.Cleanup = OperationCleanupPlan{ Available: true, PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()), Steps: append([]string(nil), cleanup.steps...), } return inspection } func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record operation.Record) (OperationInspection, error) { var resolution operation.Resolution var cleanup operationCleanupInspection switch { case strings.HasPrefix(record.Action, "receive-peer-migration:"): plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil) if err != nil { return OperationInspection{}, err } if len(plan.steps) != 0 { cleanup = plan break } resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, true, false) case operation.IsCreateLikeAction(record.Action): absent, err := l.deleteArtifactsAbsent(record.ResourceID) if err != nil { return OperationInspection{}, err } resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, absent, l.traffic.contains(record.ResourceID)) case record.Action == "delete-vm" || record.Action == "force-delete-vm" || strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"): plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil) if err != nil { return OperationInspection{}, err } cleanup = plan resolution = inspectAbsentDeleteResolution(plan, record.Action == "force-delete-vm") case strings.HasPrefix(record.Action, "cleanup-backup-workspace:"): resolution = l.reconcileAbsentDomainBackupWorkspaceCleanup(record) default: if record.ExpectedState != nil { resolution = expectedStateResourceAbsent(record) } } return buildOperationInspection(record, resolution, OperationAbortInspection{}, cleanup), nil } func (l *Libvirt) reconcileAbsentDomainBackupWorkspaceCleanup( record operation.Record, ) operation.Resolution { parts := strings.Split(record.Action, ":") if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) { return operation.Resolution{} } _, volume, _, err := l.backupWorkspaceResidue( record.ResourceID, parts[1], ) if err != nil { return operation.Resolution{} } if volume == nil { result, _ := json.Marshal(CleanupBackupWorkspaceResult{ BackupKey: parts[1], Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true, }) return operation.Resolution{ Resolved: true, Succeeded: true, Result: result, } } defer volume.Free() if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil { return operation.Resolution{} } return operation.Resolution{ Resolved: true, Succeeded: false, ErrorCode: "backup_workspace_cleanup_not_applied", ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied", } } func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePresent bool) (operation.Resolution, operationCleanupInspection) { if !managedArtifactsAbsent { return operation.Resolution{}, operationCleanupInspection{} } resolution := operation.Resolution{ Resolved: true, Succeeded: false, Result: []byte(`{"creation_absent":true,"managed_artifacts_absent":true}`), ErrorCode: "creation_absent", ErrorMessage: "host evidence confirms the VM and managed artifacts are absent", } if trafficStatePresent { return resolution, newOperationCleanupInspection( []string{CleanupStepRemoveTrafficState}, "traffic-state:"+uuid, ) } return resolution, operationCleanupInspection{} } func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection, error) { if strings.HasPrefix(record.Action, "cleanup-peer-migration-target:") { // This successor is authorized only to clean an absent target. A domain // appearing before or after dispatch is never converted into a delete // plan by reconciliation. return operation.Resolution{}, operationCleanupInspection{}, nil } if record.Action == "delete-vm" || record.Action == "force-delete-vm" { _, managed, err := requireManagedDomain(domain) if err != nil { return operation.Resolution{}, operationCleanupInspection{}, err } if err := l.validateDeleteVMArtifacts(ctx, record.ResourceID, managed); err != nil { return operation.Resolution{}, operationCleanupInspection{}, err } active, err := domain.IsActive() if err != nil { return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err) } if resolution, handled := inspectActiveDeleteResolution(record, active); handled { return resolution, operationCleanupInspection{}, nil } plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, domain, managed) return operation.Resolution{}, plan, err } if record.ExpectedState != nil { resolution, handled, err := l.inspectExpectedStateOperation(ctx, record, domain) if handled { return resolution, operationCleanupInspection{}, err } } if strings.HasPrefix(record.Action, "prepare-peer-migration:") { resolution, err := l.reconcilePreparedPeerMigration(record, domain) return resolution, operationCleanupInspection{}, err } if strings.HasPrefix(record.Action, "receive-peer-migration:") { resolution, err := l.reconcileReceivedPeerMigration(ctx, record, domain) return resolution, operationCleanupInspection{}, err } if record.Action == "reinstall-vm" { resolution, steps := l.inspectReinstallOperation(record, domain) return resolution, steps, nil } if strings.HasPrefix(record.Action, "create-snapshot:") || strings.HasPrefix(record.Action, "create-backup:") || strings.HasPrefix(record.Action, "cleanup-backup-workspace:") || strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:") || strings.HasPrefix(record.Action, "recover-backup:") { resolution, steps := l.inspectSnapshotOperation(ctx, record, domain) return resolution, steps, nil } if record.Action == "enter-rescue" || record.Action == "force-start-rescue" || strings.HasPrefix(record.Action, "exit-rescue:") { return l.reconcileRescue(ctx, record, domain), operationCleanupInspection{}, nil } state, _, err := domain.GetState() if err != nil { return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect operation domain state", err) } mapped := mapDomainState(state) switch { case operation.IsCreateLikeAction(record.Action): resolution, err := l.reconcileCreatedVM(ctx, record, domain) return resolution, operationCleanupInspection{}, err case record.Action == "power-start": if mapped == VMStateRunning || mapped == VMStateBlocked { return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil } case record.Action == "power-shutdown" || record.Action == "power-destroy": if mapped == VMStateShutoff { return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil } } return operation.Resolution{}, operationCleanupInspection{}, nil } func peerMigrationActionID(action, prefix string) (string, bool) { parts := strings.Split(action, ":") returnValue := "" if len(parts) == 2 && parts[0] == prefix && catalogIDPattern.MatchString(parts[1]) { returnValue = parts[1] } return returnValue, returnValue != "" } func (l *Libvirt) reconcilePreparedPeerMigration(record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) { migrationID, ok := peerMigrationActionID(record.Action, "prepare-peer-migration") if !ok { return operation.Resolution{}, nil } store, err := peermigration.NewStore( l.local.PeerMigrationStatePath, l.local.StoragePathRoots, l.local.PeerMigrationStagingPath, ) if err != nil { return operation.Resolution{}, err } manifest, manifestErr := store.LoadSource(migrationID) if manifestErr == nil { autostart, autoErr := domain.GetAutostart() active, activeErr := domain.IsActive() if autoErr != nil || activeErr != nil { return operation.Resolution{}, errors.Join(autoErr, activeErr) } if autostart || active || !strings.EqualFold(manifest.SourceUUID, record.ResourceID) { return operation.Resolution{}, nil } digest, err := manifest.Public().Digest() if err != nil { return operation.Resolution{}, err } result, err := json.Marshal(PreparePeerMigrationResult{ MigrationID: migrationID, ManifestDigest: digest, Artifacts: manifest.Public().Artifacts, SourceStopped: true, SourceWasRunning: manifest.SourceWasRunning, SourceAutostartEnabled: manifest.SourceAutostartEnabled, SourceAutostartDisabled: true, }) if err != nil { return operation.Resolution{}, err } return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, nil } if !errors.Is(manifestErr, os.ErrNotExist) { return operation.Resolution{}, nil } intent, intentErr := store.LoadSourceIntent(migrationID) if errors.Is(intentErr, os.ErrNotExist) { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "peer_migration_prepare_not_applied", ErrorMessage: "source intent was not persisted before the mutation boundary"}, nil } if intentErr != nil || !strings.EqualFold(intent.SourceUUID, record.ResourceID) { return operation.Resolution{}, nil } autostart, autoErr := domain.GetAutostart() active, activeErr := domain.IsActive() if autoErr != nil || activeErr != nil { return operation.Resolution{}, errors.Join(autoErr, activeErr) } if autostart == intent.SourceAutostartEnabled && active == intent.SourceWasRunning { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "peer_migration_prepare_not_applied", ErrorMessage: "source state is restored and no ready manifest exists"}, nil } return operation.Resolution{}, nil } func (l *Libvirt) reconcileReceivedPeerMigration(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) { migrationID, ok := peerMigrationActionID(record.Action, "receive-peer-migration") if !ok { return operation.Resolution{}, nil } store, err := peermigration.NewStore( l.local.PeerMigrationStatePath, l.local.StoragePathRoots, l.local.PeerMigrationStagingPath, ) if err != nil { return operation.Resolution{}, err } receipt, err := store.LoadTargetReceipt(migrationID) if err != nil || !strings.EqualFold(receipt.TargetUUID, record.ResourceID) { return operation.Resolution{}, nil } autostart, autoErr := domain.GetAutostart() active, activeErr := domain.IsActive() if autoErr != nil || activeErr != nil || autostart || active { return operation.Resolution{}, errors.Join(autoErr, activeErr) } vm, complete, err := l.reconcileStagedPeerMigrationVM(ctx, record, domain, receipt) if err != nil || !complete { return operation.Resolution{}, err } result, err := json.Marshal(ReceivePeerMigrationResult{ VMMetadata: vm, MigrationID: migrationID, ManifestDigest: receipt.ManifestDigest, Artifacts: receipt.Artifacts, StagedPoweredOff: true, }) if err != nil { return operation.Resolution{}, err } return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, nil } func (l *Libvirt) reconcileStagedPeerMigrationVM(ctx context.Context, record operation.Record, domain *libvirt.Domain, receipt peermigration.TargetReceipt) (VMMetadata, bool, error) { document, definition, metadata, pool, root, err := l.managedRootContext(domain) if err != nil { return VMMetadata{}, false, nil } defer pool.Free() defer root.Free() autostart, autostartErr := domain.GetAutostart() active, activeErr := domain.IsActive() state, _, stateErr := domain.GetState() if autostartErr != nil || activeErr != nil || stateErr != nil { return VMMetadata{}, false, errors.Join(autostartErr, activeErr, stateErr) } if autostart || active || state != libvirt.DOMAIN_SHUTOFF { return VMMetadata{}, false, nil } rootInfo, err := validateStandaloneQCOW2Volume(root, 0) if err != nil { return VMMetadata{}, false, nil } var rootArtifact *peermigration.Artifact snapshotArtifacts := make(map[string]peermigration.Artifact) for index := range receipt.Artifacts { artifact := &receipt.Artifacts[index] if artifact.Role == "root" { rootArtifact = artifact continue } snapshotArtifacts[artifact.SnapshotKey] = *artifact } if rootArtifact == nil || rootInfo.Capacity < rootArtifact.VirtualSizeBytes || len(metadata.Snapshots) != len(snapshotArtifacts) { return VMMetadata{}, false, nil } for _, stored := range metadata.Snapshots { artifact, ok := snapshotArtifacts[stored.Key] if !ok || stored.Name != artifact.Name || stored.Consistency != artifact.Consistency || stored.SizeBytes != artifact.VirtualSizeBytes || stored.CreatedAt != artifact.CreatedAt { return VMMetadata{}, false, nil } volume, lookupErr := l.lookupOptionalManagedVolume( l.storagePoolName(StorageRoleManaged), snapshotVolumeName(definition.UUID, stored.Key), ) if lookupErr != nil { return VMMetadata{}, false, lookupErr } if volume == nil { return VMMetadata{}, false, nil } _, inspectErr := validateStandaloneQCOW2Volume(volume, artifact.VirtualSizeBytes) volume.Free() if inspectErr != nil { return VMMetadata{}, false, nil } } seed, err := l.lookupOptionalManagedVolume( l.storagePoolName(StorageRoleSeed), seedVolumeName(definition.UUID), ) if err != nil || seed == nil { return VMMetadata{}, false, err } seedPath, seedErr := seed.GetPath() seed.Free() if seedErr != nil || definition.Devices == nil { return VMMetadata{}, false, seedErr } seedAttached := false filterName := networkFilterName(definition.UUID) filterAttached := managedNetworkLayoutMatches(definition, metadata) for _, disk := range definition.Devices.Disks { if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sda" && disk.Source != nil && disk.Source.File != nil && disk.Source.File.File == seedPath && disk.ReadOnly != nil { seedAttached = true } } var policy PlatformNetworkPolicySnapshot if !seedAttached || !filterAttached || json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &policy) != nil || ValidatePlatformNetworkPolicy(policy) != nil || l.verifyManagedFilter(filterName, policy) != nil { return VMMetadata{}, false, nil } if record.ExpectedState != nil && record.ExpectedState.Kind == operation.ExpectedCreateLikeNetwork { seedContents, seedContentsErr := l.managedSeedContents(ctx, record.ResourceID) if seedContentsErr != nil { _, evidenceErr := expectedEvidenceError("create-like network", seedContentsErr) return VMMetadata{}, false, evidenceErr } if !createLikeNetworkEvidenceMatches( record, definition, metadata, seedContents, ) { return VMMetadata{}, false, nil } } vm, err := metadataFromXML(document) if err != nil { return VMMetadata{}, false, nil } vm.Status = VMStateShutoff return vm, true, nil } func inspectActiveDeleteResolution(record operation.Record, active bool) (operation.Resolution, bool) { if !active { return operation.Resolution{}, false } if record.Action == "delete-vm" && operation.RequiresExplicitForceDelete(record) { // Preserve an authoritative force_required outcome written by the // original graceful DeleteVM call after its bounded wait. return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "force_required", ErrorMessage: "guest remains active and requires an explicitly authorized force delete"}, true } if record.Action == "delete-vm" && (record.CauseCode == "graceful_shutdown_ambiguous" || recoveredGracefulAttemptRequiresRetry(record)) { // The prior ShutdownFlags response was lost and activity could not be // observed conclusively, or the Agent restarted at the dispatch // boundary before it persisted any cause. A later active observation // can classify the already-terminal operation as failed, but cannot grant // force authorization or dispatch another mutation. return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "graceful_shutdown_unconfirmed", ErrorMessage: "graceful shutdown result was not confirmed; the resource remains blocked for manual remediation"}, true } if record.Action == "force-delete-vm" && forceDeleteCanBeConfirmedForRetry(record) { // Never replay DestroyFlags automatically. Once no native call can still // hold Journal admission, a token-confirmed administrator inspection may // close an old force attempt which still observes an active Domain. The // original graceful force_required record remains the authorization // anchor, so a retry still needs a new operation ID and data-loss ack. return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "force_delete_unconfirmed_active", ErrorMessage: "force delete did not reach an inactive domain; inspect and explicitly authorize a new force operation"}, true } // A force operation may already have crossed Destroy before its result was // lost. Read-only inspection must not turn that ambiguity into a cleanup // plan which would replay Destroy. return operation.Resolution{}, true } func forceDeleteCanBeConfirmedForRetry(record operation.Record) bool { return record.State == operation.StateFailed && record.EffectMayHaveApplied } func recoveredGracefulAttemptRequiresRetry(record operation.Record) bool { if record.CauseCode != "" { return false } return record.State == operation.StateFailed && record.EffectMayHaveApplied } func inspectAbsentDeleteResolution(cleanup operationCleanupInspection, forced bool) operation.Resolution { if len(cleanup.steps) != 0 { return operation.Resolution{} } return resolvedDeleteSuccess(DeleteVMResult{ Deleted: true, ManagedArtifactsAbsent: true, Forced: forced, }) } func (l *Libvirt) inspectDeleteCleanupSteps(ctx context.Context, uuid string, domain *libvirt.Domain, managed *domainMetadataXML) (operationCleanupInspection, error) { steps := make([]string, 0, 7) evidence := make([]string, 0, 8) if domain != nil { jobEvidence, safe := inspectCleanupJobEvidence(domain) if !safe { return operationCleanupInspection{}, nil } evidence = append(evidence, jobEvidence...) deleteEvidence, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain) if err != nil { return operationCleanupInspection{}, err } evidence = append(evidence, deleteEvidence...) active, err := domain.IsActive() if err != nil { return operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err) } if active { return operationCleanupInspection{}, nil } evidence = append(evidence, "domain:"+uuid+":inactive") steps = append(steps, CleanupStepUndefineDomain) } pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged)) if err != nil { return operationCleanupInspection{}, translateLibvirtError("inspect managed storage pool", err) } inventory, err := pool.ListStorageVolumes() if err != nil { pool.Free() return operationCleanupInspection{}, translateLibvirtError("inspect managed volumes", err) } names, err := collectVMManagedVolumeNames(uuid, managed, inventory) if err != nil { pool.Free() return operationCleanupInspection{}, err } managedVolumesExist := false for _, name := range names { if err := ctx.Err(); err != nil { pool.Free() return operationCleanupInspection{}, err } volume, lookupErr := pool.LookupStorageVolByName(name) if isNoStorageVolume(lookupErr) { continue } if lookupErr != nil { pool.Free() return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", lookupErr) } managedVolumesExist = true if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil { volume.Free() pool.Free() return operationCleanupInspection{}, validateErr } volumeDigest, digestErr := storageVolumeInspectionDigest(volume) if digestErr != nil { volume.Free() pool.Free() return operationCleanupInspection{}, digestErr } path, pathErr := volume.GetPath() volume.Free() if pathErr != nil { pool.Free() return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", pathErr) } var referenced bool if domain != nil { referenced, err = l.otherDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), name, uuid, ) } else { referenced, err = l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), name, ) } if err != nil { pool.Free() return operationCleanupInspection{}, err } if referenced { pool.Free() return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed cleanup volume is referenced by another domain", nil) } evidence = append(evidence, "managed-volume:"+name+":"+volumeDigest) } pool.Free() if managedVolumesExist { steps = append(steps, CleanupStepDeleteManagedVolumes) } seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid)) if err != nil { return operationCleanupInspection{}, err } if seed != nil { seedDigest, digestErr := storageVolumeInspectionDigest(seed) if digestErr != nil { seed.Free() return operationCleanupInspection{}, digestErr } path, pathErr := seed.GetPath() seed.Free() if pathErr != nil { return operationCleanupInspection{}, translateLibvirtError("inspect managed seed cleanup volume", pathErr) } referenced, referenceErr := l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), ) if domain != nil { referenced, referenceErr = l.otherDomainUsesVolume( path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid, ) } if referenceErr != nil { return operationCleanupInspection{}, referenceErr } if referenced { return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed seed cleanup volume is referenced by another domain", nil) } steps = append(steps, CleanupStepDeleteSeedVolume) evidence = append(evidence, "seed-volume:"+seedVolumeName(uuid)+":"+seedDigest) } filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid)) if err != nil { return operationCleanupInspection{}, err } if filter != nil { document, xmlErr := filter.GetXMLDesc(0) filter.Free() if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) { return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_invalid", "network filter cannot be confirmed as panel managed", xmlErr) } excludedUUID := "" if domain != nil { excludedUUID = uuid } referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), excludedUUID) if referenceErr != nil { return operationCleanupInspection{}, referenceErr } if referenced { return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil) } steps = append(steps, CleanupStepUndefineNetworkFilter) digest := sha256.Sum256([]byte(document)) evidence = append(evidence, "network-filter:"+networkFilterName(uuid)+":"+hex.EncodeToString(digest[:])) } nvramExists := false for _, name := range managedNVRAMArtifactNames(uuid) { path := filepath.Join(l.local.NVRAMDirectory, name) info, statErr := os.Lstat(path) if errors.Is(statErr, os.ErrNotExist) { continue } if statErr != nil { return operationCleanupInspection{}, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr) } if !info.Mode().IsRegular() { return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil) } referenced, referenceErr := l.anyDomainUsesPath(path) if domain != nil { referenced, referenceErr = l.otherDomainUsesPath(path, uuid) } if referenceErr != nil { return operationCleanupInspection{}, referenceErr } if referenced { return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil) } nvramExists = true evidence = append(evidence, "nvram:"+name+":"+strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10)) } if nvramExists { steps = append(steps, CleanupStepDeleteNVRAM) } trafficPresent := l.traffic.contains(uuid) if trafficPresent || len(steps) != 0 { // cleanupDeletedVMArtifacts durably rewrites traffic state even if an // earlier failed fsync left only an on-disk entry. Make that mutation // explicit in every delete cleanup plan. steps = append(steps, CleanupStepRemoveTrafficState) if trafficPresent { evidence = append(evidence, "traffic-state:"+uuid+":present") } else { evidence = append(evidence, "traffic-state:"+uuid+":rewrite") } } return newOperationCleanupInspection(steps, evidence...), nil } func managedFilterDocument(document, expectedName string) bool { var filter libvirtxml.NWFilter if filter.Unmarshal(document) != nil || filter.Name != expectedName || filter.Chain != "root" { return false } return managedFilterBaselineValid(&filter) } func storageVolumeInspectionDigest(volume *libvirt.StorageVol) (string, error) { document, err := volume.GetXMLDesc(0) if err != nil { return "", translateLibvirtError("read cleanup volume XML", err) } digest := sha256.Sum256([]byte(document)) return hex.EncodeToString(digest[:]), nil } func (l *Libvirt) inspectReinstallOperation(record operation.Record, domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) { document, metadata, err := requireManagedDomain(domain) if err != nil { return operation.Resolution{}, operationCleanupInspection{} } replacementName := reinstallVolumeName(record.ResourceID, record.OperationID) replacement, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName) if err != nil { return operation.Resolution{}, operationCleanupInspection{} } replacementExists := replacement != nil if replacement != nil { replacement.Free() } if diskTransitionCommitted(metadata, replacementName, "reinstall", record.OperationID) && replacementExists { state, _, stateErr := domain.GetState() if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "reinstall", record.OperationID) { return operation.Resolution{}, operationCleanupInspection{} } if metadata.PendingCleanupVolume != "" { return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata) } result, parseErr := metadataFromXML(document) if parseErr != nil { return operation.Resolution{}, operationCleanupInspection{} } result.Status = mapDomainState(state) result.ReinstallCommitted = true result.CleanupPending = false encoded, _ := json.Marshal(result) return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{} } if !replacementExists && metadata.DiskTransitionID != record.OperationID { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "reinstall_not_committed", ErrorMessage: "host evidence confirms the reinstall did not switch the root volume"}, operationCleanupInspection{} } return operation.Resolution{}, operationCleanupInspection{} } func (l *Libvirt) inspectSnapshotOperation(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) { parts := strings.Split(record.Action, ":") if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) { return operation.Resolution{}, operationCleanupInspection{} } key := parts[1] document, metadata, err := requireManagedDomain(domain) if err != nil { return operation.Resolution{}, operationCleanupInspection{} } var stored *domainSnapshotMetadataXML for index := range metadata.Snapshots { if metadata.Snapshots[index].Key == key { stored = &metadata.Snapshots[index] break } } volumeName := snapshotVolumeName(record.ResourceID, key) var volume *libvirt.StorageVol legacyBackupVolume := false if parts[0] == "create-backup" || parts[0] == "cleanup-backup-workspace" { volumeName, volume, legacyBackupVolume, err = l.backupWorkspaceResidue( record.ResourceID, key, ) } else { volume, err = l.lookupOptionalManagedVolume( l.storagePoolName(StorageRoleManaged), volumeName, ) } if err != nil { return operation.Resolution{}, operationCleanupInspection{} } volumeExists := volume != nil if volume != nil { defer volume.Free() } switch parts[0] { case "create-backup": if stored != nil && volume != nil && volumeName != snapshotVolumeName(record.ResourceID, key) { return operation.Resolution{}, operationCleanupInspection{} } if !backupResidueCleanupAllowed(record, stored) { return operation.Resolution{}, operationCleanupInspection{} } return operation.Resolution{}, l.backupResidueCleanupInspection( domain, volume, volumeName, key, stored, ) case "cleanup-backup-workspace": if len(parts) != 2 { return operation.Resolution{}, operationCleanupInspection{} } if stored != nil && volume != nil && volumeName != snapshotVolumeName(record.ResourceID, key) { return operation.Resolution{}, operationCleanupInspection{} } if stored == nil && !volumeExists { result, _ := json.Marshal(CleanupBackupWorkspaceResult{ BackupKey: key, Cleaned: true, WorkspaceAbsent: true, MetadataAbsent: true, }) return operation.Resolution{ Resolved: true, Succeeded: true, Result: result, }, operationCleanupInspection{} } if (stored == nil && volumeExists && !legacyBackupVolume) || (stored != nil && volumeExists && legacyBackupVolume) { return operation.Resolution{ Resolved: true, Succeeded: false, ErrorCode: "backup_workspace_cleanup_not_applied", ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied", }, operationCleanupInspection{} } return operation.Resolution{}, operationCleanupInspection{} case "create-snapshot": if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") { return operation.Resolution{}, operationCleanupInspection{} } if stored != nil && volumeExists && snapshotConsistencyMatchesRequest(parts[2], stored.Consistency) { info, validateErr := validateQCOW2Volume(volume, stored.SizeBytes) if validateErr != nil || info.Capacity != stored.SizeBytes { return operation.Resolution{}, operationCleanupInspection{} } result := Snapshot{ID: key, SnapshotKey: key, Name: stored.Name, Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(), State: "ready", Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, Created: true} encoded, _ := json.Marshal(result) return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{} } if stored == nil && !volumeExists { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_created", ErrorMessage: "host evidence confirms the snapshot is absent"}, operationCleanupInspection{} } if stored == nil && volumeExists { return operation.Resolution{}, l.snapshotVolumeCleanupInspection( domain, volume, snapshotVolumeName(record.ResourceID, key), CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key), ) } if stored != nil && !volumeExists { return operation.Resolution{}, cleanupInspectionWithIdleDomain( domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key, ) } case "delete-snapshot": if len(parts) != 2 { return operation.Resolution{}, operationCleanupInspection{} } if stored == nil && !volumeExists { result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true}) return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{} } if stored != nil && volumeExists { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_deleted", ErrorMessage: "host evidence confirms the snapshot remains present"}, operationCleanupInspection{} } if stored != nil && !volumeExists { return operation.Resolution{}, cleanupInspectionWithIdleDomain( domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key, ) } if stored == nil && volumeExists { return operation.Resolution{}, l.snapshotVolumeCleanupInspection( domain, volume, snapshotVolumeName(record.ResourceID, key), CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key), ) } case "restore-snapshot": if len(parts) != 2 || stored == nil || !volumeExists { return operation.Resolution{}, operationCleanupInspection{} } replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID) replacement, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName) if lookupErr != nil { return operation.Resolution{}, operationCleanupInspection{} } replacementExists := replacement != nil if replacement != nil { replacement.Free() } if diskTransitionCommitted(metadata, replacementName, "snapshot_restore", record.OperationID) && replacementExists { state, _, stateErr := domain.GetState() if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "snapshot_restore", record.OperationID) { return operation.Resolution{}, operationCleanupInspection{} } if metadata.PendingCleanupVolume != "" { return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata) } result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Restored: true, PowerState: mapDomainState(state), ActiveVolumeName: replacementName, CleanupPending: false}) return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{} } if !replacementExists && metadata.DiskTransitionID != record.OperationID { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "snapshot_not_restored", ErrorMessage: "host evidence confirms the root volume was not switched"}, operationCleanupInspection{} } case "recover-backup": if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) || !sha256Pattern.MatchString(parts[2]) { return operation.Resolution{}, operationCleanupInspection{} } key, sourceSHA256 := parts[1], parts[2] replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID) replacement, lookupErr := l.lookupOptionalManagedVolume( l.storagePoolName(StorageRoleManaged), replacementName) if lookupErr != nil { return operation.Resolution{}, operationCleanupInspection{} } replacementExists := replacement != nil if replacement != nil { replacement.Free() } seedContents, seedErr := l.managedSeedContents(ctx, record.ResourceID) if seedErr != nil { return operation.Resolution{}, operationCleanupInspection{} } seedOperation, seedErr := seedMetadataMarker( seedContents, backupRecoveryOperationMetadata, ) if seedErr != nil { return operation.Resolution{}, operationCleanupInspection{} } seedCommitted := seedOperation == record.OperationID diskCommitted := diskTransitionCommitted(metadata, replacementName, "backup_recover", record.OperationID) evidence := classifyBackupRecoveryEvidence( replacementExists, diskCommitted, metadata.DiskTransitionID == record.OperationID, seedCommitted, ) if evidence == backupRecoveryEvidenceApplied { state, _, stateErr := domain.GetState() if stateErr != nil || !diskTransitionPowerStateMatches( metadata, state, "backup_recover", record.OperationID) { return operation.Resolution{}, operationCleanupInspection{} } if metadata.PendingCleanupVolume != "" { return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata) } result, _ := json.Marshal(RecoverBackupResult{ BackupKey: key, SHA256: sourceSHA256, Restored: true, PowerState: mapDomainState(state), ActiveVolumeName: replacementName, }) return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{} } if evidence == backupRecoveryEvidenceNotApplied { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "backup_not_recovered", ErrorMessage: "host evidence confirms the root volume was not switched"}, operationCleanupInspection{} } } return operation.Resolution{}, operationCleanupInspection{} } type backupRecoveryEvidence uint8 const ( backupRecoveryEvidenceUnknown backupRecoveryEvidence = iota backupRecoveryEvidenceApplied backupRecoveryEvidenceNotApplied ) // classifyBackupRecoveryEvidence is deliberately fail-closed. Recovery is // applied only when both the root-disk transition and the seed transition are // visible. Any partial evidence remains unresolved and fenced. func classifyBackupRecoveryEvidence(replacementExists, diskCommitted, diskTransitionClaimed, seedCommitted bool) backupRecoveryEvidence { if replacementExists && diskCommitted && seedCommitted { return backupRecoveryEvidenceApplied } if !replacementExists && !diskTransitionClaimed && !seedCommitted { return backupRecoveryEvidenceNotApplied } return backupRecoveryEvidenceUnknown } func backupResidueCleanupAllowed(record operation.Record, stored *domainSnapshotMetadataXML) bool { parts := strings.Split(record.Action, ":") if len(parts) != 3 || parts[0] != "create-backup" || !catalogIDPattern.MatchString(parts[1]) || (parts[2] != "filesystem" && parts[2] != "crash") { return false } if record.State == operation.StateSucceeded { if !successfulBackupCleanupPending(record) { return false } } else if record.State != operation.StateFailed || !record.EffectMayHaveApplied || record.ExecutionFenced { return false } return stored == nil || (stored.Key == parts[1] && stored.SizeBytes > 0 && snapshotConsistencyMatchesRequest(parts[2], stored.Consistency)) } func successfulBackupCleanupPending(record operation.Record) bool { if record.State != operation.StateSucceeded || !recordHasCleanupPending(record) || len(operationEffectProjection(record, record.Result)) == 0 { return false } return strings.HasPrefix(record.Action, "create-backup:") } // backupResidueCleanupInspection exposes only an evidence-bound cleanup plan // for a local immutable copy left by an already-terminal backup worker. It // never asserts whether the remote multipart object committed: failed records // therefore remain partial, while successful records retain their immutable // applied object projection through appliedBackupInspectionWithCleanup. func (l *Libvirt) backupResidueCleanupInspection(domain cleanupJobDomain, volume *libvirt.StorageVol, volumeName, key string, stored *domainSnapshotMetadataXML) operationCleanupInspection { if volume == nil && stored == nil { return operationCleanupInspection{} } if volume != nil { expectedCapacity := uint64(0) if stored != nil { expectedCapacity = stored.SizeBytes } if _, err := validateStandaloneQCOW2Volume(volume, expectedCapacity); err != nil { return operationCleanupInspection{} } plan := l.snapshotVolumeCleanupInspection( domain, volume, volumeName, CleanupStepDeleteSnapshotVolume, "backup-volume:"+volumeName, ) if len(plan.steps) == 0 { return operationCleanupInspection{} } if stored != nil { plan.steps = append(plan.steps, CleanupStepRemoveSnapshotMetadata) plan.evidence = append(plan.evidence, "backup-metadata:"+key+":"+ stored.Consistency+":"+strconv.FormatUint(stored.SizeBytes, 10)) sort.Strings(plan.evidence) } return plan } return cleanupInspectionWithIdleDomain( domain, []string{CleanupStepRemoveSnapshotMetadata}, "backup-metadata:"+key+":"+stored.Consistency+":"+ strconv.FormatUint(stored.SizeBytes, 10), ) } func (l *Libvirt) absentDomainBackupWorkspaceCleanupInspection( volume *libvirt.StorageVol, volumeName string, ) operationCleanupInspection { if volume == nil { return operationCleanupInspection{} } if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil { return operationCleanupInspection{} } digest, err := storageVolumeInspectionDigest(volume) if err != nil { return operationCleanupInspection{} } path, err := volume.GetPath() if err != nil { return operationCleanupInspection{} } referenced, err := l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), volumeName, ) if err != nil || referenced { return operationCleanupInspection{} } return newOperationCleanupInspection( []string{CleanupStepDeleteSnapshotVolume}, "domain-state:absent", "backup-volume:"+volumeName+":"+digest, ) } func diskTransitionPowerStateMatches(metadata *domainMetadataXML, state libvirt.DomainState, kind, operationID string) bool { if !diskTransitionMarkerMatches(metadata, kind, operationID) { return true } if metadata.DiskTransitionRunning { return state == libvirt.DOMAIN_RUNNING || state == libvirt.DOMAIN_BLOCKED } return state == libvirt.DOMAIN_SHUTOFF || state == libvirt.DOMAIN_SHUTDOWN } func (l *Libvirt) inspectDiskTransitionCleanupSteps(domain *libvirt.Domain, document string, metadata *domainMetadataXML) operationCleanupInspection { pending := metadata.PendingCleanupVolume if pending == "" || !isVMOwnedRootVolumeName(metadataUUID(document), pending) || pending == metadata.RootVolumeName { return operationCleanupInspection{} } jobEvidence, safe := inspectCleanupJobEvidence(domain) if !safe { return operationCleanupInspection{} } steps := make([]string, 0, 2) evidence := append([]string(nil), jobEvidence...) evidence = append(evidence, "pending-root:"+pending, "active-root:"+metadata.RootVolumeName) old, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), pending) if err != nil { return operationCleanupInspection{} } if old != nil { if _, validateErr := validateQCOW2Volume(old, 0); validateErr != nil { old.Free() return operationCleanupInspection{} } oldDigest, digestErr := storageVolumeInspectionDigest(old) if digestErr != nil { old.Free() return operationCleanupInspection{} } path, pathErr := old.GetPath() old.Free() if pathErr != nil { return operationCleanupInspection{} } used, referenceErr := l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), pending, ) if referenceErr != nil || used { return operationCleanupInspection{} } steps = append(steps, CleanupStepDeletePreviousRoot) evidence = append(evidence, "pending-root-state:present:"+oldDigest) } else { evidence = append(evidence, "pending-root-state:absent") } steps = append(steps, CleanupStepPersistCleanupMetadata) return newOperationCleanupInspection(steps, evidence...) } type cleanupJobDomain interface { IsActive() (bool, error) GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) GetXMLDesc(libvirt.DomainXMLFlags) (string, error) GetBlockJobInfo(string, libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error) } func inspectCleanupJobEvidence(domain cleanupJobDomain) ([]string, bool) { active, err := domain.IsActive() if err != nil { return nil, false } flags := libvirt.DOMAIN_XML_INACTIVE evidence := []string{"domain-runtime:inactive"} if active { flags = 0 job, jobErr := domain.GetJobStats(0) if jobErr != nil || job == nil || job.Type != libvirt.DOMAIN_JOB_NONE { return nil, false } evidence[0] = "domain-runtime:active:job-none" } document, err := domain.GetXMLDesc(flags) if err != nil { return nil, false } documentDigest := sha256.Sum256([]byte(document)) evidence = append(evidence, "domain-xml:"+hex.EncodeToString(documentDigest[:])) if active { persistentDocument, persistentErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE) if persistentErr != nil { return nil, false } persistentDigest := sha256.Sum256([]byte(persistentDocument)) evidence = append(evidence, "persistent-domain-xml:"+hex.EncodeToString(persistentDigest[:])) } targets, err := cleanupDiskTargets(document) if err != nil { return nil, false } for _, target := range targets { evidence = append(evidence, "disk-target:"+target) if !active { continue } info, infoErr := domain.GetBlockJobInfo(target, 0) if infoErr != nil || !blockJobInfoAbsent(info) { return nil, false } } return evidence, true } func cleanupDiskTargets(document string) ([]string, error) { var definition libvirtxml.Domain if err := definition.Unmarshal(document); err != nil || definition.Devices == nil { return nil, errors.New("domain disk evidence is invalid") } seen := make(map[string]struct{}, len(definition.Devices.Disks)) for _, disk := range definition.Devices.Disks { if disk.Target == nil || disk.Target.Dev == "" { return nil, errors.New("domain disk target evidence is incomplete") } seen[disk.Target.Dev] = struct{}{} } targets := make([]string, 0, len(seen)) for target := range seen { targets = append(targets, target) } sort.Strings(targets) return targets, nil } func blockJobInfoAbsent(info *libvirt.DomainBlockJobInfo) bool { // The Go binding does not expose virDomainGetBlockJobInfo's 0/1 return // value. A 0 (no job) is represented by the zero-valued info struct. return info != nil && info.Type == libvirt.DOMAIN_BLOCK_JOB_TYPE_UNKNOWN && info.Bandwidth == 0 && info.Cur == 0 && info.End == 0 } func cleanupInspectionWithIdleDomain(domain cleanupJobDomain, steps []string, evidence ...string) operationCleanupInspection { jobEvidence, safe := inspectCleanupJobEvidence(domain) if !safe { return operationCleanupInspection{} } evidence = append(evidence, jobEvidence...) return newOperationCleanupInspection(steps, evidence...) } func ensureNoCleanupBlockingJobs(domain cleanupJobDomain) error { if _, safe := inspectCleanupJobEvidence(domain); !safe { return NewError(ErrConflict, "cleanup_job_not_idle", "domain cleanup is blocked by active or unreadable job state", nil) } return nil } func (l *Libvirt) snapshotVolumeCleanupInspection(domain cleanupJobDomain, volume *libvirt.StorageVol, volumeName, step, targetEvidence string) operationCleanupInspection { if volume == nil { return operationCleanupInspection{} } jobEvidence, safe := inspectCleanupJobEvidence(domain) if !safe { return operationCleanupInspection{} } if _, err := validateQCOW2Volume(volume, 0); err != nil { return operationCleanupInspection{} } volumeDigest, err := storageVolumeInspectionDigest(volume) if err != nil { return operationCleanupInspection{} } path, err := volume.GetPath() if err != nil { return operationCleanupInspection{} } referenced, err := l.anyDomainUsesVolume( path, l.storagePoolName(StorageRoleManaged), volumeName, ) if err != nil || referenced { return operationCleanupInspection{} } evidence := append([]string{targetEvidence + ":" + volumeDigest}, jobEvidence...) return newOperationCleanupInspection([]string{step}, evidence...) } func (l *Libvirt) inspectMatchingBackupJob(record operation.Record, domain *libvirt.Domain) OperationAbortInspection { if record.State != operation.StateDispatching && (record.State != operation.StateFailed || !record.EffectMayHaveApplied) { return OperationAbortInspection{} } parts := strings.Split(record.Action, ":") if len(parts) != 3 || (parts[0] != "create-snapshot" && parts[0] != "create-backup") || !catalogIDPattern.MatchString(parts[1]) || (parts[2] != "filesystem" && parts[2] != "crash") { return OperationAbortInspection{} } job, err := domain.GetJobStats(0) if err != nil || job == nil || (job.Type != libvirt.DOMAIN_JOB_BOUNDED && job.Type != libvirt.DOMAIN_JOB_UNBOUNDED) || !job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP { return OperationAbortInspection{} } backupXML, err := domain.BackupGetXMLDesc(0) if err != nil { return OperationAbortInspection{} } _, volume, _, err := l.backupWorkspaceResidue(record.ResourceID, parts[1]) if err != nil || volume == nil { return OperationAbortInspection{} } if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil { volume.Free() return OperationAbortInspection{} } targetPath, pathErr := volume.GetPath() volume.Free() if pathErr != nil || !backupPushMatchesTarget(backupXML, targetPath) { return OperationAbortInspection{} } return OperationAbortInspection{ Available: true, Kind: "backup", InspectionToken: operationInspectionToken(record, "backup", backupXML), } } func backupPushMatchesTarget(backupXML, targetPath string) bool { var backup libvirtxml.DomainBackup if backup.Unmarshal(backupXML) != nil || backup.Pull != nil || backup.Push == nil || backup.Push.Disks == nil || len(backup.Push.Disks.Disks) != 1 { return false } disk := backup.Push.Disks.Disks[0] return disk.Name == "vda" && disk.Backup == "yes" && disk.Target != nil && disk.Target.File != nil && filepath.Clean(disk.Target.File.File) == filepath.Clean(targetPath) } func buildOperationInspection(record operation.Record, resolution operation.Resolution, abort OperationAbortInspection, cleanup operationCleanupInspection) OperationInspection { result := OperationInspection{ OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID, RequestFingerprint: record.Fingerprint, OperationStatus: record.State, CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain, SummaryCode: InspectionSummaryIncomplete, EffectResolution: InspectionEffectInconclusive, AbortJob: abort, Cleanup: OperationCleanupPlan{}, } if abort.Available { result.SummaryCode = InspectionSummaryAbortableJob return result } if len(cleanup.steps) != 0 && len(cleanup.evidence) != 0 { result.SummaryCode = InspectionSummaryCleanupRequired result.EffectResolution = InspectionEffectPartial result.Cleanup = OperationCleanupPlan{ Available: true, PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()), Steps: append([]string(nil), cleanup.steps...), } return result } if !resolution.Resolved { return result } if record.Action == "resize-vm" && !resolution.Succeeded && resolution.ErrorCode == "resources_partially_resized" { result.ErrorCode = resolution.ErrorCode result.EffectResolution = InspectionEffectPartial return result } if resolution.Succeeded { result.Outcome = InspectionOutcomeSucceeded result.SummaryCode = InspectionSummarySucceeded projection := operationEffectProjection(record, resolution.Result) result.EffectResolution = inspectedEffectResolution(record, resolution, projection) if result.EffectResolution == InspectionEffectApplied { result.EffectProjection = projection } else { result.Outcome = InspectionOutcomeUncertain result.SummaryCode = InspectionSummaryIncomplete } } else { result.Outcome = InspectionOutcomeFailed result.SummaryCode = InspectionSummaryFailed projection := failedOperationEffectProjection(record, resolution) result.EffectResolution = inspectedEffectResolution(record, resolution, projection) if result.EffectResolution == InspectionEffectNotApplied { result.EffectProjection = projection } else { result.Outcome = InspectionOutcomeUncertain result.SummaryCode = InspectionSummaryIncomplete } if inspectionErrorCodePattern.MatchString(resolution.ErrorCode) { result.ErrorCode = resolution.ErrorCode } } return result } // inspectedEffectResolution separates immutable operation evidence from a // read-only observation that merely happens to match the requested target. // Power, service, network-link, catalog and storage state may be changed out // of band, so those observations remain inconclusive even when the target is // currently reached. Only a Journal terminal record or a mutation-specific // durable marker may authorize automated ledger convergence. func inspectedEffectResolution(record operation.Record, resolution operation.Resolution, projection json.RawMessage) string { if record.State == operation.StateSucceeded { return InspectionEffectApplied } if record.State == operation.StateFailed && !record.EffectMayHaveApplied { return InspectionEffectNotApplied } if record.State != operation.StateFailed || !record.EffectMayHaveApplied || !resolution.Resolved { return InspectionEffectInconclusive } if resolution.Succeeded { if len(projection) == 0 { return InspectionEffectInconclusive } switch { case record.Action == "update-settings", record.Action == "resize-vm", record.Action == "change-network", record.Action == "reinstall-vm", record.Action == "apply-firewall", strings.HasPrefix(record.Action, "apply-vpc-network:"), strings.HasPrefix(record.Action, "remove-vpc-network:"), record.Action == "apply-vpc-port", record.Action == "remove-vpc-port", strings.HasPrefix(record.Action, "restore-snapshot:"), strings.HasPrefix(record.Action, "recover-backup:"): return InspectionEffectApplied default: return InspectionEffectInconclusive } } switch { case operation.IsCreateLikeAction(record.Action) && resolution.ErrorCode == "creation_absent" && len(projection) != 0, record.Action == "update-settings" && resolution.ErrorCode == "settings_not_committed", record.Action == "change-network" && resolution.ErrorCode == "network_not_committed", record.Action == "reinstall-vm" && resolution.ErrorCode == "reinstall_not_committed", record.Action == "apply-firewall" && resolution.ErrorCode == "firewall_not_committed", strings.HasPrefix(record.Action, "restore-snapshot:") && resolution.ErrorCode == "snapshot_not_restored", strings.HasPrefix(record.Action, "recover-backup:") && resolution.ErrorCode == "backup_not_recovered": return InspectionEffectNotApplied default: return InspectionEffectInconclusive } } func inspectionFromRecord(record operation.Record) OperationInspection { switch record.State { case operation.StateSucceeded: return buildOperationInspection(record, operation.Resolution{ Resolved: true, Succeeded: true, Result: append([]byte(nil), record.Result...), }, OperationAbortInspection{}, operationCleanupInspection{}) case operation.StateFailed: resolution := operation.Resolution{Resolved: true, Succeeded: false, Result: append([]byte(nil), record.Result...)} if record.Error != nil { resolution.ErrorCode = record.Error.Code resolution.ErrorMessage = record.Error.Message } return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}) } return inconclusiveInspectionFromRecord(record) } func inconclusiveInspectionFromRecord(record operation.Record) OperationInspection { return OperationInspection{ OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID, RequestFingerprint: record.Fingerprint, OperationStatus: record.State, CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain, SummaryCode: InspectionSummaryIncomplete, EffectResolution: InspectionEffectInconclusive, } } func failedOperationEffectProjection(record operation.Record, resolution operation.Resolution) json.RawMessage { if !operation.IsCreateLikeAction(record.Action) || resolution.ErrorCode != "creation_absent" { return nil } var evidence struct { CreationAbsent bool `json:"creation_absent"` ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"` } if !decodeOperationEffectResult(resolution.Result, &evidence) || !evidence.CreationAbsent || !evidence.ManagedArtifactsAbsent { return nil } encoded, err := json.Marshal(evidence) if err != nil { return nil } return encoded } const maxOperationEffectProjectionBytes = 64 << 10 // operationEffectProjection never forwards Journal bytes. It first decodes a // result into the exact response type allowed for the recorded action and then // rebuilds a deliberately smaller projection. Unknown actions, malformed or // oversized results, and identity mismatches simply omit the optional // projection while keeping the independently derived effect resolution. func operationEffectProjection(record operation.Record, raw []byte) json.RawMessage { if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes || !json.Valid(raw) { return nil } var projection any switch { case record.Action == "configure-storage": var value StorageConfigurationResult if !decodeOperationEffectResult(raw, &value) { return nil } projection = struct { Configured bool `json:"configured"` }{Configured: value.Configured} case strings.HasPrefix(record.Action, "apply-vpc-network:") || strings.HasPrefix(record.Action, "remove-vpc-network:"): var value vpcrouting.Result expected := record.ExpectedState bridge := "" if expected != nil { bridge, _ = vpcrouting.BridgeName(expected.VPCKey) } if !decodeOperationEffectResult(raw, &value) || expected == nil || expected.Kind != operation.ExpectedVPCNetwork || value.VPCKey != expected.VPCKey || value.DesiredRevision != expected.VPCRevision || value.DesiredDigest != expected.VPCDigest || (expected.VPCPresent && (!value.Applied || value.Removed || value.Bridge != bridge)) || (!expected.VPCPresent && (!value.Removed || value.Applied || value.Bridge != "")) { return nil } projection = value case record.Action == "apply-vpc-port" || record.Action == "remove-vpc-port": var value VPCPortResult expected := record.ExpectedState if !decodeOperationEffectResult(raw, &value) || expected == nil || expected.Kind != operation.ExpectedVPCPort || value.UUID != record.ResourceID || value.PortKey != expected.VPCPortKey || value.VPCKey != expected.VPCKey || (expected.VPCPresent && (!value.Attached || value.Removed || value.VPCPort == nil || *value.VPCPort != expectedVPCPortMetadata(*expected))) || (!expected.VPCPresent && (!value.Removed || value.Attached || value.VPCPort != nil)) { return nil } projection = value case strings.HasPrefix(record.Action, "create-backup:"): parts := strings.Split(record.Action, ":") var value BackupResult if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) || (parts[2] != "filesystem" && parts[2] != "crash") || !decodeOperationEffectResult(raw, &value) || value.BackupKey != parts[1] || value.SizeBytes <= 0 || value.VirtualSizeBytes == 0 || !lowerHexDigest(value.SHA256) || value.SourceContract == nil || *value.SourceContract != fixedPortableRestoreContract() || !snapshotConsistencyMatchesRequest(parts[2], value.Consistency) { return nil } projection = value case strings.HasPrefix(record.Action, "prepare-peer-migration:"): parts := strings.Split(record.Action, ":") var value PreparePeerMigrationResult if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) || !decodeOperationEffectResult(raw, &value) || value.MigrationID != parts[1] || !lowerHexDigest(value.ManifestDigest) || !value.SourceStopped || !value.SourceAutostartDisabled || len(value.Artifacts) < 1 || len(value.Artifacts) > peermigration.MaxArtifacts || !validPeerMigrationProjectionArtifacts(parts[1], record.ResourceID, value.Artifacts) { return nil } projection = value case strings.HasPrefix(record.Action, "receive-peer-migration:"): parts := strings.Split(record.Action, ":") var value ReceivePeerMigrationResult if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) || !decodeOperationEffectResult(raw, &value) || value.MigrationID != parts[1] || value.UUID != record.ResourceID || !lowerHexDigest(value.ManifestDigest) || !value.StagedPoweredOff || len(value.Artifacts) < 1 || len(value.Artifacts) > peermigration.MaxArtifacts || !validPeerMigrationProjectionArtifacts(parts[1], record.ResourceID, value.Artifacts) { return nil } projection = struct { operationVMEffectProjection MigrationID string `json:"migration_id"` ManifestDigest string `json:"manifest_digest"` Artifacts []peermigration.Artifact `json:"artifacts"` StagedPoweredOff bool `json:"staged_powered_off"` }{ operationVMEffectProjection: vmEffectProjection(value.VMMetadata), MigrationID: value.MigrationID, ManifestDigest: value.ManifestDigest, Artifacts: value.Artifacts, StagedPoweredOff: value.StagedPoweredOff, } case strings.HasPrefix(record.Action, "restore-backup:"): parts := strings.Split(record.Action, ":") var value RestoreBackupResult if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) || !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID || value.BackupKey != parts[1] || !lowerHexDigest(value.SHA256) { return nil } projection = struct { operationVMEffectProjection BackupKey string `json:"backup_key"` SHA256 string `json:"sha256"` }{ operationVMEffectProjection: vmEffectProjection(value.VMMetadata), BackupKey: value.BackupKey, SHA256: value.SHA256, } case strings.HasPrefix(record.Action, "recover-backup:"): parts := strings.Split(record.Action, ":") var value RecoverBackupResult if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) || !sha256Pattern.MatchString(parts[2]) || !decodeOperationEffectResult(raw, &value) || !value.Restored || value.BackupKey != parts[1] || value.SHA256 != parts[2] { return nil } projection = struct { BackupKey string `json:"backup_key"` SHA256 string `json:"sha256,omitempty"` Restored bool `json:"restored"` PowerState VMStateName `json:"power_state"` }{value.BackupKey, value.SHA256, value.Restored, value.PowerState} case operation.IsCreateLikeAction(record.Action) || record.Action == "update-settings" || record.Action == "change-network" || record.Action == "reinstall-vm": var value VMMetadata if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID { return nil } projection = vmEffectProjection(value) case record.Action == "resize-vm": var value ResizeResourcesResult expected := record.ExpectedState if !decodeOperationEffectResult(raw, &value) || expected == nil || expected.Kind != operation.ExpectedResizeResources || expected.TransitionID != record.OperationID || value.UUID != record.ResourceID || !value.ResourcesResized || value.FromVCPUs != expected.FromVCPUs || value.TargetVCPUs != expected.TargetVCPUs || value.FromMemoryMiB != expected.FromMemoryMiB || value.TargetMemoryMiB != expected.TargetMemoryMiB || value.FromDiskGiB != expected.FromDiskGiB || value.TargetDiskGiB != expected.TargetDiskGiB || value.FromBandwidthMbps != expected.FromBandwidthMbps || value.TargetBandwidthMbps != expected.TargetBandwidthMbps || value.AppliedVCPUs != value.TargetVCPUs || value.AppliedMemoryMiB != value.TargetMemoryMiB || value.AppliedDiskGiB != value.TargetDiskGiB || value.AppliedBandwidthMbps != value.TargetBandwidthMbps || !value.PowerStateRestored || (value.OriginallyRunning && value.Status != VMStateRunning && value.Status != VMStateBlocked) || (!value.OriginallyRunning && value.Status != VMStateShutoff && value.Status != VMStateCrashed) { return nil } projection = value case record.Action == "delete-vm" || record.Action == "force-delete-vm": var value DeleteVMResult if !decodeOperationEffectResult(raw, &value) { return nil } projection = value case strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"): parts := strings.Split(record.Action, ":") var value DeleteVMResult if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) || !decodeOperationEffectResult(raw, &value) || !value.Deleted || !value.ManagedArtifactsAbsent || value.Forced { return nil } projection = value case strings.HasPrefix(record.Action, "activate-peer-migration:") || strings.HasPrefix(record.Action, "rollback-peer-migration:"): parts := strings.Split(record.Action, ":") var value PeerMigrationStateResult expected := record.ExpectedState if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) || !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID || expected == nil || expected.Kind != operation.ExpectedPeerMigrationState || value.Running != expected.StartIfSuspended || value.AutostartEnabled != expected.Enabled { return nil } projection = struct { UUID string `json:"uuid"` State VMStateName `json:"state"` Running bool `json:"running"` AutostartEnabled bool `json:"autostart_enabled"` }{value.UUID, value.State, value.Running, value.AutostartEnabled} case strings.HasPrefix(record.Action, "power-") || strings.HasPrefix(record.Action, "service-"): var value VMState if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID { return nil } projection = struct { UUID string `json:"uuid"` State VMStateName `json:"state"` }{UUID: value.UUID, State: value.State} case record.Action == "network-suspend" || record.Action == "network-resume": var value struct { NetworkEnabled bool `json:"network_enabled"` } if !decodeOperationEffectResult(raw, &value) { return nil } projection = value case strings.HasPrefix(record.Action, "create-snapshot:"): var value Snapshot if !decodeOperationEffectResult(raw, &value) { return nil } projection = operationSnapshotEffectProjection{ ID: value.ID, SnapshotKey: value.SnapshotKey, Name: value.Name, CreatedAt: value.CreatedAt, State: value.State, Consistency: value.Consistency, SizeBytes: value.SizeBytes, Created: value.Created, } case strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:"): var value SnapshotMutationResult if !decodeOperationEffectResult(raw, &value) { return nil } projection = struct { SnapshotKey string `json:"snapshot_key"` Restored bool `json:"restored,omitempty"` Deleted bool `json:"deleted,omitempty"` PowerState VMStateName `json:"power_state,omitempty"` CleanupPending bool `json:"cleanup_pending,omitempty"` }{value.SnapshotKey, value.Restored, value.Deleted, value.PowerState, value.CleanupPending} case record.Action == "apply-firewall": var value FirewallPolicy if !decodeOperationEffectResult(raw, &value) { return nil } projection = struct { Revision uint64 `json:"revision"` DefaultIn string `json:"default_in"` DefaultOut string `json:"default_out"` PolicyDigest string `json:"policy_digest,omitempty"` Rules []FirewallRule `json:"rules"` }{value.Revision, value.DefaultIn, value.DefaultOut, value.PolicyDigest, append([]FirewallRule(nil), value.Rules...)} case record.Action == "enter-rescue" || record.Action == "force-start-rescue" || strings.HasPrefix(record.Action, "exit-rescue:"): var value RescueState if !decodeOperationEffectResult(raw, &value) { return nil } projection = struct { Active bool `json:"active"` Staged bool `json:"staged"` PendingManualRestart bool `json:"pending_manual_restart,omitempty"` TemplateID string `json:"template_id,omitempty"` Revision uint64 `json:"revision,omitempty"` GenerationToken string `json:"generation_token,omitempty"` NetworkSuspended bool `json:"network_suspended"` }{ value.Active, value.Staged, value.PendingManualRestart, value.TemplateID, value.Revision, value.GenerationToken, value.NetworkSuspended, } case record.Action == "import-template" || record.Action == "update-template": var value ImageTemplate if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "template:"+value.ID { return nil } projection = operationCatalogImportProjection{ ID: value.ID, Name: value.Name, OSFamily: value.OSFamily, Version: value.Version, Architecture: value.Architecture, SizeBytes: value.SizeBytes, VirtualSizeBytes: value.VirtualSizeBytes, SHA256: value.SHA256, Revision: value.Revision, GenerationToken: value.GenerationToken, CreatedAt: value.CreatedAt, } case record.Action == "import-rescue-template": var value RescueTemplate if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "rescue-template:"+value.ID { return nil } projection = operationCatalogImportProjection{ ID: value.ID, Name: value.Name, SizeBytes: value.SizeBytes, SHA256: value.SHA256, CreatedAt: value.CreatedAt, } case record.Action == "import-user-iso": var value UserISO if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "user-iso:"+strconv.FormatInt(value.OwnerID, 10)+":"+value.ID { return nil } projection = operationCatalogImportProjection{ ID: value.ID, OwnerID: value.OwnerID, Name: value.Name, SizeBytes: value.SizeBytes, SHA256: value.SHA256, CreatedAt: value.CreatedAt, } case record.Action == "delete-template" || record.Action == "delete-rescue-template" || record.Action == "delete-user-iso": var value CatalogDeleteResult if !decodeOperationEffectResult(raw, &value) { return nil } projection = value default: return nil } encoded, err := json.Marshal(projection) if err != nil || len(encoded) == 0 || len(encoded) > maxOperationEffectProjectionBytes { return nil } return encoded } func validPeerMigrationProjectionArtifacts(migrationID, uuid string, artifacts []peermigration.Artifact) bool { for _, artifact := range artifacts { if artifact.Path != "" { return false } } return peermigration.ValidateManifest(peermigration.Manifest{ Version: peermigration.ProtocolVersion, MigrationID: migrationID, SourceUUID: strings.ToLower(uuid), ExpiresAt: time.Now().Add(time.Hour).Unix(), Artifacts: artifacts, }, time.Now()) == nil } type operationVMEffectProjection struct { UUID string `json:"uuid"` Name string `json:"name,omitempty"` Status VMStateName `json:"status,omitempty"` VCPUs uint `json:"vcpu,omitempty"` MemoryMiB uint64 `json:"memory_mib,omitempty"` DiskGiB uint64 `json:"disk_gib,omitempty"` BandwidthMbps uint64 `json:"bandwidth_mbps,omitempty"` Hostname string `json:"hostname,omitempty"` PublicNetwork publicnetwork.Intent `json:"public_network"` TemplateID string `json:"template_id,omitempty"` Machine string `json:"machine"` Firmware string `json:"firmware"` SecureBoot bool `json:"secure_boot"` VPCPort *VPCPortMetadata `json:"vpc_port,omitempty"` SettingsChanged bool `json:"settings_changed,omitempty"` NetworkChanged bool `json:"network_changed,omitempty"` ReinstallCommitted bool `json:"reinstall_committed,omitempty"` CleanupPending bool `json:"cleanup_pending,omitempty"` } func vmEffectProjection(value VMMetadata) operationVMEffectProjection { return operationVMEffectProjection{ UUID: value.UUID, Name: value.Name, Status: value.Status, VCPUs: value.VCPUs, MemoryMiB: value.MemoryMiB, DiskGiB: value.DiskGiB, BandwidthMbps: value.BandwidthMbps, Hostname: value.Hostname, PublicNetwork: value.PublicNetwork, TemplateID: value.TemplateID, Machine: value.Machine, Firmware: value.Firmware, SecureBoot: value.SecureBoot, VPCPort: value.VPCPort, SettingsChanged: value.SettingsChanged, NetworkChanged: value.NetworkChanged, ReinstallCommitted: value.ReinstallCommitted, CleanupPending: value.CleanupPending, } } type operationSnapshotEffectProjection struct { ID string `json:"id"` SnapshotKey string `json:"snapshot_key"` Name string `json:"name,omitempty"` CreatedAt time.Time `json:"created_at"` State string `json:"state"` Consistency string `json:"consistency"` SizeBytes uint64 `json:"size_bytes,omitempty"` Created bool `json:"created,omitempty"` } type operationCatalogImportProjection struct { ID string `json:"id"` OwnerID int64 `json:"owner_id,omitempty"` Name string `json:"name,omitempty"` OSFamily string `json:"os_family,omitempty"` Version string `json:"version,omitempty"` Architecture string `json:"architecture,omitempty"` SizeBytes uint64 `json:"size_bytes,omitempty"` VirtualSizeBytes uint64 `json:"virtual_size_bytes,omitempty"` SHA256 string `json:"sha256,omitempty"` Revision uint64 `json:"revision,omitempty"` GenerationToken string `json:"generation_token,omitempty"` CreatedAt time.Time `json:"created_at"` } func decodeOperationEffectResult(raw []byte, target any) bool { if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes { return false } trimmed := strings.TrimSpace(string(raw)) return len(trimmed) >= 2 && trimmed[0] == '{' && trimmed[len(trimmed)-1] == '}' && json.Unmarshal(raw, target) == nil } func recordHasCleanupPending(record operation.Record) bool { if len(record.Result) == 0 { return false } var result struct { CleanupPending bool `json:"cleanup_pending"` } return json.Unmarshal(record.Result, &result) == nil && result.CleanupPending } func operationInspectionToken(record operation.Record, kind, evidence string) string { digest := sha256.New() for _, part := range []string{ "virt-cloud-operation-inspection-v1", record.OperationID, record.ResourceID, record.Action, record.Fingerprint, kind, evidence, } { digest.Write([]byte(part)) digest.Write([]byte{0}) } return hex.EncodeToString(digest.Sum(nil)) } func lowerHexDigest(value string) bool { if len(value) != sha256.Size*2 { return false } for _, current := range value { if (current < '0' || current > '9') && (current < 'a' || current > 'f') { return false } } return true }