You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_operation_admin.go

1479 lines
58 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
var inspectionErrorCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
// operationCleanupInspection carries the exact, private host evidence used to
// bind a cleanup plan. Only the stable step codes are serialized; resource
// names, XML digests, and file attributes stay inside the Agent and feed the
// one-way plan identifier.
type operationCleanupInspection struct {
steps []string
evidence []string
}
func newOperationCleanupInspection(steps []string, evidence ...string) operationCleanupInspection {
if len(evidence) == 0 {
return operationCleanupInspection{}
}
normalizedEvidence := append([]string(nil), evidence...)
sort.Strings(normalizedEvidence)
return operationCleanupInspection{
steps: append([]string(nil), steps...), evidence: normalizedEvidence,
}
}
func (inspection operationCleanupInspection) tokenEvidence() string {
encoded, _ := json.Marshal(struct {
Steps []string `json:"steps"`
Evidence []string `json:"evidence"`
}{Steps: inspection.steps, Evidence: inspection.evidence})
return string(encoded)
}
// InspectOperation deliberately bypasses Journal admission. A dispatched
// libvirt call retains its resource lease until the real call returns, but an
// administrator must still be able to read its state and identify an owned
// cancellable backup job while that call is stuck. Every operation below is a
// libvirt/filesystem read. It may return a cleanup plan, but inspection never
// executes that plan or changes a host resource.
func (l *Libvirt) InspectOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
if err := ctx.Err(); err != nil {
return OperationInspection{}, err
}
if successfulBackupCleanupPending(record) {
if record.InspectionEvidenceUnavailable {
return inconclusiveInspectionFromRecord(record), nil
}
return l.inspectSuccessfulBackupCleanupPending(ctx, record), nil
}
if storedOperationInspectionAvailable(record) {
if record.InspectionEvidenceUnavailable {
return inconclusiveInspectionFromRecord(record), nil
}
inspection := inspectionFromRecord(record)
// A recovered successful receipt is useful for convergence only when its
// action-specific result can be reduced to the bounded public projection.
// Receipt status alone must not turn an unprojectable or damaged result
// into an "applied" verdict after Journal detail retention.
if record.InspectionReceiptRecovered && record.State == operation.StateSucceeded &&
len(inspection.EffectProjection) == 0 {
return inconclusiveInspectionFromRecord(record), nil
}
return inspection, nil
}
if resolution, handled, err := l.reconcileStorageConfiguration(record); handled {
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
}
if resolution, handled, err := l.reconcileCatalogOperation(ctx, record); handled {
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
}
domain, err := l.domainByUUID(ctx, record.ResourceID)
if errors.Is(err, ErrNotFound) {
return l.inspectAbsentDomainOperation(ctx, record)
}
if err != nil {
return OperationInspection{}, err
}
defer domain.Free()
abort := l.inspectMatchingBackupJob(record, domain)
resolution, cleanup, inspectErr := l.inspectExistingDomainOperation(ctx, record, domain)
if abort.Available {
// No cleanup may be approved while the matching backup is still live.
cleanup = operationCleanupInspection{}
}
return buildOperationInspection(record, resolution, abort, cleanup), inspectErr
}
func storedOperationInspectionAvailable(record operation.Record) bool {
if record.Tombstone {
return true
}
if record.State == operation.StateSucceeded && strings.HasPrefix(record.Action, "create-backup:") {
// A successful backup result is immutable proof that the remote object
// committed, even when its optional cleanup_pending marker reports only
// local residue. Never discard the sole object SHA/size projection by
// replacing it with a weaker current-host observation.
return true
}
if record.State == operation.StateSucceeded && !recordHasCleanupPending(record) {
return true
}
if record.State == operation.StateFailed && record.EffectMayHaveApplied {
// A failed Journal row describes the response observed by the original
// caller, not necessarily the durable host effect. Reuse would make the
// operation look definitively failed forever and bypass the read-only
// expected-state reconcilers below.
return false
}
return record.State == operation.StateFailed && !operation.IsCreateLikeAction(record.Action) &&
!operation.RequiresExplicitForceDelete(record)
}
// inspectSuccessfulBackupCleanupPending preserves the immutable applied
// object projection while adding an optional, read-only cleanup plan for an
// exact operation-owned local snapshot residue. Host read failures never
// downgrade the already verified remote object identity.
func (l *Libvirt) inspectSuccessfulBackupCleanupPending(ctx context.Context,
record operation.Record) OperationInspection {
inspection := inspectionFromRecord(record)
domain, err := l.domainByUUID(ctx, record.ResourceID)
if err != nil {
return inspection
}
defer domain.Free()
_, cleanup := l.inspectSnapshotOperation(record, domain)
return appliedBackupInspectionWithCleanup(record, inspection, cleanup)
}
func appliedBackupInspectionWithCleanup(record operation.Record, inspection OperationInspection,
cleanup operationCleanupInspection) OperationInspection {
if len(cleanup.steps) == 0 || len(cleanup.evidence) == 0 {
return inspection
}
inspection.SummaryCode = InspectionSummaryCleanupRequired
inspection.Cleanup = OperationCleanupPlan{
Available: true,
PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()),
Steps: append([]string(nil), cleanup.steps...),
}
return inspection
}
func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
var resolution operation.Resolution
var cleanup operationCleanupInspection
switch {
case operation.IsCreateLikeAction(record.Action):
absent, err := l.deleteArtifactsAbsent(record.ResourceID)
if err != nil {
return OperationInspection{}, err
}
resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, absent, l.traffic.contains(record.ResourceID))
case record.Action == "delete-vm" || record.Action == "force-delete-vm":
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil)
if err != nil {
return OperationInspection{}, err
}
cleanup = plan
resolution = inspectAbsentDeleteResolution(plan, record.Action == "force-delete-vm")
default:
if record.ExpectedState != nil {
resolution = expectedStateResourceAbsent(record)
}
}
return buildOperationInspection(record, resolution, OperationAbortInspection{}, cleanup), nil
}
func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePresent bool) (operation.Resolution, operationCleanupInspection) {
if !managedArtifactsAbsent {
return operation.Resolution{}, operationCleanupInspection{}
}
resolution := operation.Resolution{
Resolved: true, Succeeded: false,
Result: []byte(`{"creation_absent":true,"managed_artifacts_absent":true}`),
ErrorCode: "creation_absent",
ErrorMessage: "host evidence confirms the VM and managed artifacts are absent",
}
if trafficStatePresent {
return resolution, newOperationCleanupInspection(
[]string{CleanupStepRemoveTrafficState}, "traffic-state:"+uuid,
)
}
return resolution, operationCleanupInspection{}
}
func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record operation.Record,
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection, error) {
if record.Action == "delete-vm" || record.Action == "force-delete-vm" {
_, managed, err := requireManagedDomain(domain)
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}, err
}
if err := l.validateDeleteVMArtifacts(ctx, record.ResourceID, managed); err != nil {
return operation.Resolution{}, operationCleanupInspection{}, err
}
active, err := domain.IsActive()
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
}
if resolution, handled := inspectActiveDeleteResolution(record, active); handled {
return resolution, operationCleanupInspection{}, nil
}
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, domain, managed)
return operation.Resolution{}, plan, err
}
if record.ExpectedState != nil {
switch record.ExpectedState.Kind {
case operation.ExpectedUpdateSettings, operation.ExpectedResizeResources, operation.ExpectedChangeNetwork,
operation.ExpectedNetworkState, operation.ExpectedServiceState,
operation.ExpectedFirewall:
resolution, _, err := l.inspectExpectedStateOperation(ctx, record, domain)
return resolution, operationCleanupInspection{}, err
}
}
if record.Action == "reinstall-vm" {
resolution, steps := l.inspectReinstallOperation(record, domain)
return resolution, steps, nil
}
if strings.HasPrefix(record.Action, "create-snapshot:") ||
strings.HasPrefix(record.Action, "create-backup:") ||
strings.HasPrefix(record.Action, "delete-snapshot:") ||
strings.HasPrefix(record.Action, "restore-snapshot:") {
resolution, steps := l.inspectSnapshotOperation(record, domain)
return resolution, steps, nil
}
if record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
strings.HasPrefix(record.Action, "exit-rescue:") {
return l.reconcileRescue(ctx, record, domain), operationCleanupInspection{}, nil
}
state, _, err := domain.GetState()
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect operation domain state", err)
}
mapped := mapDomainState(state)
switch {
case operation.IsCreateLikeAction(record.Action):
resolution, err := l.reconcileCreatedVM(domain)
return resolution, operationCleanupInspection{}, err
case record.Action == "power-start":
if mapped == VMStateRunning || mapped == VMStateBlocked {
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
}
case record.Action == "power-shutdown" || record.Action == "power-destroy":
if mapped == VMStateShutoff {
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
}
}
return operation.Resolution{}, operationCleanupInspection{}, nil
}
func inspectActiveDeleteResolution(record operation.Record, active bool) (operation.Resolution, bool) {
if !active {
return operation.Resolution{}, false
}
if record.Action == "delete-vm" && operation.RequiresExplicitForceDelete(record) {
// Preserve an authoritative force_required outcome written by the
// original graceful DeleteVM call after its bounded wait.
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "force_required", ErrorMessage: "guest remains active and requires an explicitly authorized force delete"}, true
}
if record.Action == "delete-vm" &&
(record.CauseCode == "graceful_shutdown_ambiguous" ||
recoveredGracefulAttemptRequiresRetry(record)) {
// The prior ShutdownFlags response was lost and activity could not be
// observed conclusively, or the Agent restarted at the dispatch
// boundary before it persisted any cause. A later active observation
// can classify the already-terminal operation as failed, but cannot grant
// force authorization or dispatch another mutation.
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "graceful_shutdown_unconfirmed",
ErrorMessage: "graceful shutdown result was not confirmed; the resource remains blocked for manual remediation"}, true
}
if record.Action == "force-delete-vm" && forceDeleteCanBeConfirmedForRetry(record) {
// Never replay DestroyFlags automatically. Once no native call can still
// hold Journal admission, a token-confirmed administrator inspection may
// close an old force attempt which still observes an active Domain. The
// original graceful force_required record remains the authorization
// anchor, so a retry still needs a new operation ID and data-loss ack.
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "force_delete_unconfirmed_active",
ErrorMessage: "force delete did not reach an inactive domain; inspect and explicitly authorize a new force operation"}, true
}
// A force operation may already have crossed Destroy before its result was
// lost. Read-only inspection must not turn that ambiguity into a cleanup
// plan which would replay Destroy.
return operation.Resolution{}, true
}
func forceDeleteCanBeConfirmedForRetry(record operation.Record) bool {
return record.State == operation.StateFailed && record.EffectMayHaveApplied
}
func recoveredGracefulAttemptRequiresRetry(record operation.Record) bool {
if record.CauseCode != "" {
return false
}
return record.State == operation.StateFailed && record.EffectMayHaveApplied
}
func inspectAbsentDeleteResolution(cleanup operationCleanupInspection, forced bool) operation.Resolution {
if len(cleanup.steps) != 0 {
return operation.Resolution{}
}
return resolvedDeleteSuccess(DeleteVMResult{
Deleted: true, ManagedArtifactsAbsent: true, Forced: forced,
})
}
func (l *Libvirt) inspectDeleteCleanupSteps(ctx context.Context, uuid string,
domain *libvirt.Domain, managed *domainMetadataXML) (operationCleanupInspection, error) {
steps := make([]string, 0, 7)
evidence := make([]string, 0, 8)
if domain != nil {
jobEvidence, safe := inspectCleanupJobEvidence(domain)
if !safe {
return operationCleanupInspection{}, nil
}
evidence = append(evidence, jobEvidence...)
deleteEvidence, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain)
if err != nil {
return operationCleanupInspection{}, err
}
evidence = append(evidence, deleteEvidence...)
active, err := domain.IsActive()
if err != nil {
return operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
}
if active {
return operationCleanupInspection{}, nil
}
evidence = append(evidence, "domain:"+uuid+":inactive")
steps = append(steps, CleanupStepUndefineDomain)
}
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
if err != nil {
return operationCleanupInspection{}, translateLibvirtError("inspect managed storage pool", err)
}
inventory, err := pool.ListStorageVolumes()
if err != nil {
pool.Free()
return operationCleanupInspection{}, translateLibvirtError("inspect managed volumes", err)
}
names, err := collectVMManagedVolumeNames(uuid, managed, inventory)
if err != nil {
pool.Free()
return operationCleanupInspection{}, err
}
managedVolumesExist := false
for _, name := range names {
if err := ctx.Err(); err != nil {
pool.Free()
return operationCleanupInspection{}, err
}
volume, lookupErr := pool.LookupStorageVolByName(name)
if isNoStorageVolume(lookupErr) {
continue
}
if lookupErr != nil {
pool.Free()
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", lookupErr)
}
managedVolumesExist = true
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
volume.Free()
pool.Free()
return operationCleanupInspection{}, validateErr
}
volumeDigest, digestErr := storageVolumeInspectionDigest(volume)
if digestErr != nil {
volume.Free()
pool.Free()
return operationCleanupInspection{}, digestErr
}
path, pathErr := volume.GetPath()
volume.Free()
if pathErr != nil {
pool.Free()
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", pathErr)
}
var referenced bool
if domain != nil {
referenced, err = l.otherDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), name, uuid,
)
} else {
referenced, err = l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), name,
)
}
if err != nil {
pool.Free()
return operationCleanupInspection{}, err
}
if referenced {
pool.Free()
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed cleanup volume is referenced by another domain", nil)
}
evidence = append(evidence, "managed-volume:"+name+":"+volumeDigest)
}
pool.Free()
if managedVolumesExist {
steps = append(steps, CleanupStepDeleteManagedVolumes)
}
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
if err != nil {
return operationCleanupInspection{}, err
}
if seed != nil {
seedDigest, digestErr := storageVolumeInspectionDigest(seed)
if digestErr != nil {
seed.Free()
return operationCleanupInspection{}, digestErr
}
path, pathErr := seed.GetPath()
seed.Free()
if pathErr != nil {
return operationCleanupInspection{}, translateLibvirtError("inspect managed seed cleanup volume", pathErr)
}
referenced, referenceErr := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
)
if domain != nil {
referenced, referenceErr = l.otherDomainUsesVolume(
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid,
)
}
if referenceErr != nil {
return operationCleanupInspection{}, referenceErr
}
if referenced {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed seed cleanup volume is referenced by another domain", nil)
}
steps = append(steps, CleanupStepDeleteSeedVolume)
evidence = append(evidence, "seed-volume:"+seedVolumeName(uuid)+":"+seedDigest)
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if err != nil {
return operationCleanupInspection{}, err
}
if filter != nil {
document, xmlErr := filter.GetXMLDesc(0)
filter.Free()
if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_invalid", "network filter cannot be confirmed as panel managed", xmlErr)
}
excludedUUID := ""
if domain != nil {
excludedUUID = uuid
}
referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), excludedUUID)
if referenceErr != nil {
return operationCleanupInspection{}, referenceErr
}
if referenced {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil)
}
steps = append(steps, CleanupStepUndefineNetworkFilter)
digest := sha256.Sum256([]byte(document))
evidence = append(evidence, "network-filter:"+networkFilterName(uuid)+":"+hex.EncodeToString(digest[:]))
}
nvramExists := false
for _, name := range managedNVRAMArtifactNames(uuid) {
path := filepath.Join(l.local.NVRAMDirectory, name)
info, statErr := os.Lstat(path)
if errors.Is(statErr, os.ErrNotExist) {
continue
}
if statErr != nil {
return operationCleanupInspection{}, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr)
}
if !info.Mode().IsRegular() {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil)
}
referenced, referenceErr := l.anyDomainUsesPath(path)
if domain != nil {
referenced, referenceErr = l.otherDomainUsesPath(path, uuid)
}
if referenceErr != nil {
return operationCleanupInspection{}, referenceErr
}
if referenced {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil)
}
nvramExists = true
evidence = append(evidence, "nvram:"+name+":"+strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10))
}
if nvramExists {
steps = append(steps, CleanupStepDeleteNVRAM)
}
trafficPresent := l.traffic.contains(uuid)
if trafficPresent || len(steps) != 0 {
// cleanupDeletedVMArtifacts durably rewrites traffic state even if an
// earlier failed fsync left only an on-disk entry. Make that mutation
// explicit in every delete cleanup plan.
steps = append(steps, CleanupStepRemoveTrafficState)
if trafficPresent {
evidence = append(evidence, "traffic-state:"+uuid+":present")
} else {
evidence = append(evidence, "traffic-state:"+uuid+":rewrite")
}
}
return newOperationCleanupInspection(steps, evidence...), nil
}
func managedFilterDocument(document, expectedName string) bool {
var filter libvirtxml.NWFilter
if filter.Unmarshal(document) != nil || filter.Name != expectedName || filter.Chain != "root" {
return false
}
baselineReferences := 0
for _, entry := range filter.Entries {
if entry.Ref == nil {
continue
}
if entry.Ref.Filter != "clean-traffic" {
return false
}
baselineReferences++
}
return baselineReferences == 1
}
func storageVolumeInspectionDigest(volume *libvirt.StorageVol) (string, error) {
document, err := volume.GetXMLDesc(0)
if err != nil {
return "", translateLibvirtError("read cleanup volume XML", err)
}
digest := sha256.Sum256([]byte(document))
return hex.EncodeToString(digest[:]), nil
}
func (l *Libvirt) inspectReinstallOperation(record operation.Record,
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
replacementName := reinstallVolumeName(record.ResourceID, record.OperationID)
replacement, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
replacementExists := replacement != nil
if replacement != nil {
replacement.Free()
}
if diskTransitionCommitted(metadata, replacementName, "reinstall", record.OperationID) && replacementExists {
state, _, stateErr := domain.GetState()
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "reinstall", record.OperationID) {
return operation.Resolution{}, operationCleanupInspection{}
}
if metadata.PendingCleanupVolume != "" {
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
}
result, parseErr := metadataFromXML(document)
if parseErr != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
result.Status = mapDomainState(state)
result.ReinstallCommitted = true
result.CleanupPending = false
encoded, _ := json.Marshal(result)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
}
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "reinstall_not_committed", ErrorMessage: "host evidence confirms the reinstall did not switch the root volume"}, operationCleanupInspection{}
}
return operation.Resolution{}, operationCleanupInspection{}
}
func (l *Libvirt) inspectSnapshotOperation(record operation.Record,
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
parts := strings.Split(record.Action, ":")
if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) {
return operation.Resolution{}, operationCleanupInspection{}
}
key := parts[1]
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
var stored *domainSnapshotMetadataXML
for index := range metadata.Snapshots {
if metadata.Snapshots[index].Key == key {
stored = &metadata.Snapshots[index]
break
}
}
volume, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, key),
)
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
volumeExists := volume != nil
if volume != nil {
defer volume.Free()
}
switch parts[0] {
case "create-backup":
if !backupResidueCleanupAllowed(record, stored) {
return operation.Resolution{}, operationCleanupInspection{}
}
return operation.Resolution{}, l.backupResidueCleanupInspection(
domain, volume, snapshotVolumeName(record.ResourceID, key), key, stored,
)
case "create-snapshot":
if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") {
return operation.Resolution{}, operationCleanupInspection{}
}
if stored != nil && volumeExists && snapshotConsistencyMatchesRequest(parts[2], stored.Consistency) {
info, validateErr := validateQCOW2Volume(volume, stored.SizeBytes)
if validateErr != nil || info.Capacity != stored.SizeBytes {
return operation.Resolution{}, operationCleanupInspection{}
}
result := Snapshot{ID: key, SnapshotKey: key, Name: stored.Name,
Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(),
State: "ready", Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, Created: true}
encoded, _ := json.Marshal(result)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
}
if stored == nil && !volumeExists {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "snapshot_not_created", ErrorMessage: "host evidence confirms the snapshot is absent"}, operationCleanupInspection{}
}
if stored == nil && volumeExists {
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
domain, volume, snapshotVolumeName(record.ResourceID, key),
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
)
}
if stored != nil && !volumeExists {
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
)
}
case "delete-snapshot":
if len(parts) != 2 {
return operation.Resolution{}, operationCleanupInspection{}
}
if stored == nil && !volumeExists {
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
}
if stored != nil && volumeExists {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "snapshot_not_deleted", ErrorMessage: "host evidence confirms the snapshot remains present"}, operationCleanupInspection{}
}
if stored != nil && !volumeExists {
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
)
}
if stored == nil && volumeExists {
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
domain, volume, snapshotVolumeName(record.ResourceID, key),
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
)
}
case "restore-snapshot":
if len(parts) != 2 || stored == nil || !volumeExists {
return operation.Resolution{}, operationCleanupInspection{}
}
replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID)
replacement, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
if lookupErr != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
replacementExists := replacement != nil
if replacement != nil {
replacement.Free()
}
if diskTransitionCommitted(metadata, replacementName, "snapshot_restore", record.OperationID) && replacementExists {
state, _, stateErr := domain.GetState()
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "snapshot_restore", record.OperationID) {
return operation.Resolution{}, operationCleanupInspection{}
}
if metadata.PendingCleanupVolume != "" {
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
}
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Restored: true,
PowerState: mapDomainState(state), ActiveVolumeName: replacementName, CleanupPending: false})
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
}
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "snapshot_not_restored", ErrorMessage: "host evidence confirms the root volume was not switched"}, operationCleanupInspection{}
}
}
return operation.Resolution{}, operationCleanupInspection{}
}
func backupResidueCleanupAllowed(record operation.Record, stored *domainSnapshotMetadataXML) bool {
parts := strings.Split(record.Action, ":")
if len(parts) != 3 || parts[0] != "create-backup" ||
!catalogIDPattern.MatchString(parts[1]) ||
(parts[2] != "filesystem" && parts[2] != "crash") {
return false
}
if record.State == operation.StateSucceeded {
if !successfulBackupCleanupPending(record) {
return false
}
} else if record.State != operation.StateFailed || !record.EffectMayHaveApplied || record.ExecutionFenced {
return false
}
return stored == nil || (stored.Key == parts[1] && stored.SizeBytes > 0 &&
snapshotConsistencyMatchesRequest(parts[2], stored.Consistency))
}
func successfulBackupCleanupPending(record operation.Record) bool {
if record.State != operation.StateSucceeded || !recordHasCleanupPending(record) ||
len(operationEffectProjection(record, record.Result)) == 0 {
return false
}
return strings.HasPrefix(record.Action, "create-backup:")
}
// backupResidueCleanupInspection exposes only an evidence-bound cleanup plan
// for a local immutable copy left by an already-terminal backup worker. It
// never asserts whether the remote multipart object committed: failed records
// therefore remain partial, while successful records retain their immutable
// applied object projection through appliedBackupInspectionWithCleanup.
func (l *Libvirt) backupResidueCleanupInspection(domain cleanupJobDomain,
volume *libvirt.StorageVol, volumeName, key string,
stored *domainSnapshotMetadataXML) operationCleanupInspection {
if volume == nil && stored == nil {
return operationCleanupInspection{}
}
if volume != nil {
expectedCapacity := uint64(0)
if stored != nil {
expectedCapacity = stored.SizeBytes
}
if _, err := validateStandaloneQCOW2Volume(volume, expectedCapacity); err != nil {
return operationCleanupInspection{}
}
plan := l.snapshotVolumeCleanupInspection(
domain, volume, volumeName, CleanupStepDeleteSnapshotVolume,
"backup-volume:"+volumeName,
)
if len(plan.steps) == 0 {
return operationCleanupInspection{}
}
if stored != nil {
plan.steps = append(plan.steps, CleanupStepRemoveSnapshotMetadata)
plan.evidence = append(plan.evidence, "backup-metadata:"+key+":"+
stored.Consistency+":"+strconv.FormatUint(stored.SizeBytes, 10))
sort.Strings(plan.evidence)
}
return plan
}
return cleanupInspectionWithIdleDomain(
domain, []string{CleanupStepRemoveSnapshotMetadata},
"backup-metadata:"+key+":"+stored.Consistency+":"+
strconv.FormatUint(stored.SizeBytes, 10),
)
}
func diskTransitionPowerStateMatches(metadata *domainMetadataXML, state libvirt.DomainState,
kind, operationID string) bool {
if !diskTransitionMarkerMatches(metadata, kind, operationID) {
return true
}
if metadata.DiskTransitionRunning {
return state == libvirt.DOMAIN_RUNNING || state == libvirt.DOMAIN_BLOCKED
}
return state == libvirt.DOMAIN_SHUTOFF || state == libvirt.DOMAIN_SHUTDOWN
}
func (l *Libvirt) inspectDiskTransitionCleanupSteps(domain *libvirt.Domain, document string,
metadata *domainMetadataXML) operationCleanupInspection {
pending := metadata.PendingCleanupVolume
if pending == "" || !isVMOwnedRootVolumeName(metadataUUID(document), pending) ||
pending == metadata.RootVolumeName {
return operationCleanupInspection{}
}
jobEvidence, safe := inspectCleanupJobEvidence(domain)
if !safe {
return operationCleanupInspection{}
}
steps := make([]string, 0, 2)
evidence := append([]string(nil), jobEvidence...)
evidence = append(evidence, "pending-root:"+pending, "active-root:"+metadata.RootVolumeName)
old, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), pending)
if err != nil {
return operationCleanupInspection{}
}
if old != nil {
if _, validateErr := validateQCOW2Volume(old, 0); validateErr != nil {
old.Free()
return operationCleanupInspection{}
}
oldDigest, digestErr := storageVolumeInspectionDigest(old)
if digestErr != nil {
old.Free()
return operationCleanupInspection{}
}
path, pathErr := old.GetPath()
old.Free()
if pathErr != nil {
return operationCleanupInspection{}
}
used, referenceErr := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), pending,
)
if referenceErr != nil || used {
return operationCleanupInspection{}
}
steps = append(steps, CleanupStepDeletePreviousRoot)
evidence = append(evidence, "pending-root-state:present:"+oldDigest)
} else {
evidence = append(evidence, "pending-root-state:absent")
}
steps = append(steps, CleanupStepPersistCleanupMetadata)
return newOperationCleanupInspection(steps, evidence...)
}
type cleanupJobDomain interface {
IsActive() (bool, error)
GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error)
GetXMLDesc(libvirt.DomainXMLFlags) (string, error)
GetBlockJobInfo(string, libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error)
}
func inspectCleanupJobEvidence(domain cleanupJobDomain) ([]string, bool) {
active, err := domain.IsActive()
if err != nil {
return nil, false
}
flags := libvirt.DOMAIN_XML_INACTIVE
evidence := []string{"domain-runtime:inactive"}
if active {
flags = 0
job, jobErr := domain.GetJobStats(0)
if jobErr != nil || job == nil || job.Type != libvirt.DOMAIN_JOB_NONE {
return nil, false
}
evidence[0] = "domain-runtime:active:job-none"
}
document, err := domain.GetXMLDesc(flags)
if err != nil {
return nil, false
}
documentDigest := sha256.Sum256([]byte(document))
evidence = append(evidence, "domain-xml:"+hex.EncodeToString(documentDigest[:]))
if active {
persistentDocument, persistentErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if persistentErr != nil {
return nil, false
}
persistentDigest := sha256.Sum256([]byte(persistentDocument))
evidence = append(evidence, "persistent-domain-xml:"+hex.EncodeToString(persistentDigest[:]))
}
targets, err := cleanupDiskTargets(document)
if err != nil {
return nil, false
}
for _, target := range targets {
evidence = append(evidence, "disk-target:"+target)
if !active {
continue
}
info, infoErr := domain.GetBlockJobInfo(target, 0)
if infoErr != nil || !blockJobInfoAbsent(info) {
return nil, false
}
}
return evidence, true
}
func cleanupDiskTargets(document string) ([]string, error) {
var definition libvirtxml.Domain
if err := definition.Unmarshal(document); err != nil || definition.Devices == nil {
return nil, errors.New("domain disk evidence is invalid")
}
seen := make(map[string]struct{}, len(definition.Devices.Disks))
for _, disk := range definition.Devices.Disks {
if disk.Target == nil || disk.Target.Dev == "" {
return nil, errors.New("domain disk target evidence is incomplete")
}
seen[disk.Target.Dev] = struct{}{}
}
targets := make([]string, 0, len(seen))
for target := range seen {
targets = append(targets, target)
}
sort.Strings(targets)
return targets, nil
}
func blockJobInfoAbsent(info *libvirt.DomainBlockJobInfo) bool {
// The Go binding does not expose virDomainGetBlockJobInfo's 0/1 return
// value. A 0 (no job) is represented by the zero-valued info struct.
return info != nil && info.Type == libvirt.DOMAIN_BLOCK_JOB_TYPE_UNKNOWN &&
info.Bandwidth == 0 && info.Cur == 0 && info.End == 0
}
func cleanupInspectionWithIdleDomain(domain cleanupJobDomain, steps []string,
evidence ...string) operationCleanupInspection {
jobEvidence, safe := inspectCleanupJobEvidence(domain)
if !safe {
return operationCleanupInspection{}
}
evidence = append(evidence, jobEvidence...)
return newOperationCleanupInspection(steps, evidence...)
}
func ensureNoCleanupBlockingJobs(domain cleanupJobDomain) error {
if _, safe := inspectCleanupJobEvidence(domain); !safe {
return NewError(ErrConflict, "cleanup_job_not_idle", "domain cleanup is blocked by active or unreadable job state", nil)
}
return nil
}
func (l *Libvirt) snapshotVolumeCleanupInspection(domain cleanupJobDomain, volume *libvirt.StorageVol,
volumeName, step, targetEvidence string) operationCleanupInspection {
if volume == nil {
return operationCleanupInspection{}
}
jobEvidence, safe := inspectCleanupJobEvidence(domain)
if !safe {
return operationCleanupInspection{}
}
if _, err := validateQCOW2Volume(volume, 0); err != nil {
return operationCleanupInspection{}
}
volumeDigest, err := storageVolumeInspectionDigest(volume)
if err != nil {
return operationCleanupInspection{}
}
path, err := volume.GetPath()
if err != nil {
return operationCleanupInspection{}
}
referenced, err := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), volumeName,
)
if err != nil || referenced {
return operationCleanupInspection{}
}
evidence := append([]string{targetEvidence + ":" + volumeDigest}, jobEvidence...)
return newOperationCleanupInspection([]string{step}, evidence...)
}
func (l *Libvirt) inspectMatchingBackupJob(record operation.Record, domain *libvirt.Domain) OperationAbortInspection {
if record.State != operation.StateDispatching &&
(record.State != operation.StateFailed || !record.EffectMayHaveApplied) {
return OperationAbortInspection{}
}
parts := strings.Split(record.Action, ":")
if len(parts) != 3 || (parts[0] != "create-snapshot" && parts[0] != "create-backup") ||
!catalogIDPattern.MatchString(parts[1]) || (parts[2] != "filesystem" && parts[2] != "crash") {
return OperationAbortInspection{}
}
job, err := domain.GetJobStats(0)
if err != nil || job == nil ||
(job.Type != libvirt.DOMAIN_JOB_BOUNDED && job.Type != libvirt.DOMAIN_JOB_UNBOUNDED) ||
!job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
return OperationAbortInspection{}
}
backupXML, err := domain.BackupGetXMLDesc(0)
if err != nil {
return OperationAbortInspection{}
}
volume, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, parts[1]),
)
if err != nil || volume == nil {
return OperationAbortInspection{}
}
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
volume.Free()
return OperationAbortInspection{}
}
targetPath, pathErr := volume.GetPath()
volume.Free()
if pathErr != nil || !backupPushMatchesTarget(backupXML, targetPath) {
return OperationAbortInspection{}
}
return OperationAbortInspection{
Available: true, Kind: "backup",
InspectionToken: operationInspectionToken(record, "backup", backupXML),
}
}
func backupPushMatchesTarget(backupXML, targetPath string) bool {
var backup libvirtxml.DomainBackup
if backup.Unmarshal(backupXML) != nil || backup.Pull != nil || backup.Push == nil ||
backup.Push.Disks == nil || len(backup.Push.Disks.Disks) != 1 {
return false
}
disk := backup.Push.Disks.Disks[0]
return disk.Name == "vda" && disk.Backup == "yes" && disk.Target != nil &&
disk.Target.File != nil && filepath.Clean(disk.Target.File.File) == filepath.Clean(targetPath)
}
func buildOperationInspection(record operation.Record, resolution operation.Resolution,
abort OperationAbortInspection, cleanup operationCleanupInspection) OperationInspection {
result := OperationInspection{
OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID,
RequestFingerprint: record.Fingerprint, OperationStatus: record.State,
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
SummaryCode: InspectionSummaryIncomplete,
EffectResolution: InspectionEffectInconclusive,
AbortJob: abort, Cleanup: OperationCleanupPlan{},
}
if abort.Available {
result.SummaryCode = InspectionSummaryAbortableJob
return result
}
if len(cleanup.steps) != 0 && len(cleanup.evidence) != 0 {
result.SummaryCode = InspectionSummaryCleanupRequired
result.EffectResolution = InspectionEffectPartial
result.Cleanup = OperationCleanupPlan{
Available: true,
PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()),
Steps: append([]string(nil), cleanup.steps...),
}
return result
}
if !resolution.Resolved {
return result
}
if record.Action == "resize-vm" && !resolution.Succeeded && resolution.ErrorCode == "resources_partially_resized" {
result.ErrorCode = resolution.ErrorCode
result.EffectResolution = InspectionEffectPartial
return result
}
if resolution.Succeeded {
result.Outcome = InspectionOutcomeSucceeded
result.SummaryCode = InspectionSummarySucceeded
projection := operationEffectProjection(record, resolution.Result)
result.EffectResolution = inspectedEffectResolution(record, resolution, projection)
if result.EffectResolution == InspectionEffectApplied {
result.EffectProjection = projection
} else {
result.Outcome = InspectionOutcomeUncertain
result.SummaryCode = InspectionSummaryIncomplete
}
} else {
result.Outcome = InspectionOutcomeFailed
result.SummaryCode = InspectionSummaryFailed
projection := failedOperationEffectProjection(record, resolution)
result.EffectResolution = inspectedEffectResolution(record, resolution, projection)
if result.EffectResolution == InspectionEffectNotApplied {
result.EffectProjection = projection
} else {
result.Outcome = InspectionOutcomeUncertain
result.SummaryCode = InspectionSummaryIncomplete
}
if inspectionErrorCodePattern.MatchString(resolution.ErrorCode) {
result.ErrorCode = resolution.ErrorCode
}
}
return result
}
// inspectedEffectResolution separates immutable operation evidence from a
// read-only observation that merely happens to match the requested target.
// Power, service, network-link, catalog and storage state may be changed out
// of band, so those observations remain inconclusive even when the target is
// currently reached. Only a Journal terminal record or a mutation-specific
// durable marker may authorize automated ledger convergence.
func inspectedEffectResolution(record operation.Record, resolution operation.Resolution,
projection json.RawMessage) string {
if record.State == operation.StateSucceeded {
return InspectionEffectApplied
}
if record.State == operation.StateFailed && !record.EffectMayHaveApplied {
return InspectionEffectNotApplied
}
if record.State != operation.StateFailed || !record.EffectMayHaveApplied || !resolution.Resolved {
return InspectionEffectInconclusive
}
if resolution.Succeeded {
if len(projection) == 0 {
return InspectionEffectInconclusive
}
switch {
case record.Action == "update-settings",
record.Action == "resize-vm",
record.Action == "change-network",
record.Action == "reinstall-vm",
record.Action == "apply-firewall",
strings.HasPrefix(record.Action, "restore-snapshot:"):
return InspectionEffectApplied
default:
return InspectionEffectInconclusive
}
}
switch {
case operation.IsCreateLikeAction(record.Action) && resolution.ErrorCode == "creation_absent" && len(projection) != 0,
record.Action == "update-settings" && resolution.ErrorCode == "settings_not_committed",
record.Action == "change-network" && resolution.ErrorCode == "network_not_committed",
record.Action == "reinstall-vm" && resolution.ErrorCode == "reinstall_not_committed",
record.Action == "apply-firewall" && resolution.ErrorCode == "firewall_not_committed",
strings.HasPrefix(record.Action, "restore-snapshot:") && resolution.ErrorCode == "snapshot_not_restored":
return InspectionEffectNotApplied
default:
return InspectionEffectInconclusive
}
}
func inspectionFromRecord(record operation.Record) OperationInspection {
switch record.State {
case operation.StateSucceeded:
return buildOperationInspection(record, operation.Resolution{
Resolved: true, Succeeded: true, Result: append([]byte(nil), record.Result...),
}, OperationAbortInspection{}, operationCleanupInspection{})
case operation.StateFailed:
resolution := operation.Resolution{Resolved: true, Succeeded: false, Result: append([]byte(nil), record.Result...)}
if record.Error != nil {
resolution.ErrorCode = record.Error.Code
resolution.ErrorMessage = record.Error.Message
}
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{})
}
return inconclusiveInspectionFromRecord(record)
}
func inconclusiveInspectionFromRecord(record operation.Record) OperationInspection {
return OperationInspection{
OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID,
RequestFingerprint: record.Fingerprint, OperationStatus: record.State,
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
SummaryCode: InspectionSummaryIncomplete, EffectResolution: InspectionEffectInconclusive,
}
}
func failedOperationEffectProjection(record operation.Record, resolution operation.Resolution) json.RawMessage {
if !operation.IsCreateLikeAction(record.Action) || resolution.ErrorCode != "creation_absent" {
return nil
}
var evidence struct {
CreationAbsent bool `json:"creation_absent"`
ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"`
}
if !decodeOperationEffectResult(resolution.Result, &evidence) ||
!evidence.CreationAbsent || !evidence.ManagedArtifactsAbsent {
return nil
}
encoded, err := json.Marshal(evidence)
if err != nil {
return nil
}
return encoded
}
const maxOperationEffectProjectionBytes = 64 << 10
// operationEffectProjection never forwards Journal bytes. It first decodes a
// result into the exact response type allowed for the recorded action and then
// rebuilds a deliberately smaller projection. Unknown actions, malformed or
// oversized results, and identity mismatches simply omit the optional
// projection while keeping the independently derived effect resolution.
func operationEffectProjection(record operation.Record, raw []byte) json.RawMessage {
if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes || !json.Valid(raw) {
return nil
}
var projection any
switch {
case record.Action == "configure-storage":
var value StorageConfigurationResult
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = struct {
Configured bool `json:"configured"`
}{Configured: value.Configured}
case strings.HasPrefix(record.Action, "create-backup:"):
parts := strings.Split(record.Action, ":")
var value BackupResult
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) ||
(parts[2] != "filesystem" && parts[2] != "crash") ||
!decodeOperationEffectResult(raw, &value) || value.BackupKey != parts[1] ||
value.SizeBytes <= 0 || value.VirtualSizeBytes == 0 ||
!lowerHexDigest(value.SHA256) ||
!snapshotConsistencyMatchesRequest(parts[2], value.Consistency) {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "restore-backup:"):
parts := strings.Split(record.Action, ":")
var value RestoreBackupResult
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
!decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID ||
value.BackupKey != parts[1] || !lowerHexDigest(value.SHA256) {
return nil
}
projection = struct {
operationVMEffectProjection
BackupKey string `json:"backup_key"`
SHA256 string `json:"sha256"`
}{
operationVMEffectProjection: vmEffectProjection(value.VMMetadata),
BackupKey: value.BackupKey,
SHA256: value.SHA256,
}
case operation.IsCreateLikeAction(record.Action) || record.Action == "update-settings" ||
record.Action == "change-network" || record.Action == "reinstall-vm":
var value VMMetadata
if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID {
return nil
}
projection = vmEffectProjection(value)
case record.Action == "resize-vm":
var value ResizeResourcesResult
expected := record.ExpectedState
if !decodeOperationEffectResult(raw, &value) || expected == nil || expected.Kind != operation.ExpectedResizeResources ||
expected.TransitionID != record.OperationID || value.UUID != record.ResourceID || !value.ResourcesResized ||
value.FromVCPUs != expected.FromVCPUs || value.TargetVCPUs != expected.TargetVCPUs ||
value.FromMemoryMiB != expected.FromMemoryMiB || value.TargetMemoryMiB != expected.TargetMemoryMiB ||
value.FromDiskGiB != expected.FromDiskGiB || value.TargetDiskGiB != expected.TargetDiskGiB ||
value.FromBandwidthMbps != expected.FromBandwidthMbps || value.TargetBandwidthMbps != expected.TargetBandwidthMbps ||
value.AppliedVCPUs != value.TargetVCPUs || value.AppliedMemoryMiB != value.TargetMemoryMiB ||
value.AppliedDiskGiB != value.TargetDiskGiB || value.AppliedBandwidthMbps != value.TargetBandwidthMbps ||
!value.PowerStateRestored ||
(value.OriginallyRunning && value.Status != VMStateRunning && value.Status != VMStateBlocked) ||
(!value.OriginallyRunning && value.Status != VMStateShutoff && value.Status != VMStateCrashed) {
return nil
}
projection = value
case record.Action == "delete-vm" || record.Action == "force-delete-vm":
var value DeleteVMResult
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "power-") || strings.HasPrefix(record.Action, "service-"):
var value VMState
if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID {
return nil
}
projection = struct {
UUID string `json:"uuid"`
State VMStateName `json:"state"`
}{UUID: value.UUID, State: value.State}
case record.Action == "network-suspend" || record.Action == "network-resume":
var value struct {
NetworkEnabled bool `json:"network_enabled"`
}
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "create-snapshot:"):
var value Snapshot
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = operationSnapshotEffectProjection{
ID: value.ID, SnapshotKey: value.SnapshotKey, Name: value.Name,
CreatedAt: value.CreatedAt, State: value.State, Consistency: value.Consistency,
SizeBytes: value.SizeBytes, Created: value.Created,
}
case strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:"):
var value SnapshotMutationResult
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = struct {
SnapshotKey string `json:"snapshot_key"`
Restored bool `json:"restored,omitempty"`
Deleted bool `json:"deleted,omitempty"`
PowerState VMStateName `json:"power_state,omitempty"`
CleanupPending bool `json:"cleanup_pending,omitempty"`
}{value.SnapshotKey, value.Restored, value.Deleted, value.PowerState, value.CleanupPending}
case record.Action == "apply-firewall":
var value FirewallPolicy
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = struct {
Revision uint64 `json:"revision"`
DefaultIn string `json:"default_in"`
DefaultOut string `json:"default_out"`
PolicyDigest string `json:"policy_digest,omitempty"`
Rules []FirewallRule `json:"rules"`
}{value.Revision, value.DefaultIn, value.DefaultOut, value.PolicyDigest, append([]FirewallRule(nil), value.Rules...)}
case record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
strings.HasPrefix(record.Action, "exit-rescue:"):
var value RescueState
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = struct {
Active bool `json:"active"`
Staged bool `json:"staged"`
PendingManualRestart bool `json:"pending_manual_restart,omitempty"`
TemplateID string `json:"template_id,omitempty"`
NetworkSuspended bool `json:"network_suspended"`
}{value.Active, value.Staged, value.PendingManualRestart, value.TemplateID, value.NetworkSuspended}
case record.Action == "import-template" || record.Action == "update-template":
var value ImageTemplate
if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "template:"+value.ID {
return nil
}
projection = operationCatalogImportProjection{
ID: value.ID, Name: value.Name, OSFamily: value.OSFamily, Version: value.Version,
Architecture: value.Architecture, SizeBytes: value.SizeBytes,
VirtualSizeBytes: value.VirtualSizeBytes, SHA256: value.SHA256,
Revision: value.Revision, GenerationToken: value.GenerationToken, CreatedAt: value.CreatedAt,
}
case record.Action == "import-rescue-template":
var value RescueTemplate
if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "rescue-template:"+value.ID {
return nil
}
projection = operationCatalogImportProjection{
ID: value.ID, Name: value.Name, SizeBytes: value.SizeBytes,
SHA256: value.SHA256, CreatedAt: value.CreatedAt,
}
case record.Action == "import-user-iso":
var value UserISO
if !decodeOperationEffectResult(raw, &value) ||
record.ResourceID != "user-iso:"+strconv.FormatInt(value.OwnerID, 10)+":"+value.ID {
return nil
}
projection = operationCatalogImportProjection{
ID: value.ID, OwnerID: value.OwnerID, Name: value.Name,
SizeBytes: value.SizeBytes, SHA256: value.SHA256, CreatedAt: value.CreatedAt,
}
case record.Action == "delete-template" || record.Action == "delete-rescue-template" ||
record.Action == "delete-user-iso":
var value CatalogDeleteResult
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = value
default:
return nil
}
encoded, err := json.Marshal(projection)
if err != nil || len(encoded) == 0 || len(encoded) > maxOperationEffectProjectionBytes {
return nil
}
return encoded
}
type operationVMEffectProjection struct {
UUID string `json:"uuid"`
Name string `json:"name,omitempty"`
Status VMStateName `json:"status,omitempty"`
VCPUs uint `json:"vcpu,omitempty"`
MemoryMiB uint64 `json:"memory_mib,omitempty"`
DiskGiB uint64 `json:"disk_gib,omitempty"`
BandwidthMbps uint64 `json:"bandwidth_mbps,omitempty"`
Hostname string `json:"hostname,omitempty"`
PublicIPCIDR string `json:"public_ip_cidr,omitempty"`
PublicPoolKey string `json:"public_pool_key,omitempty"`
PublicPoolCIDR string `json:"public_pool_cidr,omitempty"`
Gateway string `json:"gateway,omitempty"`
DNSServers []string `json:"dns_servers,omitempty"`
NetworkSelector string `json:"network,omitempty"`
TemplateID string `json:"template_id,omitempty"`
Machine string `json:"machine,omitempty"`
Firmware string `json:"firmware,omitempty"`
SecureBoot bool `json:"secure_boot,omitempty"`
SettingsChanged bool `json:"settings_changed,omitempty"`
NetworkChanged bool `json:"network_changed,omitempty"`
ReinstallCommitted bool `json:"reinstall_committed,omitempty"`
CleanupPending bool `json:"cleanup_pending,omitempty"`
}
func vmEffectProjection(value VMMetadata) operationVMEffectProjection {
return operationVMEffectProjection{
UUID: value.UUID, Name: value.Name, Status: value.Status,
VCPUs: value.VCPUs, MemoryMiB: value.MemoryMiB, DiskGiB: value.DiskGiB,
BandwidthMbps: value.BandwidthMbps, Hostname: value.Hostname,
PublicIPCIDR: value.PublicIPCIDR, PublicPoolKey: value.PublicPoolKey,
PublicPoolCIDR: value.PublicPoolCIDR, Gateway: value.Gateway,
DNSServers: append([]string(nil), value.DNSServers...),
NetworkSelector: value.NetworkSelector, TemplateID: value.TemplateID,
Machine: value.Machine, Firmware: value.Firmware, SecureBoot: value.SecureBoot,
SettingsChanged: value.SettingsChanged, NetworkChanged: value.NetworkChanged,
ReinstallCommitted: value.ReinstallCommitted, CleanupPending: value.CleanupPending,
}
}
type operationSnapshotEffectProjection struct {
ID string `json:"id"`
SnapshotKey string `json:"snapshot_key"`
Name string `json:"name,omitempty"`
CreatedAt time.Time `json:"created_at"`
State string `json:"state"`
Consistency string `json:"consistency"`
SizeBytes uint64 `json:"size_bytes,omitempty"`
Created bool `json:"created,omitempty"`
}
type operationCatalogImportProjection struct {
ID string `json:"id"`
OwnerID int64 `json:"owner_id,omitempty"`
Name string `json:"name,omitempty"`
OSFamily string `json:"os_family,omitempty"`
Version string `json:"version,omitempty"`
Architecture string `json:"architecture,omitempty"`
SizeBytes uint64 `json:"size_bytes,omitempty"`
VirtualSizeBytes uint64 `json:"virtual_size_bytes,omitempty"`
SHA256 string `json:"sha256,omitempty"`
Revision uint64 `json:"revision,omitempty"`
GenerationToken string `json:"generation_token,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
func decodeOperationEffectResult(raw []byte, target any) bool {
if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes {
return false
}
trimmed := strings.TrimSpace(string(raw))
return len(trimmed) >= 2 && trimmed[0] == '{' && trimmed[len(trimmed)-1] == '}' &&
json.Unmarshal(raw, target) == nil
}
func recordHasCleanupPending(record operation.Record) bool {
if len(record.Result) == 0 {
return false
}
var result struct {
CleanupPending bool `json:"cleanup_pending"`
}
return json.Unmarshal(record.Result, &result) == nil && result.CleanupPending
}
func operationInspectionToken(record operation.Record, kind, evidence string) string {
digest := sha256.New()
for _, part := range []string{
"virt-cloud-operation-inspection-v1", record.OperationID, record.ResourceID,
record.Action, record.Fingerprint, kind, evidence,
} {
digest.Write([]byte(part))
digest.Write([]byte{0})
}
return hex.EncodeToString(digest.Sum(nil))
}
func lowerHexDigest(value string) bool {
if len(value) != sha256.Size*2 {
return false
}
for _, current := range value {
if (current < '0' || current > '9') && (current < 'a' || current > 'f') {
return false
}
}
return true
}