You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_operation_admin.go

2038 lines
80 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/peermigration"
"git.luoli.live/root/virt/agent/publicnetwork"
"git.luoli.live/root/virt/agent/vpcrouting"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
var inspectionErrorCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
// operationCleanupInspection carries the exact, private host evidence used to
// bind a cleanup plan. Only the stable step codes are serialized; resource
// names, XML digests, and file attributes stay inside the Agent and feed the
// one-way plan identifier.
type operationCleanupInspection struct {
steps []string
evidence []string
}
func newOperationCleanupInspection(steps []string, evidence ...string) operationCleanupInspection {
if len(evidence) == 0 {
return operationCleanupInspection{}
}
normalizedEvidence := append([]string(nil), evidence...)
sort.Strings(normalizedEvidence)
return operationCleanupInspection{
steps: append([]string(nil), steps...), evidence: normalizedEvidence,
}
}
func (inspection operationCleanupInspection) tokenEvidence() string {
encoded, _ := json.Marshal(struct {
Steps []string `json:"steps"`
Evidence []string `json:"evidence"`
}{Steps: inspection.steps, Evidence: inspection.evidence})
return string(encoded)
}
// InspectOperation deliberately bypasses Journal admission. A dispatched
// libvirt call retains its resource lease until the real call returns, but an
// administrator must still be able to read its state and identify an owned
// cancellable backup job while that call is stuck. Every operation below is a
// libvirt/filesystem read. It may return a cleanup plan, but inspection never
// executes that plan or changes a host resource.
func (l *Libvirt) InspectOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
if err := ctx.Err(); err != nil {
return OperationInspection{}, err
}
if successfulBackupCleanupPending(record) {
if record.InspectionEvidenceUnavailable {
return inconclusiveInspectionFromRecord(record), nil
}
return l.inspectSuccessfulBackupCleanupPending(ctx, record), nil
}
if storedOperationInspectionAvailable(record) {
if record.InspectionEvidenceUnavailable {
return inconclusiveInspectionFromRecord(record), nil
}
inspection := inspectionFromRecord(record)
// A recovered successful receipt is useful for convergence only when its
// action-specific result can be reduced to the bounded public projection.
// Receipt status alone must not turn an unprojectable or damaged result
// into an "applied" verdict after Journal detail retention.
if record.InspectionReceiptRecovered && record.State == operation.StateSucceeded &&
len(inspection.EffectProjection) == 0 {
return inconclusiveInspectionFromRecord(record), nil
}
return inspection, nil
}
if resolution, handled, err := l.reconcileStorageConfiguration(record); handled {
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
}
if resolution, handled, err := l.reconcileCatalogOperation(ctx, record); handled {
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
}
if resolution, handled, err := l.reconcileVPCNetwork(record); handled {
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
}
domain, err := l.domainByUUID(ctx, record.ResourceID)
if errors.Is(err, ErrNotFound) {
return l.inspectAbsentDomainOperation(ctx, record)
}
if err != nil {
return OperationInspection{}, err
}
defer domain.Free()
abort := l.inspectMatchingBackupJob(record, domain)
resolution, cleanup, inspectErr := l.inspectExistingDomainOperation(ctx, record, domain)
if abort.Available {
// No cleanup may be approved while the matching backup is still live.
cleanup = operationCleanupInspection{}
}
return buildOperationInspection(record, resolution, abort, cleanup), inspectErr
}
func storedOperationInspectionAvailable(record operation.Record) bool {
if record.Tombstone {
return true
}
if record.State == operation.StateSucceeded && strings.HasPrefix(record.Action, "create-backup:") {
// A successful backup result is immutable proof that the remote object
// committed, even when its optional cleanup_pending marker reports only
// local residue. Never discard the sole object SHA/size projection by
// replacing it with a weaker current-host observation.
return true
}
if record.State == operation.StateSucceeded && !recordHasCleanupPending(record) {
return true
}
if record.State == operation.StateFailed && record.EffectMayHaveApplied {
// A failed Journal row describes the response observed by the original
// caller, not necessarily the durable host effect. Reuse would make the
// operation look definitively failed forever and bypass the read-only
// expected-state reconcilers below.
return false
}
return record.State == operation.StateFailed && !operation.IsCreateLikeAction(record.Action) &&
!operation.RequiresExplicitForceDelete(record)
}
// inspectSuccessfulBackupCleanupPending preserves the immutable applied
// object projection while adding an optional, read-only cleanup plan for an
// exact operation-owned local workspace residue. Host read failures never
// downgrade the already verified remote object identity.
func (l *Libvirt) inspectSuccessfulBackupCleanupPending(ctx context.Context,
record operation.Record) OperationInspection {
inspection := inspectionFromRecord(record)
domain, err := l.domainByUUID(ctx, record.ResourceID)
if errors.Is(err, ErrNotFound) {
parts := strings.Split(record.Action, ":")
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) {
return inspection
}
volumeName, volume, _, residueErr := l.backupWorkspaceResidue(
record.ResourceID, parts[1],
)
if residueErr != nil || volume == nil {
return inspection
}
defer volume.Free()
cleanup := l.absentDomainBackupWorkspaceCleanupInspection(
volume, volumeName,
)
return appliedBackupInspectionWithCleanup(
record, inspection, cleanup,
)
}
if err != nil {
return inspection
}
defer domain.Free()
_, cleanup := l.inspectSnapshotOperation(ctx, record, domain)
return appliedBackupInspectionWithCleanup(record, inspection, cleanup)
}
func appliedBackupInspectionWithCleanup(record operation.Record, inspection OperationInspection,
cleanup operationCleanupInspection) OperationInspection {
if len(cleanup.steps) == 0 || len(cleanup.evidence) == 0 {
return inspection
}
inspection.SummaryCode = InspectionSummaryCleanupRequired
inspection.Cleanup = OperationCleanupPlan{
Available: true,
PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()),
Steps: append([]string(nil), cleanup.steps...),
}
return inspection
}
func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
var resolution operation.Resolution
var cleanup operationCleanupInspection
switch {
case strings.HasPrefix(record.Action, "receive-peer-migration:"):
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil)
if err != nil {
return OperationInspection{}, err
}
if len(plan.steps) != 0 {
cleanup = plan
break
}
resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, true, false)
case operation.IsCreateLikeAction(record.Action):
absent, err := l.deleteArtifactsAbsent(record.ResourceID)
if err != nil {
return OperationInspection{}, err
}
resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, absent, l.traffic.contains(record.ResourceID))
case record.Action == "delete-vm" || record.Action == "force-delete-vm" ||
strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"):
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil)
if err != nil {
return OperationInspection{}, err
}
cleanup = plan
resolution = inspectAbsentDeleteResolution(plan, record.Action == "force-delete-vm")
case strings.HasPrefix(record.Action, "cleanup-backup-workspace:"):
resolution = l.reconcileAbsentDomainBackupWorkspaceCleanup(record)
default:
if record.ExpectedState != nil {
resolution = expectedStateResourceAbsent(record)
}
}
return buildOperationInspection(record, resolution, OperationAbortInspection{}, cleanup), nil
}
func (l *Libvirt) reconcileAbsentDomainBackupWorkspaceCleanup(
record operation.Record,
) operation.Resolution {
parts := strings.Split(record.Action, ":")
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) {
return operation.Resolution{}
}
_, volume, _, err := l.backupWorkspaceResidue(
record.ResourceID, parts[1],
)
if err != nil {
return operation.Resolution{}
}
if volume == nil {
result, _ := json.Marshal(CleanupBackupWorkspaceResult{
BackupKey: parts[1], Cleaned: true,
WorkspaceAbsent: true, MetadataAbsent: true,
})
return operation.Resolution{
Resolved: true, Succeeded: true, Result: result,
}
}
defer volume.Free()
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
return operation.Resolution{}
}
return operation.Resolution{
Resolved: true, Succeeded: false,
ErrorCode: "backup_workspace_cleanup_not_applied",
ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied",
}
}
func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePresent bool) (operation.Resolution, operationCleanupInspection) {
if !managedArtifactsAbsent {
return operation.Resolution{}, operationCleanupInspection{}
}
resolution := operation.Resolution{
Resolved: true, Succeeded: false,
Result: []byte(`{"creation_absent":true,"managed_artifacts_absent":true}`),
ErrorCode: "creation_absent",
ErrorMessage: "host evidence confirms the VM and managed artifacts are absent",
}
if trafficStatePresent {
return resolution, newOperationCleanupInspection(
[]string{CleanupStepRemoveTrafficState}, "traffic-state:"+uuid,
)
}
return resolution, operationCleanupInspection{}
}
func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record operation.Record,
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection, error) {
if strings.HasPrefix(record.Action, "cleanup-peer-migration-target:") {
// This successor is authorized only to clean an absent target. A domain
// appearing before or after dispatch is never converted into a delete
// plan by reconciliation.
return operation.Resolution{}, operationCleanupInspection{}, nil
}
if record.Action == "delete-vm" || record.Action == "force-delete-vm" {
_, managed, err := requireManagedDomain(domain)
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}, err
}
if err := l.validateDeleteVMArtifacts(ctx, record.ResourceID, managed); err != nil {
return operation.Resolution{}, operationCleanupInspection{}, err
}
active, err := domain.IsActive()
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
}
if resolution, handled := inspectActiveDeleteResolution(record, active); handled {
return resolution, operationCleanupInspection{}, nil
}
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, domain, managed)
return operation.Resolution{}, plan, err
}
if record.ExpectedState != nil {
resolution, handled, err := l.inspectExpectedStateOperation(ctx, record, domain)
if handled {
return resolution, operationCleanupInspection{}, err
}
}
if strings.HasPrefix(record.Action, "prepare-peer-migration:") {
resolution, err := l.reconcilePreparedPeerMigration(record, domain)
return resolution, operationCleanupInspection{}, err
}
if strings.HasPrefix(record.Action, "receive-peer-migration:") {
resolution, err := l.reconcileReceivedPeerMigration(ctx, record, domain)
return resolution, operationCleanupInspection{}, err
}
if record.Action == "reinstall-vm" {
resolution, steps := l.inspectReinstallOperation(record, domain)
return resolution, steps, nil
}
if strings.HasPrefix(record.Action, "create-snapshot:") ||
strings.HasPrefix(record.Action, "create-backup:") ||
strings.HasPrefix(record.Action, "cleanup-backup-workspace:") ||
strings.HasPrefix(record.Action, "delete-snapshot:") ||
strings.HasPrefix(record.Action, "restore-snapshot:") ||
strings.HasPrefix(record.Action, "recover-backup:") {
resolution, steps := l.inspectSnapshotOperation(ctx, record, domain)
return resolution, steps, nil
}
if record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
strings.HasPrefix(record.Action, "exit-rescue:") {
return l.reconcileRescue(ctx, record, domain), operationCleanupInspection{}, nil
}
state, _, err := domain.GetState()
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect operation domain state", err)
}
mapped := mapDomainState(state)
switch {
case operation.IsCreateLikeAction(record.Action):
resolution, err := l.reconcileCreatedVM(ctx, record, domain)
return resolution, operationCleanupInspection{}, err
case record.Action == "power-start":
if mapped == VMStateRunning || mapped == VMStateBlocked {
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
}
case record.Action == "power-shutdown" || record.Action == "power-destroy":
if mapped == VMStateShutoff {
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
}
}
return operation.Resolution{}, operationCleanupInspection{}, nil
}
func peerMigrationActionID(action, prefix string) (string, bool) {
parts := strings.Split(action, ":")
returnValue := ""
if len(parts) == 2 && parts[0] == prefix && catalogIDPattern.MatchString(parts[1]) {
returnValue = parts[1]
}
return returnValue, returnValue != ""
}
func (l *Libvirt) reconcilePreparedPeerMigration(record operation.Record,
domain *libvirt.Domain) (operation.Resolution, error) {
migrationID, ok := peerMigrationActionID(record.Action, "prepare-peer-migration")
if !ok {
return operation.Resolution{}, nil
}
store, err := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
)
if err != nil {
return operation.Resolution{}, err
}
manifest, manifestErr := store.LoadSource(migrationID)
if manifestErr == nil {
autostart, autoErr := domain.GetAutostart()
active, activeErr := domain.IsActive()
if autoErr != nil || activeErr != nil {
return operation.Resolution{}, errors.Join(autoErr, activeErr)
}
if autostart || active || !strings.EqualFold(manifest.SourceUUID, record.ResourceID) {
return operation.Resolution{}, nil
}
digest, err := manifest.Public().Digest()
if err != nil {
return operation.Resolution{}, err
}
result, err := json.Marshal(PreparePeerMigrationResult{
MigrationID: migrationID, ManifestDigest: digest,
Artifacts: manifest.Public().Artifacts, SourceStopped: true,
SourceWasRunning: manifest.SourceWasRunning,
SourceAutostartEnabled: manifest.SourceAutostartEnabled,
SourceAutostartDisabled: true,
})
if err != nil {
return operation.Resolution{}, err
}
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, nil
}
if !errors.Is(manifestErr, os.ErrNotExist) {
return operation.Resolution{}, nil
}
intent, intentErr := store.LoadSourceIntent(migrationID)
if errors.Is(intentErr, os.ErrNotExist) {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "peer_migration_prepare_not_applied",
ErrorMessage: "source intent was not persisted before the mutation boundary"}, nil
}
if intentErr != nil || !strings.EqualFold(intent.SourceUUID, record.ResourceID) {
return operation.Resolution{}, nil
}
autostart, autoErr := domain.GetAutostart()
active, activeErr := domain.IsActive()
if autoErr != nil || activeErr != nil {
return operation.Resolution{}, errors.Join(autoErr, activeErr)
}
if autostart == intent.SourceAutostartEnabled && active == intent.SourceWasRunning {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "peer_migration_prepare_not_applied",
ErrorMessage: "source state is restored and no ready manifest exists"}, nil
}
return operation.Resolution{}, nil
}
func (l *Libvirt) reconcileReceivedPeerMigration(ctx context.Context,
record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
migrationID, ok := peerMigrationActionID(record.Action, "receive-peer-migration")
if !ok {
return operation.Resolution{}, nil
}
store, err := peermigration.NewStore(
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
l.local.PeerMigrationStagingPath,
)
if err != nil {
return operation.Resolution{}, err
}
receipt, err := store.LoadTargetReceipt(migrationID)
if err != nil || !strings.EqualFold(receipt.TargetUUID, record.ResourceID) {
return operation.Resolution{}, nil
}
autostart, autoErr := domain.GetAutostart()
active, activeErr := domain.IsActive()
if autoErr != nil || activeErr != nil || autostart || active {
return operation.Resolution{}, errors.Join(autoErr, activeErr)
}
vm, complete, err := l.reconcileStagedPeerMigrationVM(ctx, record, domain, receipt)
if err != nil || !complete {
return operation.Resolution{}, err
}
result, err := json.Marshal(ReceivePeerMigrationResult{
VMMetadata: vm, MigrationID: migrationID,
ManifestDigest: receipt.ManifestDigest,
Artifacts: receipt.Artifacts, StagedPoweredOff: true,
})
if err != nil {
return operation.Resolution{}, err
}
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, nil
}
func (l *Libvirt) reconcileStagedPeerMigrationVM(ctx context.Context,
record operation.Record, domain *libvirt.Domain,
receipt peermigration.TargetReceipt) (VMMetadata, bool, error) {
document, definition, metadata, pool, root, err := l.managedRootContext(domain)
if err != nil {
return VMMetadata{}, false, nil
}
defer pool.Free()
defer root.Free()
autostart, autostartErr := domain.GetAutostart()
active, activeErr := domain.IsActive()
state, _, stateErr := domain.GetState()
if autostartErr != nil || activeErr != nil || stateErr != nil {
return VMMetadata{}, false, errors.Join(autostartErr, activeErr, stateErr)
}
if autostart || active || state != libvirt.DOMAIN_SHUTOFF {
return VMMetadata{}, false, nil
}
rootInfo, err := validateStandaloneQCOW2Volume(root, 0)
if err != nil {
return VMMetadata{}, false, nil
}
var rootArtifact *peermigration.Artifact
snapshotArtifacts := make(map[string]peermigration.Artifact)
for index := range receipt.Artifacts {
artifact := &receipt.Artifacts[index]
if artifact.Role == "root" {
rootArtifact = artifact
continue
}
snapshotArtifacts[artifact.SnapshotKey] = *artifact
}
if rootArtifact == nil || rootInfo.Capacity < rootArtifact.VirtualSizeBytes ||
len(metadata.Snapshots) != len(snapshotArtifacts) {
return VMMetadata{}, false, nil
}
for _, stored := range metadata.Snapshots {
artifact, ok := snapshotArtifacts[stored.Key]
if !ok || stored.Name != artifact.Name || stored.Consistency != artifact.Consistency ||
stored.SizeBytes != artifact.VirtualSizeBytes || stored.CreatedAt != artifact.CreatedAt {
return VMMetadata{}, false, nil
}
volume, lookupErr := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged),
snapshotVolumeName(definition.UUID, stored.Key),
)
if lookupErr != nil {
return VMMetadata{}, false, lookupErr
}
if volume == nil {
return VMMetadata{}, false, nil
}
_, inspectErr := validateStandaloneQCOW2Volume(volume, artifact.VirtualSizeBytes)
volume.Free()
if inspectErr != nil {
return VMMetadata{}, false, nil
}
}
seed, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleSeed), seedVolumeName(definition.UUID),
)
if err != nil || seed == nil {
return VMMetadata{}, false, err
}
seedPath, seedErr := seed.GetPath()
seed.Free()
if seedErr != nil || definition.Devices == nil {
return VMMetadata{}, false, seedErr
}
seedAttached := false
filterName := networkFilterName(definition.UUID)
filterAttached := managedNetworkLayoutMatches(definition, metadata)
for _, disk := range definition.Devices.Disks {
if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sda" &&
disk.Source != nil && disk.Source.File != nil &&
disk.Source.File.File == seedPath && disk.ReadOnly != nil {
seedAttached = true
}
}
var policy PlatformNetworkPolicySnapshot
if !seedAttached || !filterAttached ||
json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &policy) != nil ||
ValidatePlatformNetworkPolicy(policy) != nil ||
l.verifyManagedFilter(filterName, policy) != nil {
return VMMetadata{}, false, nil
}
if record.ExpectedState != nil && record.ExpectedState.Kind == operation.ExpectedCreateLikeNetwork {
seedContents, seedContentsErr := l.managedSeedContents(ctx, record.ResourceID)
if seedContentsErr != nil {
_, evidenceErr := expectedEvidenceError("create-like network", seedContentsErr)
return VMMetadata{}, false, evidenceErr
}
if !createLikeNetworkEvidenceMatches(
record, definition, metadata, seedContents,
) {
return VMMetadata{}, false, nil
}
}
vm, err := metadataFromXML(document)
if err != nil {
return VMMetadata{}, false, nil
}
vm.Status = VMStateShutoff
return vm, true, nil
}
func inspectActiveDeleteResolution(record operation.Record, active bool) (operation.Resolution, bool) {
if !active {
return operation.Resolution{}, false
}
if record.Action == "delete-vm" && operation.RequiresExplicitForceDelete(record) {
// Preserve an authoritative force_required outcome written by the
// original graceful DeleteVM call after its bounded wait.
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "force_required", ErrorMessage: "guest remains active and requires an explicitly authorized force delete"}, true
}
if record.Action == "delete-vm" &&
(record.CauseCode == "graceful_shutdown_ambiguous" ||
recoveredGracefulAttemptRequiresRetry(record)) {
// The prior ShutdownFlags response was lost and activity could not be
// observed conclusively, or the Agent restarted at the dispatch
// boundary before it persisted any cause. A later active observation
// can classify the already-terminal operation as failed, but cannot grant
// force authorization or dispatch another mutation.
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "graceful_shutdown_unconfirmed",
ErrorMessage: "graceful shutdown result was not confirmed; the resource remains blocked for manual remediation"}, true
}
if record.Action == "force-delete-vm" && forceDeleteCanBeConfirmedForRetry(record) {
// Never replay DestroyFlags automatically. Once no native call can still
// hold Journal admission, a token-confirmed administrator inspection may
// close an old force attempt which still observes an active Domain. The
// original graceful force_required record remains the authorization
// anchor, so a retry still needs a new operation ID and data-loss ack.
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "force_delete_unconfirmed_active",
ErrorMessage: "force delete did not reach an inactive domain; inspect and explicitly authorize a new force operation"}, true
}
// A force operation may already have crossed Destroy before its result was
// lost. Read-only inspection must not turn that ambiguity into a cleanup
// plan which would replay Destroy.
return operation.Resolution{}, true
}
func forceDeleteCanBeConfirmedForRetry(record operation.Record) bool {
return record.State == operation.StateFailed && record.EffectMayHaveApplied
}
func recoveredGracefulAttemptRequiresRetry(record operation.Record) bool {
if record.CauseCode != "" {
return false
}
return record.State == operation.StateFailed && record.EffectMayHaveApplied
}
func inspectAbsentDeleteResolution(cleanup operationCleanupInspection, forced bool) operation.Resolution {
if len(cleanup.steps) != 0 {
return operation.Resolution{}
}
return resolvedDeleteSuccess(DeleteVMResult{
Deleted: true, ManagedArtifactsAbsent: true, Forced: forced,
})
}
func (l *Libvirt) inspectDeleteCleanupSteps(ctx context.Context, uuid string,
domain *libvirt.Domain, managed *domainMetadataXML) (operationCleanupInspection, error) {
steps := make([]string, 0, 7)
evidence := make([]string, 0, 8)
if domain != nil {
jobEvidence, safe := inspectCleanupJobEvidence(domain)
if !safe {
return operationCleanupInspection{}, nil
}
evidence = append(evidence, jobEvidence...)
deleteEvidence, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain)
if err != nil {
return operationCleanupInspection{}, err
}
evidence = append(evidence, deleteEvidence...)
active, err := domain.IsActive()
if err != nil {
return operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
}
if active {
return operationCleanupInspection{}, nil
}
evidence = append(evidence, "domain:"+uuid+":inactive")
steps = append(steps, CleanupStepUndefineDomain)
}
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
if err != nil {
return operationCleanupInspection{}, translateLibvirtError("inspect managed storage pool", err)
}
inventory, err := pool.ListStorageVolumes()
if err != nil {
pool.Free()
return operationCleanupInspection{}, translateLibvirtError("inspect managed volumes", err)
}
names, err := collectVMManagedVolumeNames(uuid, managed, inventory)
if err != nil {
pool.Free()
return operationCleanupInspection{}, err
}
managedVolumesExist := false
for _, name := range names {
if err := ctx.Err(); err != nil {
pool.Free()
return operationCleanupInspection{}, err
}
volume, lookupErr := pool.LookupStorageVolByName(name)
if isNoStorageVolume(lookupErr) {
continue
}
if lookupErr != nil {
pool.Free()
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", lookupErr)
}
managedVolumesExist = true
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
volume.Free()
pool.Free()
return operationCleanupInspection{}, validateErr
}
volumeDigest, digestErr := storageVolumeInspectionDigest(volume)
if digestErr != nil {
volume.Free()
pool.Free()
return operationCleanupInspection{}, digestErr
}
path, pathErr := volume.GetPath()
volume.Free()
if pathErr != nil {
pool.Free()
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", pathErr)
}
var referenced bool
if domain != nil {
referenced, err = l.otherDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), name, uuid,
)
} else {
referenced, err = l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), name,
)
}
if err != nil {
pool.Free()
return operationCleanupInspection{}, err
}
if referenced {
pool.Free()
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed cleanup volume is referenced by another domain", nil)
}
evidence = append(evidence, "managed-volume:"+name+":"+volumeDigest)
}
pool.Free()
if managedVolumesExist {
steps = append(steps, CleanupStepDeleteManagedVolumes)
}
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
if err != nil {
return operationCleanupInspection{}, err
}
if seed != nil {
seedDigest, digestErr := storageVolumeInspectionDigest(seed)
if digestErr != nil {
seed.Free()
return operationCleanupInspection{}, digestErr
}
path, pathErr := seed.GetPath()
seed.Free()
if pathErr != nil {
return operationCleanupInspection{}, translateLibvirtError("inspect managed seed cleanup volume", pathErr)
}
referenced, referenceErr := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
)
if domain != nil {
referenced, referenceErr = l.otherDomainUsesVolume(
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid,
)
}
if referenceErr != nil {
return operationCleanupInspection{}, referenceErr
}
if referenced {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed seed cleanup volume is referenced by another domain", nil)
}
steps = append(steps, CleanupStepDeleteSeedVolume)
evidence = append(evidence, "seed-volume:"+seedVolumeName(uuid)+":"+seedDigest)
}
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
if err != nil {
return operationCleanupInspection{}, err
}
if filter != nil {
document, xmlErr := filter.GetXMLDesc(0)
filter.Free()
if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_invalid", "network filter cannot be confirmed as panel managed", xmlErr)
}
excludedUUID := ""
if domain != nil {
excludedUUID = uuid
}
referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), excludedUUID)
if referenceErr != nil {
return operationCleanupInspection{}, referenceErr
}
if referenced {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil)
}
steps = append(steps, CleanupStepUndefineNetworkFilter)
digest := sha256.Sum256([]byte(document))
evidence = append(evidence, "network-filter:"+networkFilterName(uuid)+":"+hex.EncodeToString(digest[:]))
}
nvramExists := false
for _, name := range managedNVRAMArtifactNames(uuid) {
path := filepath.Join(l.local.NVRAMDirectory, name)
info, statErr := os.Lstat(path)
if errors.Is(statErr, os.ErrNotExist) {
continue
}
if statErr != nil {
return operationCleanupInspection{}, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr)
}
if !info.Mode().IsRegular() {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil)
}
referenced, referenceErr := l.anyDomainUsesPath(path)
if domain != nil {
referenced, referenceErr = l.otherDomainUsesPath(path, uuid)
}
if referenceErr != nil {
return operationCleanupInspection{}, referenceErr
}
if referenced {
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil)
}
nvramExists = true
evidence = append(evidence, "nvram:"+name+":"+strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10))
}
if nvramExists {
steps = append(steps, CleanupStepDeleteNVRAM)
}
trafficPresent := l.traffic.contains(uuid)
if trafficPresent || len(steps) != 0 {
// cleanupDeletedVMArtifacts durably rewrites traffic state even if an
// earlier failed fsync left only an on-disk entry. Make that mutation
// explicit in every delete cleanup plan.
steps = append(steps, CleanupStepRemoveTrafficState)
if trafficPresent {
evidence = append(evidence, "traffic-state:"+uuid+":present")
} else {
evidence = append(evidence, "traffic-state:"+uuid+":rewrite")
}
}
return newOperationCleanupInspection(steps, evidence...), nil
}
func managedFilterDocument(document, expectedName string) bool {
var filter libvirtxml.NWFilter
if filter.Unmarshal(document) != nil || filter.Name != expectedName || filter.Chain != "root" {
return false
}
return managedFilterBaselineValid(&filter)
}
func storageVolumeInspectionDigest(volume *libvirt.StorageVol) (string, error) {
document, err := volume.GetXMLDesc(0)
if err != nil {
return "", translateLibvirtError("read cleanup volume XML", err)
}
digest := sha256.Sum256([]byte(document))
return hex.EncodeToString(digest[:]), nil
}
func (l *Libvirt) inspectReinstallOperation(record operation.Record,
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
replacementName := reinstallVolumeName(record.ResourceID, record.OperationID)
replacement, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
replacementExists := replacement != nil
if replacement != nil {
replacement.Free()
}
if diskTransitionCommitted(metadata, replacementName, "reinstall", record.OperationID) && replacementExists {
state, _, stateErr := domain.GetState()
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "reinstall", record.OperationID) {
return operation.Resolution{}, operationCleanupInspection{}
}
if metadata.PendingCleanupVolume != "" {
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
}
result, parseErr := metadataFromXML(document)
if parseErr != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
result.Status = mapDomainState(state)
result.ReinstallCommitted = true
result.CleanupPending = false
encoded, _ := json.Marshal(result)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
}
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "reinstall_not_committed", ErrorMessage: "host evidence confirms the reinstall did not switch the root volume"}, operationCleanupInspection{}
}
return operation.Resolution{}, operationCleanupInspection{}
}
func (l *Libvirt) inspectSnapshotOperation(ctx context.Context, record operation.Record,
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
parts := strings.Split(record.Action, ":")
if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) {
return operation.Resolution{}, operationCleanupInspection{}
}
key := parts[1]
document, metadata, err := requireManagedDomain(domain)
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
var stored *domainSnapshotMetadataXML
for index := range metadata.Snapshots {
if metadata.Snapshots[index].Key == key {
stored = &metadata.Snapshots[index]
break
}
}
volumeName := snapshotVolumeName(record.ResourceID, key)
var volume *libvirt.StorageVol
legacyBackupVolume := false
if parts[0] == "create-backup" || parts[0] == "cleanup-backup-workspace" {
volumeName, volume, legacyBackupVolume, err = l.backupWorkspaceResidue(
record.ResourceID, key,
)
} else {
volume, err = l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), volumeName,
)
}
if err != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
volumeExists := volume != nil
if volume != nil {
defer volume.Free()
}
switch parts[0] {
case "create-backup":
if stored != nil && volume != nil &&
volumeName != snapshotVolumeName(record.ResourceID, key) {
return operation.Resolution{}, operationCleanupInspection{}
}
if !backupResidueCleanupAllowed(record, stored) {
return operation.Resolution{}, operationCleanupInspection{}
}
return operation.Resolution{}, l.backupResidueCleanupInspection(
domain, volume, volumeName, key, stored,
)
case "cleanup-backup-workspace":
if len(parts) != 2 {
return operation.Resolution{}, operationCleanupInspection{}
}
if stored != nil && volume != nil &&
volumeName != snapshotVolumeName(record.ResourceID, key) {
return operation.Resolution{}, operationCleanupInspection{}
}
if stored == nil && !volumeExists {
result, _ := json.Marshal(CleanupBackupWorkspaceResult{
BackupKey: key, Cleaned: true,
WorkspaceAbsent: true, MetadataAbsent: true,
})
return operation.Resolution{
Resolved: true, Succeeded: true, Result: result,
}, operationCleanupInspection{}
}
if (stored == nil && volumeExists && !legacyBackupVolume) ||
(stored != nil && volumeExists && legacyBackupVolume) {
return operation.Resolution{
Resolved: true, Succeeded: false,
ErrorCode: "backup_workspace_cleanup_not_applied",
ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied",
}, operationCleanupInspection{}
}
return operation.Resolution{}, operationCleanupInspection{}
case "create-snapshot":
if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") {
return operation.Resolution{}, operationCleanupInspection{}
}
if stored != nil && volumeExists && snapshotConsistencyMatchesRequest(parts[2], stored.Consistency) {
info, validateErr := validateQCOW2Volume(volume, stored.SizeBytes)
if validateErr != nil || info.Capacity != stored.SizeBytes {
return operation.Resolution{}, operationCleanupInspection{}
}
result := Snapshot{ID: key, SnapshotKey: key, Name: stored.Name,
Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(),
State: "ready", Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, Created: true}
encoded, _ := json.Marshal(result)
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
}
if stored == nil && !volumeExists {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "snapshot_not_created", ErrorMessage: "host evidence confirms the snapshot is absent"}, operationCleanupInspection{}
}
if stored == nil && volumeExists {
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
domain, volume, snapshotVolumeName(record.ResourceID, key),
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
)
}
if stored != nil && !volumeExists {
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
)
}
case "delete-snapshot":
if len(parts) != 2 {
return operation.Resolution{}, operationCleanupInspection{}
}
if stored == nil && !volumeExists {
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
}
if stored != nil && volumeExists {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "snapshot_not_deleted", ErrorMessage: "host evidence confirms the snapshot remains present"}, operationCleanupInspection{}
}
if stored != nil && !volumeExists {
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
)
}
if stored == nil && volumeExists {
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
domain, volume, snapshotVolumeName(record.ResourceID, key),
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
)
}
case "restore-snapshot":
if len(parts) != 2 || stored == nil || !volumeExists {
return operation.Resolution{}, operationCleanupInspection{}
}
replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID)
replacement, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
if lookupErr != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
replacementExists := replacement != nil
if replacement != nil {
replacement.Free()
}
if diskTransitionCommitted(metadata, replacementName, "snapshot_restore", record.OperationID) && replacementExists {
state, _, stateErr := domain.GetState()
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "snapshot_restore", record.OperationID) {
return operation.Resolution{}, operationCleanupInspection{}
}
if metadata.PendingCleanupVolume != "" {
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
}
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Restored: true,
PowerState: mapDomainState(state), ActiveVolumeName: replacementName, CleanupPending: false})
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
}
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "snapshot_not_restored", ErrorMessage: "host evidence confirms the root volume was not switched"}, operationCleanupInspection{}
}
case "recover-backup":
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) ||
!sha256Pattern.MatchString(parts[2]) {
return operation.Resolution{}, operationCleanupInspection{}
}
key, sourceSHA256 := parts[1], parts[2]
replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID)
replacement, lookupErr := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleManaged), replacementName)
if lookupErr != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
replacementExists := replacement != nil
if replacement != nil {
replacement.Free()
}
seedContents, seedErr := l.managedSeedContents(ctx, record.ResourceID)
if seedErr != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
seedOperation, seedErr := seedMetadataMarker(
seedContents, backupRecoveryOperationMetadata,
)
if seedErr != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
seedCommitted := seedOperation == record.OperationID
diskCommitted := diskTransitionCommitted(metadata, replacementName,
"backup_recover", record.OperationID)
evidence := classifyBackupRecoveryEvidence(
replacementExists, diskCommitted,
metadata.DiskTransitionID == record.OperationID, seedCommitted,
)
if evidence == backupRecoveryEvidenceApplied {
state, _, stateErr := domain.GetState()
if stateErr != nil || !diskTransitionPowerStateMatches(
metadata, state, "backup_recover", record.OperationID) {
return operation.Resolution{}, operationCleanupInspection{}
}
if metadata.PendingCleanupVolume != "" {
return operation.Resolution{},
l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
}
result, _ := json.Marshal(RecoverBackupResult{
BackupKey: key, SHA256: sourceSHA256, Restored: true,
PowerState: mapDomainState(state), ActiveVolumeName: replacementName,
})
return operation.Resolution{Resolved: true, Succeeded: true,
Result: result}, operationCleanupInspection{}
}
if evidence == backupRecoveryEvidenceNotApplied {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "backup_not_recovered",
ErrorMessage: "host evidence confirms the root volume was not switched"},
operationCleanupInspection{}
}
}
return operation.Resolution{}, operationCleanupInspection{}
}
type backupRecoveryEvidence uint8
const (
backupRecoveryEvidenceUnknown backupRecoveryEvidence = iota
backupRecoveryEvidenceApplied
backupRecoveryEvidenceNotApplied
)
// classifyBackupRecoveryEvidence is deliberately fail-closed. Recovery is
// applied only when both the root-disk transition and the seed transition are
// visible. Any partial evidence remains unresolved and fenced.
func classifyBackupRecoveryEvidence(replacementExists, diskCommitted,
diskTransitionClaimed, seedCommitted bool) backupRecoveryEvidence {
if replacementExists && diskCommitted && seedCommitted {
return backupRecoveryEvidenceApplied
}
if !replacementExists && !diskTransitionClaimed && !seedCommitted {
return backupRecoveryEvidenceNotApplied
}
return backupRecoveryEvidenceUnknown
}
func backupResidueCleanupAllowed(record operation.Record, stored *domainSnapshotMetadataXML) bool {
parts := strings.Split(record.Action, ":")
if len(parts) != 3 || parts[0] != "create-backup" ||
!catalogIDPattern.MatchString(parts[1]) ||
(parts[2] != "filesystem" && parts[2] != "crash") {
return false
}
if record.State == operation.StateSucceeded {
if !successfulBackupCleanupPending(record) {
return false
}
} else if record.State != operation.StateFailed || !record.EffectMayHaveApplied || record.ExecutionFenced {
return false
}
return stored == nil || (stored.Key == parts[1] && stored.SizeBytes > 0 &&
snapshotConsistencyMatchesRequest(parts[2], stored.Consistency))
}
func successfulBackupCleanupPending(record operation.Record) bool {
if record.State != operation.StateSucceeded || !recordHasCleanupPending(record) ||
len(operationEffectProjection(record, record.Result)) == 0 {
return false
}
return strings.HasPrefix(record.Action, "create-backup:")
}
// backupResidueCleanupInspection exposes only an evidence-bound cleanup plan
// for a local immutable copy left by an already-terminal backup worker. It
// never asserts whether the remote multipart object committed: failed records
// therefore remain partial, while successful records retain their immutable
// applied object projection through appliedBackupInspectionWithCleanup.
func (l *Libvirt) backupResidueCleanupInspection(domain cleanupJobDomain,
volume *libvirt.StorageVol, volumeName, key string,
stored *domainSnapshotMetadataXML) operationCleanupInspection {
if volume == nil && stored == nil {
return operationCleanupInspection{}
}
if volume != nil {
expectedCapacity := uint64(0)
if stored != nil {
expectedCapacity = stored.SizeBytes
}
if _, err := validateStandaloneQCOW2Volume(volume, expectedCapacity); err != nil {
return operationCleanupInspection{}
}
plan := l.snapshotVolumeCleanupInspection(
domain, volume, volumeName, CleanupStepDeleteSnapshotVolume,
"backup-volume:"+volumeName,
)
if len(plan.steps) == 0 {
return operationCleanupInspection{}
}
if stored != nil {
plan.steps = append(plan.steps, CleanupStepRemoveSnapshotMetadata)
plan.evidence = append(plan.evidence, "backup-metadata:"+key+":"+
stored.Consistency+":"+strconv.FormatUint(stored.SizeBytes, 10))
sort.Strings(plan.evidence)
}
return plan
}
return cleanupInspectionWithIdleDomain(
domain, []string{CleanupStepRemoveSnapshotMetadata},
"backup-metadata:"+key+":"+stored.Consistency+":"+
strconv.FormatUint(stored.SizeBytes, 10),
)
}
func (l *Libvirt) absentDomainBackupWorkspaceCleanupInspection(
volume *libvirt.StorageVol, volumeName string,
) operationCleanupInspection {
if volume == nil {
return operationCleanupInspection{}
}
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
return operationCleanupInspection{}
}
digest, err := storageVolumeInspectionDigest(volume)
if err != nil {
return operationCleanupInspection{}
}
path, err := volume.GetPath()
if err != nil {
return operationCleanupInspection{}
}
referenced, err := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), volumeName,
)
if err != nil || referenced {
return operationCleanupInspection{}
}
return newOperationCleanupInspection(
[]string{CleanupStepDeleteSnapshotVolume},
"domain-state:absent", "backup-volume:"+volumeName+":"+digest,
)
}
func diskTransitionPowerStateMatches(metadata *domainMetadataXML, state libvirt.DomainState,
kind, operationID string) bool {
if !diskTransitionMarkerMatches(metadata, kind, operationID) {
return true
}
if metadata.DiskTransitionRunning {
return state == libvirt.DOMAIN_RUNNING || state == libvirt.DOMAIN_BLOCKED
}
return state == libvirt.DOMAIN_SHUTOFF || state == libvirt.DOMAIN_SHUTDOWN
}
func (l *Libvirt) inspectDiskTransitionCleanupSteps(domain *libvirt.Domain, document string,
metadata *domainMetadataXML) operationCleanupInspection {
pending := metadata.PendingCleanupVolume
if pending == "" || !isVMOwnedRootVolumeName(metadataUUID(document), pending) ||
pending == metadata.RootVolumeName {
return operationCleanupInspection{}
}
jobEvidence, safe := inspectCleanupJobEvidence(domain)
if !safe {
return operationCleanupInspection{}
}
steps := make([]string, 0, 2)
evidence := append([]string(nil), jobEvidence...)
evidence = append(evidence, "pending-root:"+pending, "active-root:"+metadata.RootVolumeName)
old, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), pending)
if err != nil {
return operationCleanupInspection{}
}
if old != nil {
if _, validateErr := validateQCOW2Volume(old, 0); validateErr != nil {
old.Free()
return operationCleanupInspection{}
}
oldDigest, digestErr := storageVolumeInspectionDigest(old)
if digestErr != nil {
old.Free()
return operationCleanupInspection{}
}
path, pathErr := old.GetPath()
old.Free()
if pathErr != nil {
return operationCleanupInspection{}
}
used, referenceErr := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), pending,
)
if referenceErr != nil || used {
return operationCleanupInspection{}
}
steps = append(steps, CleanupStepDeletePreviousRoot)
evidence = append(evidence, "pending-root-state:present:"+oldDigest)
} else {
evidence = append(evidence, "pending-root-state:absent")
}
steps = append(steps, CleanupStepPersistCleanupMetadata)
return newOperationCleanupInspection(steps, evidence...)
}
type cleanupJobDomain interface {
IsActive() (bool, error)
GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error)
GetXMLDesc(libvirt.DomainXMLFlags) (string, error)
GetBlockJobInfo(string, libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error)
}
func inspectCleanupJobEvidence(domain cleanupJobDomain) ([]string, bool) {
active, err := domain.IsActive()
if err != nil {
return nil, false
}
flags := libvirt.DOMAIN_XML_INACTIVE
evidence := []string{"domain-runtime:inactive"}
if active {
flags = 0
job, jobErr := domain.GetJobStats(0)
if jobErr != nil || job == nil || job.Type != libvirt.DOMAIN_JOB_NONE {
return nil, false
}
evidence[0] = "domain-runtime:active:job-none"
}
document, err := domain.GetXMLDesc(flags)
if err != nil {
return nil, false
}
documentDigest := sha256.Sum256([]byte(document))
evidence = append(evidence, "domain-xml:"+hex.EncodeToString(documentDigest[:]))
if active {
persistentDocument, persistentErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if persistentErr != nil {
return nil, false
}
persistentDigest := sha256.Sum256([]byte(persistentDocument))
evidence = append(evidence, "persistent-domain-xml:"+hex.EncodeToString(persistentDigest[:]))
}
targets, err := cleanupDiskTargets(document)
if err != nil {
return nil, false
}
for _, target := range targets {
evidence = append(evidence, "disk-target:"+target)
if !active {
continue
}
info, infoErr := domain.GetBlockJobInfo(target, 0)
if infoErr != nil || !blockJobInfoAbsent(info) {
return nil, false
}
}
return evidence, true
}
func cleanupDiskTargets(document string) ([]string, error) {
var definition libvirtxml.Domain
if err := definition.Unmarshal(document); err != nil || definition.Devices == nil {
return nil, errors.New("domain disk evidence is invalid")
}
seen := make(map[string]struct{}, len(definition.Devices.Disks))
for _, disk := range definition.Devices.Disks {
if disk.Target == nil || disk.Target.Dev == "" {
return nil, errors.New("domain disk target evidence is incomplete")
}
seen[disk.Target.Dev] = struct{}{}
}
targets := make([]string, 0, len(seen))
for target := range seen {
targets = append(targets, target)
}
sort.Strings(targets)
return targets, nil
}
func blockJobInfoAbsent(info *libvirt.DomainBlockJobInfo) bool {
// The Go binding does not expose virDomainGetBlockJobInfo's 0/1 return
// value. A 0 (no job) is represented by the zero-valued info struct.
return info != nil && info.Type == libvirt.DOMAIN_BLOCK_JOB_TYPE_UNKNOWN &&
info.Bandwidth == 0 && info.Cur == 0 && info.End == 0
}
func cleanupInspectionWithIdleDomain(domain cleanupJobDomain, steps []string,
evidence ...string) operationCleanupInspection {
jobEvidence, safe := inspectCleanupJobEvidence(domain)
if !safe {
return operationCleanupInspection{}
}
evidence = append(evidence, jobEvidence...)
return newOperationCleanupInspection(steps, evidence...)
}
func ensureNoCleanupBlockingJobs(domain cleanupJobDomain) error {
if _, safe := inspectCleanupJobEvidence(domain); !safe {
return NewError(ErrConflict, "cleanup_job_not_idle", "domain cleanup is blocked by active or unreadable job state", nil)
}
return nil
}
func (l *Libvirt) snapshotVolumeCleanupInspection(domain cleanupJobDomain, volume *libvirt.StorageVol,
volumeName, step, targetEvidence string) operationCleanupInspection {
if volume == nil {
return operationCleanupInspection{}
}
jobEvidence, safe := inspectCleanupJobEvidence(domain)
if !safe {
return operationCleanupInspection{}
}
if _, err := validateQCOW2Volume(volume, 0); err != nil {
return operationCleanupInspection{}
}
volumeDigest, err := storageVolumeInspectionDigest(volume)
if err != nil {
return operationCleanupInspection{}
}
path, err := volume.GetPath()
if err != nil {
return operationCleanupInspection{}
}
referenced, err := l.anyDomainUsesVolume(
path, l.storagePoolName(StorageRoleManaged), volumeName,
)
if err != nil || referenced {
return operationCleanupInspection{}
}
evidence := append([]string{targetEvidence + ":" + volumeDigest}, jobEvidence...)
return newOperationCleanupInspection([]string{step}, evidence...)
}
func (l *Libvirt) inspectMatchingBackupJob(record operation.Record, domain *libvirt.Domain) OperationAbortInspection {
if record.State != operation.StateDispatching &&
(record.State != operation.StateFailed || !record.EffectMayHaveApplied) {
return OperationAbortInspection{}
}
parts := strings.Split(record.Action, ":")
if len(parts) != 3 || (parts[0] != "create-snapshot" && parts[0] != "create-backup") ||
!catalogIDPattern.MatchString(parts[1]) || (parts[2] != "filesystem" && parts[2] != "crash") {
return OperationAbortInspection{}
}
job, err := domain.GetJobStats(0)
if err != nil || job == nil ||
(job.Type != libvirt.DOMAIN_JOB_BOUNDED && job.Type != libvirt.DOMAIN_JOB_UNBOUNDED) ||
!job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
return OperationAbortInspection{}
}
backupXML, err := domain.BackupGetXMLDesc(0)
if err != nil {
return OperationAbortInspection{}
}
_, volume, _, err := l.backupWorkspaceResidue(record.ResourceID, parts[1])
if err != nil || volume == nil {
return OperationAbortInspection{}
}
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
volume.Free()
return OperationAbortInspection{}
}
targetPath, pathErr := volume.GetPath()
volume.Free()
if pathErr != nil || !backupPushMatchesTarget(backupXML, targetPath) {
return OperationAbortInspection{}
}
return OperationAbortInspection{
Available: true, Kind: "backup",
InspectionToken: operationInspectionToken(record, "backup", backupXML),
}
}
func backupPushMatchesTarget(backupXML, targetPath string) bool {
var backup libvirtxml.DomainBackup
if backup.Unmarshal(backupXML) != nil || backup.Pull != nil || backup.Push == nil ||
backup.Push.Disks == nil || len(backup.Push.Disks.Disks) != 1 {
return false
}
disk := backup.Push.Disks.Disks[0]
return disk.Name == "vda" && disk.Backup == "yes" && disk.Target != nil &&
disk.Target.File != nil && filepath.Clean(disk.Target.File.File) == filepath.Clean(targetPath)
}
func buildOperationInspection(record operation.Record, resolution operation.Resolution,
abort OperationAbortInspection, cleanup operationCleanupInspection) OperationInspection {
result := OperationInspection{
OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID,
RequestFingerprint: record.Fingerprint, OperationStatus: record.State,
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
SummaryCode: InspectionSummaryIncomplete,
EffectResolution: InspectionEffectInconclusive,
AbortJob: abort, Cleanup: OperationCleanupPlan{},
}
if abort.Available {
result.SummaryCode = InspectionSummaryAbortableJob
return result
}
if len(cleanup.steps) != 0 && len(cleanup.evidence) != 0 {
result.SummaryCode = InspectionSummaryCleanupRequired
result.EffectResolution = InspectionEffectPartial
result.Cleanup = OperationCleanupPlan{
Available: true,
PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()),
Steps: append([]string(nil), cleanup.steps...),
}
return result
}
if !resolution.Resolved {
return result
}
if record.Action == "resize-vm" && !resolution.Succeeded && resolution.ErrorCode == "resources_partially_resized" {
result.ErrorCode = resolution.ErrorCode
result.EffectResolution = InspectionEffectPartial
return result
}
if resolution.Succeeded {
result.Outcome = InspectionOutcomeSucceeded
result.SummaryCode = InspectionSummarySucceeded
projection := operationEffectProjection(record, resolution.Result)
result.EffectResolution = inspectedEffectResolution(record, resolution, projection)
if result.EffectResolution == InspectionEffectApplied {
result.EffectProjection = projection
} else {
result.Outcome = InspectionOutcomeUncertain
result.SummaryCode = InspectionSummaryIncomplete
}
} else {
result.Outcome = InspectionOutcomeFailed
result.SummaryCode = InspectionSummaryFailed
projection := failedOperationEffectProjection(record, resolution)
result.EffectResolution = inspectedEffectResolution(record, resolution, projection)
if result.EffectResolution == InspectionEffectNotApplied {
result.EffectProjection = projection
} else {
result.Outcome = InspectionOutcomeUncertain
result.SummaryCode = InspectionSummaryIncomplete
}
if inspectionErrorCodePattern.MatchString(resolution.ErrorCode) {
result.ErrorCode = resolution.ErrorCode
}
}
return result
}
// inspectedEffectResolution separates immutable operation evidence from a
// read-only observation that merely happens to match the requested target.
// Power, service, network-link, catalog and storage state may be changed out
// of band, so those observations remain inconclusive even when the target is
// currently reached. Only a Journal terminal record or a mutation-specific
// durable marker may authorize automated ledger convergence.
func inspectedEffectResolution(record operation.Record, resolution operation.Resolution,
projection json.RawMessage) string {
if record.State == operation.StateSucceeded {
return InspectionEffectApplied
}
if record.State == operation.StateFailed && !record.EffectMayHaveApplied {
return InspectionEffectNotApplied
}
if record.State != operation.StateFailed || !record.EffectMayHaveApplied || !resolution.Resolved {
return InspectionEffectInconclusive
}
if resolution.Succeeded {
if len(projection) == 0 {
return InspectionEffectInconclusive
}
switch {
case record.Action == "update-settings",
record.Action == "resize-vm",
record.Action == "change-network",
record.Action == "reinstall-vm",
record.Action == "apply-firewall",
strings.HasPrefix(record.Action, "apply-vpc-network:"),
strings.HasPrefix(record.Action, "remove-vpc-network:"),
record.Action == "apply-vpc-port",
record.Action == "remove-vpc-port",
strings.HasPrefix(record.Action, "restore-snapshot:"),
strings.HasPrefix(record.Action, "recover-backup:"):
return InspectionEffectApplied
default:
return InspectionEffectInconclusive
}
}
switch {
case operation.IsCreateLikeAction(record.Action) && resolution.ErrorCode == "creation_absent" && len(projection) != 0,
record.Action == "update-settings" && resolution.ErrorCode == "settings_not_committed",
record.Action == "change-network" && resolution.ErrorCode == "network_not_committed",
record.Action == "reinstall-vm" && resolution.ErrorCode == "reinstall_not_committed",
record.Action == "apply-firewall" && resolution.ErrorCode == "firewall_not_committed",
strings.HasPrefix(record.Action, "restore-snapshot:") && resolution.ErrorCode == "snapshot_not_restored",
strings.HasPrefix(record.Action, "recover-backup:") && resolution.ErrorCode == "backup_not_recovered":
return InspectionEffectNotApplied
default:
return InspectionEffectInconclusive
}
}
func inspectionFromRecord(record operation.Record) OperationInspection {
switch record.State {
case operation.StateSucceeded:
return buildOperationInspection(record, operation.Resolution{
Resolved: true, Succeeded: true, Result: append([]byte(nil), record.Result...),
}, OperationAbortInspection{}, operationCleanupInspection{})
case operation.StateFailed:
resolution := operation.Resolution{Resolved: true, Succeeded: false, Result: append([]byte(nil), record.Result...)}
if record.Error != nil {
resolution.ErrorCode = record.Error.Code
resolution.ErrorMessage = record.Error.Message
}
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{})
}
return inconclusiveInspectionFromRecord(record)
}
func inconclusiveInspectionFromRecord(record operation.Record) OperationInspection {
return OperationInspection{
OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID,
RequestFingerprint: record.Fingerprint, OperationStatus: record.State,
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
SummaryCode: InspectionSummaryIncomplete, EffectResolution: InspectionEffectInconclusive,
}
}
func failedOperationEffectProjection(record operation.Record, resolution operation.Resolution) json.RawMessage {
if !operation.IsCreateLikeAction(record.Action) || resolution.ErrorCode != "creation_absent" {
return nil
}
var evidence struct {
CreationAbsent bool `json:"creation_absent"`
ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"`
}
if !decodeOperationEffectResult(resolution.Result, &evidence) ||
!evidence.CreationAbsent || !evidence.ManagedArtifactsAbsent {
return nil
}
encoded, err := json.Marshal(evidence)
if err != nil {
return nil
}
return encoded
}
const maxOperationEffectProjectionBytes = 64 << 10
// operationEffectProjection never forwards Journal bytes. It first decodes a
// result into the exact response type allowed for the recorded action and then
// rebuilds a deliberately smaller projection. Unknown actions, malformed or
// oversized results, and identity mismatches simply omit the optional
// projection while keeping the independently derived effect resolution.
func operationEffectProjection(record operation.Record, raw []byte) json.RawMessage {
if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes || !json.Valid(raw) {
return nil
}
var projection any
switch {
case record.Action == "configure-storage":
var value StorageConfigurationResult
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = struct {
Configured bool `json:"configured"`
}{Configured: value.Configured}
case strings.HasPrefix(record.Action, "apply-vpc-network:") ||
strings.HasPrefix(record.Action, "remove-vpc-network:"):
var value vpcrouting.Result
expected := record.ExpectedState
bridge := ""
if expected != nil {
bridge, _ = vpcrouting.BridgeName(expected.VPCKey)
}
if !decodeOperationEffectResult(raw, &value) || expected == nil ||
expected.Kind != operation.ExpectedVPCNetwork || value.VPCKey != expected.VPCKey ||
value.DesiredRevision != expected.VPCRevision || value.DesiredDigest != expected.VPCDigest ||
(expected.VPCPresent && (!value.Applied || value.Removed || value.Bridge != bridge)) ||
(!expected.VPCPresent && (!value.Removed || value.Applied || value.Bridge != "")) {
return nil
}
projection = value
case record.Action == "apply-vpc-port" || record.Action == "remove-vpc-port":
var value VPCPortResult
expected := record.ExpectedState
if !decodeOperationEffectResult(raw, &value) || expected == nil ||
expected.Kind != operation.ExpectedVPCPort || value.UUID != record.ResourceID ||
value.PortKey != expected.VPCPortKey || value.VPCKey != expected.VPCKey ||
(expected.VPCPresent && (!value.Attached || value.Removed || value.VPCPort == nil ||
*value.VPCPort != expectedVPCPortMetadata(*expected))) ||
(!expected.VPCPresent && (!value.Removed || value.Attached || value.VPCPort != nil)) {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "create-backup:"):
parts := strings.Split(record.Action, ":")
var value BackupResult
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) ||
(parts[2] != "filesystem" && parts[2] != "crash") ||
!decodeOperationEffectResult(raw, &value) || value.BackupKey != parts[1] ||
value.SizeBytes <= 0 || value.VirtualSizeBytes == 0 ||
!lowerHexDigest(value.SHA256) ||
!snapshotConsistencyMatchesRequest(parts[2], value.Consistency) {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "prepare-peer-migration:"):
parts := strings.Split(record.Action, ":")
var value PreparePeerMigrationResult
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
!decodeOperationEffectResult(raw, &value) || value.MigrationID != parts[1] ||
!lowerHexDigest(value.ManifestDigest) || !value.SourceStopped ||
!value.SourceAutostartDisabled || len(value.Artifacts) < 1 ||
len(value.Artifacts) > peermigration.MaxArtifacts ||
!validPeerMigrationProjectionArtifacts(parts[1], record.ResourceID, value.Artifacts) {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "receive-peer-migration:"):
parts := strings.Split(record.Action, ":")
var value ReceivePeerMigrationResult
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
!decodeOperationEffectResult(raw, &value) || value.MigrationID != parts[1] ||
value.UUID != record.ResourceID || !lowerHexDigest(value.ManifestDigest) ||
!value.StagedPoweredOff || len(value.Artifacts) < 1 ||
len(value.Artifacts) > peermigration.MaxArtifacts ||
!validPeerMigrationProjectionArtifacts(parts[1], record.ResourceID, value.Artifacts) {
return nil
}
projection = struct {
operationVMEffectProjection
MigrationID string `json:"migration_id"`
ManifestDigest string `json:"manifest_digest"`
Artifacts []peermigration.Artifact `json:"artifacts"`
StagedPoweredOff bool `json:"staged_powered_off"`
}{
operationVMEffectProjection: vmEffectProjection(value.VMMetadata),
MigrationID: value.MigrationID, ManifestDigest: value.ManifestDigest,
Artifacts: value.Artifacts, StagedPoweredOff: value.StagedPoweredOff,
}
case strings.HasPrefix(record.Action, "restore-backup:"):
parts := strings.Split(record.Action, ":")
var value RestoreBackupResult
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
!decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID ||
value.BackupKey != parts[1] || !lowerHexDigest(value.SHA256) {
return nil
}
projection = struct {
operationVMEffectProjection
BackupKey string `json:"backup_key"`
SHA256 string `json:"sha256"`
}{
operationVMEffectProjection: vmEffectProjection(value.VMMetadata),
BackupKey: value.BackupKey, SHA256: value.SHA256,
}
case strings.HasPrefix(record.Action, "recover-backup:"):
parts := strings.Split(record.Action, ":")
var value RecoverBackupResult
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) ||
!sha256Pattern.MatchString(parts[2]) ||
!decodeOperationEffectResult(raw, &value) || !value.Restored ||
value.BackupKey != parts[1] || value.SHA256 != parts[2] {
return nil
}
projection = struct {
BackupKey string `json:"backup_key"`
SHA256 string `json:"sha256,omitempty"`
Restored bool `json:"restored"`
PowerState VMStateName `json:"power_state"`
}{value.BackupKey, value.SHA256, value.Restored, value.PowerState}
case operation.IsCreateLikeAction(record.Action) || record.Action == "update-settings" ||
record.Action == "change-network" || record.Action == "reinstall-vm":
var value VMMetadata
if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID {
return nil
}
projection = vmEffectProjection(value)
case record.Action == "resize-vm":
var value ResizeResourcesResult
expected := record.ExpectedState
if !decodeOperationEffectResult(raw, &value) || expected == nil || expected.Kind != operation.ExpectedResizeResources ||
expected.TransitionID != record.OperationID || value.UUID != record.ResourceID || !value.ResourcesResized ||
value.FromVCPUs != expected.FromVCPUs || value.TargetVCPUs != expected.TargetVCPUs ||
value.FromMemoryMiB != expected.FromMemoryMiB || value.TargetMemoryMiB != expected.TargetMemoryMiB ||
value.FromDiskGiB != expected.FromDiskGiB || value.TargetDiskGiB != expected.TargetDiskGiB ||
value.FromBandwidthMbps != expected.FromBandwidthMbps || value.TargetBandwidthMbps != expected.TargetBandwidthMbps ||
value.AppliedVCPUs != value.TargetVCPUs || value.AppliedMemoryMiB != value.TargetMemoryMiB ||
value.AppliedDiskGiB != value.TargetDiskGiB || value.AppliedBandwidthMbps != value.TargetBandwidthMbps ||
!value.PowerStateRestored ||
(value.OriginallyRunning && value.Status != VMStateRunning && value.Status != VMStateBlocked) ||
(!value.OriginallyRunning && value.Status != VMStateShutoff && value.Status != VMStateCrashed) {
return nil
}
projection = value
case record.Action == "delete-vm" || record.Action == "force-delete-vm":
var value DeleteVMResult
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"):
parts := strings.Split(record.Action, ":")
var value DeleteVMResult
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
!decodeOperationEffectResult(raw, &value) || !value.Deleted ||
!value.ManagedArtifactsAbsent || value.Forced {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "activate-peer-migration:") ||
strings.HasPrefix(record.Action, "rollback-peer-migration:"):
parts := strings.Split(record.Action, ":")
var value PeerMigrationStateResult
expected := record.ExpectedState
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
!decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID ||
expected == nil || expected.Kind != operation.ExpectedPeerMigrationState ||
value.Running != expected.StartIfSuspended ||
value.AutostartEnabled != expected.Enabled {
return nil
}
projection = struct {
UUID string `json:"uuid"`
State VMStateName `json:"state"`
Running bool `json:"running"`
AutostartEnabled bool `json:"autostart_enabled"`
}{value.UUID, value.State, value.Running, value.AutostartEnabled}
case strings.HasPrefix(record.Action, "power-") || strings.HasPrefix(record.Action, "service-"):
var value VMState
if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID {
return nil
}
projection = struct {
UUID string `json:"uuid"`
State VMStateName `json:"state"`
}{UUID: value.UUID, State: value.State}
case record.Action == "network-suspend" || record.Action == "network-resume":
var value struct {
NetworkEnabled bool `json:"network_enabled"`
}
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = value
case strings.HasPrefix(record.Action, "create-snapshot:"):
var value Snapshot
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = operationSnapshotEffectProjection{
ID: value.ID, SnapshotKey: value.SnapshotKey, Name: value.Name,
CreatedAt: value.CreatedAt, State: value.State, Consistency: value.Consistency,
SizeBytes: value.SizeBytes, Created: value.Created,
}
case strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:"):
var value SnapshotMutationResult
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = struct {
SnapshotKey string `json:"snapshot_key"`
Restored bool `json:"restored,omitempty"`
Deleted bool `json:"deleted,omitempty"`
PowerState VMStateName `json:"power_state,omitempty"`
CleanupPending bool `json:"cleanup_pending,omitempty"`
}{value.SnapshotKey, value.Restored, value.Deleted, value.PowerState, value.CleanupPending}
case record.Action == "apply-firewall":
var value FirewallPolicy
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = struct {
Revision uint64 `json:"revision"`
DefaultIn string `json:"default_in"`
DefaultOut string `json:"default_out"`
PolicyDigest string `json:"policy_digest,omitempty"`
Rules []FirewallRule `json:"rules"`
}{value.Revision, value.DefaultIn, value.DefaultOut, value.PolicyDigest, append([]FirewallRule(nil), value.Rules...)}
case record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
strings.HasPrefix(record.Action, "exit-rescue:"):
var value RescueState
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = struct {
Active bool `json:"active"`
Staged bool `json:"staged"`
PendingManualRestart bool `json:"pending_manual_restart,omitempty"`
TemplateID string `json:"template_id,omitempty"`
Revision uint64 `json:"revision,omitempty"`
GenerationToken string `json:"generation_token,omitempty"`
NetworkSuspended bool `json:"network_suspended"`
}{
value.Active,
value.Staged,
value.PendingManualRestart,
value.TemplateID,
value.Revision,
value.GenerationToken,
value.NetworkSuspended,
}
case record.Action == "import-template" || record.Action == "update-template":
var value ImageTemplate
if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "template:"+value.ID {
return nil
}
projection = operationCatalogImportProjection{
ID: value.ID, Name: value.Name, OSFamily: value.OSFamily, Version: value.Version,
Architecture: value.Architecture, SizeBytes: value.SizeBytes,
VirtualSizeBytes: value.VirtualSizeBytes, SHA256: value.SHA256,
Revision: value.Revision, GenerationToken: value.GenerationToken, CreatedAt: value.CreatedAt,
}
case record.Action == "import-rescue-template":
var value RescueTemplate
if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "rescue-template:"+value.ID {
return nil
}
projection = operationCatalogImportProjection{
ID: value.ID, Name: value.Name, SizeBytes: value.SizeBytes,
SHA256: value.SHA256, CreatedAt: value.CreatedAt,
}
case record.Action == "import-user-iso":
var value UserISO
if !decodeOperationEffectResult(raw, &value) ||
record.ResourceID != "user-iso:"+strconv.FormatInt(value.OwnerID, 10)+":"+value.ID {
return nil
}
projection = operationCatalogImportProjection{
ID: value.ID, OwnerID: value.OwnerID, Name: value.Name,
SizeBytes: value.SizeBytes, SHA256: value.SHA256, CreatedAt: value.CreatedAt,
}
case record.Action == "delete-template" || record.Action == "delete-rescue-template" ||
record.Action == "delete-user-iso":
var value CatalogDeleteResult
if !decodeOperationEffectResult(raw, &value) {
return nil
}
projection = value
default:
return nil
}
encoded, err := json.Marshal(projection)
if err != nil || len(encoded) == 0 || len(encoded) > maxOperationEffectProjectionBytes {
return nil
}
return encoded
}
func validPeerMigrationProjectionArtifacts(migrationID, uuid string,
artifacts []peermigration.Artifact) bool {
for _, artifact := range artifacts {
if artifact.Path != "" {
return false
}
}
return peermigration.ValidateManifest(peermigration.Manifest{
Version: peermigration.ProtocolVersion, MigrationID: migrationID,
SourceUUID: strings.ToLower(uuid), ExpiresAt: time.Now().Add(time.Hour).Unix(),
Artifacts: artifacts,
}, time.Now()) == nil
}
type operationVMEffectProjection struct {
UUID string `json:"uuid"`
Name string `json:"name,omitempty"`
Status VMStateName `json:"status,omitempty"`
VCPUs uint `json:"vcpu,omitempty"`
MemoryMiB uint64 `json:"memory_mib,omitempty"`
DiskGiB uint64 `json:"disk_gib,omitempty"`
BandwidthMbps uint64 `json:"bandwidth_mbps,omitempty"`
Hostname string `json:"hostname,omitempty"`
PublicNetwork publicnetwork.Intent `json:"public_network"`
TemplateID string `json:"template_id,omitempty"`
Machine string `json:"machine,omitempty"`
Firmware string `json:"firmware,omitempty"`
SecureBoot bool `json:"secure_boot,omitempty"`
SettingsChanged bool `json:"settings_changed,omitempty"`
NetworkChanged bool `json:"network_changed,omitempty"`
ReinstallCommitted bool `json:"reinstall_committed,omitempty"`
CleanupPending bool `json:"cleanup_pending,omitempty"`
}
func vmEffectProjection(value VMMetadata) operationVMEffectProjection {
return operationVMEffectProjection{
UUID: value.UUID, Name: value.Name, Status: value.Status,
VCPUs: value.VCPUs, MemoryMiB: value.MemoryMiB, DiskGiB: value.DiskGiB,
BandwidthMbps: value.BandwidthMbps, Hostname: value.Hostname,
PublicNetwork: value.PublicNetwork, TemplateID: value.TemplateID,
Machine: value.Machine, Firmware: value.Firmware, SecureBoot: value.SecureBoot,
SettingsChanged: value.SettingsChanged, NetworkChanged: value.NetworkChanged,
ReinstallCommitted: value.ReinstallCommitted, CleanupPending: value.CleanupPending,
}
}
type operationSnapshotEffectProjection struct {
ID string `json:"id"`
SnapshotKey string `json:"snapshot_key"`
Name string `json:"name,omitempty"`
CreatedAt time.Time `json:"created_at"`
State string `json:"state"`
Consistency string `json:"consistency"`
SizeBytes uint64 `json:"size_bytes,omitempty"`
Created bool `json:"created,omitempty"`
}
type operationCatalogImportProjection struct {
ID string `json:"id"`
OwnerID int64 `json:"owner_id,omitempty"`
Name string `json:"name,omitempty"`
OSFamily string `json:"os_family,omitempty"`
Version string `json:"version,omitempty"`
Architecture string `json:"architecture,omitempty"`
SizeBytes uint64 `json:"size_bytes,omitempty"`
VirtualSizeBytes uint64 `json:"virtual_size_bytes,omitempty"`
SHA256 string `json:"sha256,omitempty"`
Revision uint64 `json:"revision,omitempty"`
GenerationToken string `json:"generation_token,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
func decodeOperationEffectResult(raw []byte, target any) bool {
if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes {
return false
}
trimmed := strings.TrimSpace(string(raw))
return len(trimmed) >= 2 && trimmed[0] == '{' && trimmed[len(trimmed)-1] == '}' &&
json.Unmarshal(raw, target) == nil
}
func recordHasCleanupPending(record operation.Record) bool {
if len(record.Result) == 0 {
return false
}
var result struct {
CleanupPending bool `json:"cleanup_pending"`
}
return json.Unmarshal(record.Result, &result) == nil && result.CleanupPending
}
func operationInspectionToken(record operation.Record, kind, evidence string) string {
digest := sha256.New()
for _, part := range []string{
"virt-cloud-operation-inspection-v1", record.OperationID, record.ResourceID,
record.Action, record.Fingerprint, kind, evidence,
} {
digest.Write([]byte(part))
digest.Write([]byte{0})
}
return hex.EncodeToString(digest.Sum(nil))
}
func lowerHexDigest(value string) bool {
if len(value) != sha256.Size*2 {
return false
}
for _, current := range value {
if (current < '0' || current > '9') && (current < 'a' || current > 'f') {
return false
}
}
return true
}