You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
980 lines
36 KiB
Go
980 lines
36 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
var inspectionErrorCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
|
|
|
|
// operationCleanupInspection carries the exact, private host evidence used to
|
|
// bind a cleanup plan. Only the stable step codes are serialized; resource
|
|
// names, XML digests, and file attributes stay inside the Agent and feed the
|
|
// one-way plan identifier.
|
|
type operationCleanupInspection struct {
|
|
steps []string
|
|
evidence []string
|
|
}
|
|
|
|
func newOperationCleanupInspection(steps []string, evidence ...string) operationCleanupInspection {
|
|
if len(evidence) == 0 {
|
|
return operationCleanupInspection{}
|
|
}
|
|
normalizedEvidence := append([]string(nil), evidence...)
|
|
sort.Strings(normalizedEvidence)
|
|
return operationCleanupInspection{
|
|
steps: append([]string(nil), steps...), evidence: normalizedEvidence,
|
|
}
|
|
}
|
|
|
|
func (inspection operationCleanupInspection) tokenEvidence() string {
|
|
encoded, _ := json.Marshal(struct {
|
|
Steps []string `json:"steps"`
|
|
Evidence []string `json:"evidence"`
|
|
}{Steps: inspection.steps, Evidence: inspection.evidence})
|
|
return string(encoded)
|
|
}
|
|
|
|
// InspectOperation deliberately bypasses Journal admission. A dispatched
|
|
// libvirt call retains its resource lease until the real call returns, but an
|
|
// administrator must still be able to read its state and identify an owned
|
|
// cancellable backup job while that call is stuck. Every operation below is a
|
|
// libvirt/filesystem read. It may return a cleanup plan, but inspection never
|
|
// executes that plan or changes a host resource.
|
|
func (l *Libvirt) InspectOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
if storedOperationInspectionAvailable(record) {
|
|
return inspectionFromRecord(record), nil
|
|
}
|
|
|
|
if resolution, handled, err := l.reconcileStorageConfiguration(record); handled {
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
|
|
}
|
|
if resolution, handled, err := l.reconcileCatalogOperation(ctx, record); handled {
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
|
|
}
|
|
|
|
domain, err := l.domainByUUID(ctx, record.ResourceID)
|
|
if errors.Is(err, ErrNotFound) {
|
|
return l.inspectAbsentDomainOperation(ctx, record)
|
|
}
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
defer domain.Free()
|
|
|
|
abort := l.inspectMatchingBackupJob(record, domain)
|
|
resolution, cleanup, inspectErr := l.inspectExistingDomainOperation(ctx, record, domain)
|
|
if abort.Available {
|
|
// No cleanup may be approved while the matching backup is still live.
|
|
cleanup = operationCleanupInspection{}
|
|
}
|
|
return buildOperationInspection(record, resolution, abort, cleanup), inspectErr
|
|
}
|
|
|
|
func storedOperationInspectionAvailable(record operation.Record) bool {
|
|
if record.Tombstone {
|
|
return true
|
|
}
|
|
if record.State == operation.StateSucceeded && !recordHasCleanupPending(record) {
|
|
return true
|
|
}
|
|
return record.State == operation.StateFailed && record.Action != "create-vm" &&
|
|
!operation.RequiresExplicitForceDelete(record)
|
|
}
|
|
|
|
func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
|
|
var resolution operation.Resolution
|
|
var cleanup operationCleanupInspection
|
|
switch record.Action {
|
|
case "create-vm":
|
|
absent, err := l.deleteArtifactsAbsent(record.ResourceID)
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, absent, l.traffic.contains(record.ResourceID))
|
|
case "delete-vm", "force-delete-vm":
|
|
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil)
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
cleanup = plan
|
|
resolution = inspectAbsentDeleteResolution(plan, record.Action == "force-delete-vm")
|
|
default:
|
|
if record.ExpectedState != nil {
|
|
resolution = expectedStateResourceAbsent(record)
|
|
}
|
|
}
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, cleanup), nil
|
|
}
|
|
|
|
func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePresent bool) (operation.Resolution, operationCleanupInspection) {
|
|
if !managedArtifactsAbsent {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
resolution := operation.Resolution{
|
|
Resolved: true, Succeeded: false,
|
|
Result: []byte(`{"creation_absent":true,"managed_artifacts_absent":true}`),
|
|
ErrorCode: "creation_absent",
|
|
ErrorMessage: "host evidence confirms the VM and managed artifacts are absent",
|
|
}
|
|
if trafficStatePresent {
|
|
return resolution, newOperationCleanupInspection(
|
|
[]string{CleanupStepRemoveTrafficState}, "traffic-state:"+uuid,
|
|
)
|
|
}
|
|
return resolution, operationCleanupInspection{}
|
|
}
|
|
|
|
func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection, error) {
|
|
if record.Action == "delete-vm" || record.Action == "force-delete-vm" {
|
|
_, managed, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, err
|
|
}
|
|
if err := l.validateDeleteVMArtifacts(ctx, record.ResourceID, managed); err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, err
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
|
|
}
|
|
if resolution, handled := inspectActiveDeleteResolution(record, active); handled {
|
|
return resolution, operationCleanupInspection{}, nil
|
|
}
|
|
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, domain, managed)
|
|
return operation.Resolution{}, plan, err
|
|
}
|
|
if record.ExpectedState != nil {
|
|
switch record.ExpectedState.Kind {
|
|
case operation.ExpectedUpdateSettings, operation.ExpectedChangeNetwork,
|
|
operation.ExpectedNetworkState, operation.ExpectedServiceState,
|
|
operation.ExpectedFirewall:
|
|
resolution, _, err := l.inspectExpectedStateOperation(ctx, record, domain)
|
|
return resolution, operationCleanupInspection{}, err
|
|
}
|
|
}
|
|
if record.Action == "reinstall-vm" {
|
|
resolution, steps := l.inspectReinstallOperation(record, domain)
|
|
return resolution, steps, nil
|
|
}
|
|
if strings.HasPrefix(record.Action, "create-snapshot:") ||
|
|
strings.HasPrefix(record.Action, "delete-snapshot:") ||
|
|
strings.HasPrefix(record.Action, "restore-snapshot:") {
|
|
resolution, steps := l.inspectSnapshotOperation(record, domain)
|
|
return resolution, steps, nil
|
|
}
|
|
if record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
|
|
strings.HasPrefix(record.Action, "exit-rescue:") {
|
|
return l.reconcileRescue(ctx, record, domain), operationCleanupInspection{}, nil
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect operation domain state", err)
|
|
}
|
|
mapped := mapDomainState(state)
|
|
switch record.Action {
|
|
case "create-vm":
|
|
resolution, err := l.reconcileCreatedVM(domain)
|
|
return resolution, operationCleanupInspection{}, err
|
|
case "power-start":
|
|
if mapped == VMStateRunning || mapped == VMStateBlocked {
|
|
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
|
|
}
|
|
case "power-shutdown", "power-destroy":
|
|
if mapped == VMStateShutoff {
|
|
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
|
|
}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}, nil
|
|
}
|
|
|
|
func inspectActiveDeleteResolution(record operation.Record, active bool) (operation.Resolution, bool) {
|
|
if !active {
|
|
return operation.Resolution{}, false
|
|
}
|
|
if record.Action == "delete-vm" && operation.RequiresExplicitForceDelete(record) {
|
|
// Preserve an authoritative force_required outcome written by the
|
|
// original graceful DeleteVM call after its bounded wait.
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "force_required", ErrorMessage: "guest remains active and requires an explicitly authorized force delete"}, true
|
|
}
|
|
if record.Action == "delete-vm" &&
|
|
(record.CauseCode == "graceful_shutdown_ambiguous" ||
|
|
recoveredGracefulAttemptRequiresRetry(record)) {
|
|
// The prior ShutdownFlags response was lost and activity could not be
|
|
// observed conclusively, or the Agent restarted at the dispatch
|
|
// boundary before it persisted any cause. A later active observation
|
|
// can classify the already-terminal operation as failed, but cannot grant
|
|
// force authorization or dispatch another mutation.
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "graceful_shutdown_unconfirmed",
|
|
ErrorMessage: "graceful shutdown result was not confirmed; the resource remains blocked for manual remediation"}, true
|
|
}
|
|
if record.Action == "force-delete-vm" && forceDeleteCanBeConfirmedForRetry(record) {
|
|
// Never replay DestroyFlags automatically. Once no native call can still
|
|
// hold Journal admission, a token-confirmed administrator inspection may
|
|
// close an old force attempt which still observes an active Domain. The
|
|
// original graceful force_required record remains the authorization
|
|
// anchor, so a retry still needs a new operation ID and data-loss ack.
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "force_delete_unconfirmed_active",
|
|
ErrorMessage: "force delete did not reach an inactive domain; inspect and explicitly authorize a new force operation"}, true
|
|
}
|
|
// A force operation may already have crossed Destroy before its result was
|
|
// lost. Read-only inspection must not turn that ambiguity into a cleanup
|
|
// plan which would replay Destroy.
|
|
return operation.Resolution{}, true
|
|
}
|
|
|
|
func forceDeleteCanBeConfirmedForRetry(record operation.Record) bool {
|
|
return record.State == operation.StateFailed && record.EffectMayHaveApplied
|
|
}
|
|
|
|
func recoveredGracefulAttemptRequiresRetry(record operation.Record) bool {
|
|
if record.CauseCode != "" {
|
|
return false
|
|
}
|
|
return record.State == operation.StateFailed && record.EffectMayHaveApplied
|
|
}
|
|
|
|
func inspectAbsentDeleteResolution(cleanup operationCleanupInspection, forced bool) operation.Resolution {
|
|
if len(cleanup.steps) != 0 {
|
|
return operation.Resolution{}
|
|
}
|
|
return resolvedDeleteSuccess(DeleteVMResult{
|
|
Deleted: true, ManagedArtifactsAbsent: true, Forced: forced,
|
|
})
|
|
}
|
|
|
|
func (l *Libvirt) inspectDeleteCleanupSteps(ctx context.Context, uuid string,
|
|
domain *libvirt.Domain, managed *domainMetadataXML) (operationCleanupInspection, error) {
|
|
steps := make([]string, 0, 7)
|
|
evidence := make([]string, 0, 8)
|
|
if domain != nil {
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}, nil
|
|
}
|
|
evidence = append(evidence, jobEvidence...)
|
|
deleteEvidence, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain)
|
|
if err != nil {
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
evidence = append(evidence, deleteEvidence...)
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
|
|
}
|
|
if active {
|
|
return operationCleanupInspection{}, nil
|
|
}
|
|
evidence = append(evidence, "domain:"+uuid+":inactive")
|
|
steps = append(steps, CleanupStepUndefineDomain)
|
|
}
|
|
|
|
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed storage pool", err)
|
|
}
|
|
inventory, err := pool.ListStorageVolumes()
|
|
if err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed volumes", err)
|
|
}
|
|
names, err := collectVMManagedVolumeNames(uuid, managed, inventory)
|
|
if err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
managedVolumesExist := false
|
|
for _, name := range names {
|
|
if err := ctx.Err(); err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
volume, lookupErr := pool.LookupStorageVolByName(name)
|
|
if isNoStorageVolume(lookupErr) {
|
|
continue
|
|
}
|
|
if lookupErr != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", lookupErr)
|
|
}
|
|
managedVolumesExist = true
|
|
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
|
|
volume.Free()
|
|
pool.Free()
|
|
return operationCleanupInspection{}, validateErr
|
|
}
|
|
volumeDigest, digestErr := storageVolumeInspectionDigest(volume)
|
|
if digestErr != nil {
|
|
volume.Free()
|
|
pool.Free()
|
|
return operationCleanupInspection{}, digestErr
|
|
}
|
|
path, pathErr := volume.GetPath()
|
|
volume.Free()
|
|
if pathErr != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", pathErr)
|
|
}
|
|
var referenced bool
|
|
if domain != nil {
|
|
referenced, err = l.otherDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), name, uuid,
|
|
)
|
|
} else {
|
|
referenced, err = l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), name,
|
|
)
|
|
}
|
|
if err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
if referenced {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed cleanup volume is referenced by another domain", nil)
|
|
}
|
|
evidence = append(evidence, "managed-volume:"+name+":"+volumeDigest)
|
|
}
|
|
pool.Free()
|
|
if managedVolumesExist {
|
|
steps = append(steps, CleanupStepDeleteManagedVolumes)
|
|
}
|
|
|
|
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
|
|
if err != nil {
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
if seed != nil {
|
|
seedDigest, digestErr := storageVolumeInspectionDigest(seed)
|
|
if digestErr != nil {
|
|
seed.Free()
|
|
return operationCleanupInspection{}, digestErr
|
|
}
|
|
path, pathErr := seed.GetPath()
|
|
seed.Free()
|
|
if pathErr != nil {
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed seed cleanup volume", pathErr)
|
|
}
|
|
referenced, referenceErr := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
|
|
)
|
|
if domain != nil {
|
|
referenced, referenceErr = l.otherDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid,
|
|
)
|
|
}
|
|
if referenceErr != nil {
|
|
return operationCleanupInspection{}, referenceErr
|
|
}
|
|
if referenced {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed seed cleanup volume is referenced by another domain", nil)
|
|
}
|
|
steps = append(steps, CleanupStepDeleteSeedVolume)
|
|
evidence = append(evidence, "seed-volume:"+seedVolumeName(uuid)+":"+seedDigest)
|
|
}
|
|
|
|
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
|
|
if err != nil {
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
if filter != nil {
|
|
document, xmlErr := filter.GetXMLDesc(0)
|
|
filter.Free()
|
|
if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_invalid", "network filter cannot be confirmed as panel managed", xmlErr)
|
|
}
|
|
excludedUUID := ""
|
|
if domain != nil {
|
|
excludedUUID = uuid
|
|
}
|
|
referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), excludedUUID)
|
|
if referenceErr != nil {
|
|
return operationCleanupInspection{}, referenceErr
|
|
}
|
|
if referenced {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil)
|
|
}
|
|
steps = append(steps, CleanupStepUndefineNetworkFilter)
|
|
digest := sha256.Sum256([]byte(document))
|
|
evidence = append(evidence, "network-filter:"+networkFilterName(uuid)+":"+hex.EncodeToString(digest[:]))
|
|
}
|
|
|
|
nvramExists := false
|
|
for _, name := range managedNVRAMArtifactNames(uuid) {
|
|
path := filepath.Join(l.local.NVRAMDirectory, name)
|
|
info, statErr := os.Lstat(path)
|
|
if errors.Is(statErr, os.ErrNotExist) {
|
|
continue
|
|
}
|
|
if statErr != nil {
|
|
return operationCleanupInspection{}, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr)
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil)
|
|
}
|
|
referenced, referenceErr := l.anyDomainUsesPath(path)
|
|
if domain != nil {
|
|
referenced, referenceErr = l.otherDomainUsesPath(path, uuid)
|
|
}
|
|
if referenceErr != nil {
|
|
return operationCleanupInspection{}, referenceErr
|
|
}
|
|
if referenced {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil)
|
|
}
|
|
nvramExists = true
|
|
evidence = append(evidence, "nvram:"+name+":"+strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10))
|
|
}
|
|
if nvramExists {
|
|
steps = append(steps, CleanupStepDeleteNVRAM)
|
|
}
|
|
trafficPresent := l.traffic.contains(uuid)
|
|
if trafficPresent || len(steps) != 0 {
|
|
// cleanupDeletedVMArtifacts durably rewrites traffic state even if an
|
|
// earlier failed fsync left only an on-disk entry. Make that mutation
|
|
// explicit in every delete cleanup plan.
|
|
steps = append(steps, CleanupStepRemoveTrafficState)
|
|
if trafficPresent {
|
|
evidence = append(evidence, "traffic-state:"+uuid+":present")
|
|
} else {
|
|
evidence = append(evidence, "traffic-state:"+uuid+":rewrite")
|
|
}
|
|
}
|
|
return newOperationCleanupInspection(steps, evidence...), nil
|
|
}
|
|
|
|
func managedFilterDocument(document, expectedName string) bool {
|
|
var filter libvirtxml.NWFilter
|
|
if filter.Unmarshal(document) != nil || filter.Name != expectedName || filter.Chain != "root" {
|
|
return false
|
|
}
|
|
baselineReferences := 0
|
|
for _, entry := range filter.Entries {
|
|
if entry.Ref == nil {
|
|
continue
|
|
}
|
|
if entry.Ref.Filter != "clean-traffic" {
|
|
return false
|
|
}
|
|
baselineReferences++
|
|
}
|
|
return baselineReferences == 1
|
|
}
|
|
|
|
func storageVolumeInspectionDigest(volume *libvirt.StorageVol) (string, error) {
|
|
document, err := volume.GetXMLDesc(0)
|
|
if err != nil {
|
|
return "", translateLibvirtError("read cleanup volume XML", err)
|
|
}
|
|
digest := sha256.Sum256([]byte(document))
|
|
return hex.EncodeToString(digest[:]), nil
|
|
}
|
|
|
|
func (l *Libvirt) inspectReinstallOperation(record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementName := reinstallVolumeName(record.ResourceID, record.OperationID)
|
|
replacement, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementExists := replacement != nil
|
|
if replacement != nil {
|
|
replacement.Free()
|
|
}
|
|
if diskTransitionCommitted(metadata, replacementName, "reinstall", record.OperationID) && replacementExists {
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "reinstall", record.OperationID) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
|
|
}
|
|
result, parseErr := metadataFromXML(document)
|
|
if parseErr != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
result.Status = mapDomainState(state)
|
|
result.ReinstallCommitted = true
|
|
result.CleanupPending = false
|
|
encoded, _ := json.Marshal(result)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
|
|
}
|
|
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "reinstall_not_committed", ErrorMessage: "host evidence confirms the reinstall did not switch the root volume"}, operationCleanupInspection{}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
|
|
func (l *Libvirt) inspectSnapshotOperation(record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
key := parts[1]
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
var stored *domainSnapshotMetadataXML
|
|
for index := range metadata.Snapshots {
|
|
if metadata.Snapshots[index].Key == key {
|
|
stored = &metadata.Snapshots[index]
|
|
break
|
|
}
|
|
}
|
|
volume, err := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, key),
|
|
)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
volumeExists := volume != nil
|
|
if volume != nil {
|
|
defer volume.Free()
|
|
}
|
|
|
|
switch parts[0] {
|
|
case "create-snapshot":
|
|
if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && volumeExists && snapshotConsistencyMatchesRequest(parts[2], stored.Consistency) {
|
|
info, validateErr := validateQCOW2Volume(volume, stored.SizeBytes)
|
|
if validateErr != nil || info.Capacity != stored.SizeBytes {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
result := Snapshot{ID: key, SnapshotKey: key, Name: stored.Name,
|
|
Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(),
|
|
State: "ready", Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, Created: true}
|
|
encoded, _ := json.Marshal(result)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && !volumeExists {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_created", ErrorMessage: "host evidence confirms the snapshot is absent"}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && volumeExists {
|
|
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
|
|
domain, volume, snapshotVolumeName(record.ResourceID, key),
|
|
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
|
|
)
|
|
}
|
|
if stored != nil && !volumeExists {
|
|
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
|
|
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
|
|
)
|
|
}
|
|
case "delete-snapshot":
|
|
if len(parts) != 2 {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && !volumeExists {
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && volumeExists {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_deleted", ErrorMessage: "host evidence confirms the snapshot remains present"}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && !volumeExists {
|
|
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
|
|
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
|
|
)
|
|
}
|
|
if stored == nil && volumeExists {
|
|
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
|
|
domain, volume, snapshotVolumeName(record.ResourceID, key),
|
|
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
|
|
)
|
|
}
|
|
case "restore-snapshot":
|
|
if len(parts) != 2 || stored == nil || !volumeExists {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID)
|
|
replacement, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
|
|
if lookupErr != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementExists := replacement != nil
|
|
if replacement != nil {
|
|
replacement.Free()
|
|
}
|
|
if diskTransitionCommitted(metadata, replacementName, "snapshot_restore", record.OperationID) && replacementExists {
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "snapshot_restore", record.OperationID) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
|
|
}
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Restored: true,
|
|
PowerState: mapDomainState(state), ActiveVolumeName: replacementName, CleanupPending: false})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
|
|
}
|
|
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_restored", ErrorMessage: "host evidence confirms the root volume was not switched"}, operationCleanupInspection{}
|
|
}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
|
|
func diskTransitionPowerStateMatches(metadata *domainMetadataXML, state libvirt.DomainState,
|
|
kind, operationID string) bool {
|
|
if !diskTransitionMarkerMatches(metadata, kind, operationID) {
|
|
return true
|
|
}
|
|
if metadata.DiskTransitionRunning {
|
|
return state == libvirt.DOMAIN_RUNNING || state == libvirt.DOMAIN_BLOCKED
|
|
}
|
|
return state == libvirt.DOMAIN_SHUTOFF || state == libvirt.DOMAIN_SHUTDOWN
|
|
}
|
|
|
|
func (l *Libvirt) inspectDiskTransitionCleanupSteps(domain *libvirt.Domain, document string,
|
|
metadata *domainMetadataXML) operationCleanupInspection {
|
|
pending := metadata.PendingCleanupVolume
|
|
if pending == "" || !isVMOwnedRootVolumeName(metadataUUID(document), pending) ||
|
|
pending == metadata.RootVolumeName {
|
|
return operationCleanupInspection{}
|
|
}
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}
|
|
}
|
|
steps := make([]string, 0, 2)
|
|
evidence := append([]string(nil), jobEvidence...)
|
|
evidence = append(evidence, "pending-root:"+pending, "active-root:"+metadata.RootVolumeName)
|
|
old, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), pending)
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if old != nil {
|
|
if _, validateErr := validateQCOW2Volume(old, 0); validateErr != nil {
|
|
old.Free()
|
|
return operationCleanupInspection{}
|
|
}
|
|
oldDigest, digestErr := storageVolumeInspectionDigest(old)
|
|
if digestErr != nil {
|
|
old.Free()
|
|
return operationCleanupInspection{}
|
|
}
|
|
path, pathErr := old.GetPath()
|
|
old.Free()
|
|
if pathErr != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
used, referenceErr := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), pending,
|
|
)
|
|
if referenceErr != nil || used {
|
|
return operationCleanupInspection{}
|
|
}
|
|
steps = append(steps, CleanupStepDeletePreviousRoot)
|
|
evidence = append(evidence, "pending-root-state:present:"+oldDigest)
|
|
} else {
|
|
evidence = append(evidence, "pending-root-state:absent")
|
|
}
|
|
steps = append(steps, CleanupStepPersistCleanupMetadata)
|
|
return newOperationCleanupInspection(steps, evidence...)
|
|
}
|
|
|
|
type cleanupJobDomain interface {
|
|
IsActive() (bool, error)
|
|
GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error)
|
|
GetXMLDesc(libvirt.DomainXMLFlags) (string, error)
|
|
GetBlockJobInfo(string, libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error)
|
|
}
|
|
|
|
func inspectCleanupJobEvidence(domain cleanupJobDomain) ([]string, bool) {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
flags := libvirt.DOMAIN_XML_INACTIVE
|
|
evidence := []string{"domain-runtime:inactive"}
|
|
if active {
|
|
flags = 0
|
|
job, jobErr := domain.GetJobStats(0)
|
|
if jobErr != nil || job == nil || job.Type != libvirt.DOMAIN_JOB_NONE {
|
|
return nil, false
|
|
}
|
|
evidence[0] = "domain-runtime:active:job-none"
|
|
}
|
|
document, err := domain.GetXMLDesc(flags)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
documentDigest := sha256.Sum256([]byte(document))
|
|
evidence = append(evidence, "domain-xml:"+hex.EncodeToString(documentDigest[:]))
|
|
if active {
|
|
persistentDocument, persistentErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if persistentErr != nil {
|
|
return nil, false
|
|
}
|
|
persistentDigest := sha256.Sum256([]byte(persistentDocument))
|
|
evidence = append(evidence, "persistent-domain-xml:"+hex.EncodeToString(persistentDigest[:]))
|
|
}
|
|
targets, err := cleanupDiskTargets(document)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
for _, target := range targets {
|
|
evidence = append(evidence, "disk-target:"+target)
|
|
if !active {
|
|
continue
|
|
}
|
|
info, infoErr := domain.GetBlockJobInfo(target, 0)
|
|
if infoErr != nil || !blockJobInfoAbsent(info) {
|
|
return nil, false
|
|
}
|
|
}
|
|
return evidence, true
|
|
}
|
|
|
|
func cleanupDiskTargets(document string) ([]string, error) {
|
|
var definition libvirtxml.Domain
|
|
if err := definition.Unmarshal(document); err != nil || definition.Devices == nil {
|
|
return nil, errors.New("domain disk evidence is invalid")
|
|
}
|
|
seen := make(map[string]struct{}, len(definition.Devices.Disks))
|
|
for _, disk := range definition.Devices.Disks {
|
|
if disk.Target == nil || disk.Target.Dev == "" {
|
|
return nil, errors.New("domain disk target evidence is incomplete")
|
|
}
|
|
seen[disk.Target.Dev] = struct{}{}
|
|
}
|
|
targets := make([]string, 0, len(seen))
|
|
for target := range seen {
|
|
targets = append(targets, target)
|
|
}
|
|
sort.Strings(targets)
|
|
return targets, nil
|
|
}
|
|
|
|
func blockJobInfoAbsent(info *libvirt.DomainBlockJobInfo) bool {
|
|
// The Go binding does not expose virDomainGetBlockJobInfo's 0/1 return
|
|
// value. A 0 (no job) is represented by the zero-valued info struct.
|
|
return info != nil && info.Type == libvirt.DOMAIN_BLOCK_JOB_TYPE_UNKNOWN &&
|
|
info.Bandwidth == 0 && info.Cur == 0 && info.End == 0
|
|
}
|
|
|
|
func cleanupInspectionWithIdleDomain(domain cleanupJobDomain, steps []string,
|
|
evidence ...string) operationCleanupInspection {
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}
|
|
}
|
|
evidence = append(evidence, jobEvidence...)
|
|
return newOperationCleanupInspection(steps, evidence...)
|
|
}
|
|
|
|
func ensureNoCleanupBlockingJobs(domain cleanupJobDomain) error {
|
|
if _, safe := inspectCleanupJobEvidence(domain); !safe {
|
|
return NewError(ErrConflict, "cleanup_job_not_idle", "domain cleanup is blocked by active or unreadable job state", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) snapshotVolumeCleanupInspection(domain cleanupJobDomain, volume *libvirt.StorageVol,
|
|
volumeName, step, targetEvidence string) operationCleanupInspection {
|
|
if volume == nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if _, err := validateQCOW2Volume(volume, 0); err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
volumeDigest, err := storageVolumeInspectionDigest(volume)
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
path, err := volume.GetPath()
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
referenced, err := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), volumeName,
|
|
)
|
|
if err != nil || referenced {
|
|
return operationCleanupInspection{}
|
|
}
|
|
evidence := append([]string{targetEvidence + ":" + volumeDigest}, jobEvidence...)
|
|
return newOperationCleanupInspection([]string{step}, evidence...)
|
|
}
|
|
|
|
func (l *Libvirt) inspectMatchingBackupJob(record operation.Record, domain *libvirt.Domain) OperationAbortInspection {
|
|
if record.State != operation.StateDispatching &&
|
|
(record.State != operation.StateFailed || !record.EffectMayHaveApplied) {
|
|
return OperationAbortInspection{}
|
|
}
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) != 3 || parts[0] != "create-snapshot" || !catalogIDPattern.MatchString(parts[1]) {
|
|
return OperationAbortInspection{}
|
|
}
|
|
job, err := domain.GetJobStats(0)
|
|
if err != nil || job == nil ||
|
|
(job.Type != libvirt.DOMAIN_JOB_BOUNDED && job.Type != libvirt.DOMAIN_JOB_UNBOUNDED) ||
|
|
!job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
|
|
return OperationAbortInspection{}
|
|
}
|
|
backupXML, err := domain.BackupGetXMLDesc(0)
|
|
if err != nil {
|
|
return OperationAbortInspection{}
|
|
}
|
|
volume, err := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, parts[1]),
|
|
)
|
|
if err != nil || volume == nil {
|
|
return OperationAbortInspection{}
|
|
}
|
|
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
|
|
volume.Free()
|
|
return OperationAbortInspection{}
|
|
}
|
|
targetPath, pathErr := volume.GetPath()
|
|
volume.Free()
|
|
if pathErr != nil || !backupPushMatchesTarget(backupXML, targetPath) {
|
|
return OperationAbortInspection{}
|
|
}
|
|
return OperationAbortInspection{
|
|
Available: true, Kind: "backup",
|
|
InspectionToken: operationInspectionToken(record, "backup", backupXML),
|
|
}
|
|
}
|
|
|
|
func backupPushMatchesTarget(backupXML, targetPath string) bool {
|
|
var backup libvirtxml.DomainBackup
|
|
if backup.Unmarshal(backupXML) != nil || backup.Pull != nil || backup.Push == nil ||
|
|
backup.Push.Disks == nil || len(backup.Push.Disks.Disks) != 1 {
|
|
return false
|
|
}
|
|
disk := backup.Push.Disks.Disks[0]
|
|
return disk.Name == "vda" && disk.Backup == "yes" && disk.Target != nil &&
|
|
disk.Target.File != nil && filepath.Clean(disk.Target.File.File) == filepath.Clean(targetPath)
|
|
}
|
|
|
|
func buildOperationInspection(record operation.Record, resolution operation.Resolution,
|
|
abort OperationAbortInspection, cleanup operationCleanupInspection) OperationInspection {
|
|
result := OperationInspection{
|
|
OperationID: record.OperationID, OperationStatus: record.State,
|
|
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
|
|
SummaryCode: InspectionSummaryIncomplete,
|
|
AbortJob: abort, Cleanup: OperationCleanupPlan{},
|
|
}
|
|
if abort.Available {
|
|
result.SummaryCode = InspectionSummaryAbortableJob
|
|
return result
|
|
}
|
|
if len(cleanup.steps) != 0 && len(cleanup.evidence) != 0 {
|
|
result.SummaryCode = InspectionSummaryCleanupRequired
|
|
result.Cleanup = OperationCleanupPlan{
|
|
Available: true,
|
|
PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()),
|
|
Steps: append([]string(nil), cleanup.steps...),
|
|
}
|
|
return result
|
|
}
|
|
if !resolution.Resolved {
|
|
return result
|
|
}
|
|
if resolution.Succeeded {
|
|
result.Outcome = InspectionOutcomeSucceeded
|
|
result.SummaryCode = InspectionSummarySucceeded
|
|
} else {
|
|
result.Outcome = InspectionOutcomeFailed
|
|
result.SummaryCode = InspectionSummaryFailed
|
|
if inspectionErrorCodePattern.MatchString(resolution.ErrorCode) {
|
|
result.ErrorCode = resolution.ErrorCode
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func inspectionFromRecord(record operation.Record) OperationInspection {
|
|
switch record.State {
|
|
case operation.StateSucceeded:
|
|
return buildOperationInspection(record, operation.Resolution{
|
|
Resolved: true, Succeeded: true, Result: append([]byte(nil), record.Result...),
|
|
}, OperationAbortInspection{}, operationCleanupInspection{})
|
|
case operation.StateFailed:
|
|
resolution := operation.Resolution{Resolved: true, Succeeded: false, Result: append([]byte(nil), record.Result...)}
|
|
if record.Error != nil {
|
|
resolution.ErrorCode = record.Error.Code
|
|
resolution.ErrorMessage = record.Error.Message
|
|
}
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{})
|
|
}
|
|
return OperationInspection{
|
|
OperationID: record.OperationID, OperationStatus: record.State,
|
|
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
|
|
SummaryCode: InspectionSummaryIncomplete,
|
|
}
|
|
}
|
|
|
|
func recordHasCleanupPending(record operation.Record) bool {
|
|
if len(record.Result) == 0 {
|
|
return false
|
|
}
|
|
var result struct {
|
|
CleanupPending bool `json:"cleanup_pending"`
|
|
}
|
|
return json.Unmarshal(record.Result, &result) == nil && result.CleanupPending
|
|
}
|
|
|
|
func operationInspectionToken(record operation.Record, kind, evidence string) string {
|
|
digest := sha256.New()
|
|
for _, part := range []string{
|
|
"virt-cloud-operation-inspection-v1", record.OperationID, record.ResourceID,
|
|
record.Action, record.Fingerprint, kind, evidence,
|
|
} {
|
|
digest.Write([]byte(part))
|
|
digest.Write([]byte{0})
|
|
}
|
|
return hex.EncodeToString(digest.Sum(nil))
|
|
}
|
|
|
|
func lowerHexDigest(value string) bool {
|
|
if len(value) != sha256.Size*2 {
|
|
return false
|
|
}
|
|
for _, current := range value {
|
|
if (current < '0' || current > '9') && (current < 'a' || current > 'f') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|