You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1479 lines
58 KiB
Go
1479 lines
58 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
var inspectionErrorCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
|
|
|
|
// operationCleanupInspection carries the exact, private host evidence used to
|
|
// bind a cleanup plan. Only the stable step codes are serialized; resource
|
|
// names, XML digests, and file attributes stay inside the Agent and feed the
|
|
// one-way plan identifier.
|
|
type operationCleanupInspection struct {
|
|
steps []string
|
|
evidence []string
|
|
}
|
|
|
|
func newOperationCleanupInspection(steps []string, evidence ...string) operationCleanupInspection {
|
|
if len(evidence) == 0 {
|
|
return operationCleanupInspection{}
|
|
}
|
|
normalizedEvidence := append([]string(nil), evidence...)
|
|
sort.Strings(normalizedEvidence)
|
|
return operationCleanupInspection{
|
|
steps: append([]string(nil), steps...), evidence: normalizedEvidence,
|
|
}
|
|
}
|
|
|
|
func (inspection operationCleanupInspection) tokenEvidence() string {
|
|
encoded, _ := json.Marshal(struct {
|
|
Steps []string `json:"steps"`
|
|
Evidence []string `json:"evidence"`
|
|
}{Steps: inspection.steps, Evidence: inspection.evidence})
|
|
return string(encoded)
|
|
}
|
|
|
|
// InspectOperation deliberately bypasses Journal admission. A dispatched
|
|
// libvirt call retains its resource lease until the real call returns, but an
|
|
// administrator must still be able to read its state and identify an owned
|
|
// cancellable backup job while that call is stuck. Every operation below is a
|
|
// libvirt/filesystem read. It may return a cleanup plan, but inspection never
|
|
// executes that plan or changes a host resource.
|
|
func (l *Libvirt) InspectOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
if successfulBackupCleanupPending(record) {
|
|
if record.InspectionEvidenceUnavailable {
|
|
return inconclusiveInspectionFromRecord(record), nil
|
|
}
|
|
return l.inspectSuccessfulBackupCleanupPending(ctx, record), nil
|
|
}
|
|
if storedOperationInspectionAvailable(record) {
|
|
if record.InspectionEvidenceUnavailable {
|
|
return inconclusiveInspectionFromRecord(record), nil
|
|
}
|
|
inspection := inspectionFromRecord(record)
|
|
// A recovered successful receipt is useful for convergence only when its
|
|
// action-specific result can be reduced to the bounded public projection.
|
|
// Receipt status alone must not turn an unprojectable or damaged result
|
|
// into an "applied" verdict after Journal detail retention.
|
|
if record.InspectionReceiptRecovered && record.State == operation.StateSucceeded &&
|
|
len(inspection.EffectProjection) == 0 {
|
|
return inconclusiveInspectionFromRecord(record), nil
|
|
}
|
|
return inspection, nil
|
|
}
|
|
|
|
if resolution, handled, err := l.reconcileStorageConfiguration(record); handled {
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
|
|
}
|
|
if resolution, handled, err := l.reconcileCatalogOperation(ctx, record); handled {
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
|
|
}
|
|
|
|
domain, err := l.domainByUUID(ctx, record.ResourceID)
|
|
if errors.Is(err, ErrNotFound) {
|
|
return l.inspectAbsentDomainOperation(ctx, record)
|
|
}
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
defer domain.Free()
|
|
|
|
abort := l.inspectMatchingBackupJob(record, domain)
|
|
resolution, cleanup, inspectErr := l.inspectExistingDomainOperation(ctx, record, domain)
|
|
if abort.Available {
|
|
// No cleanup may be approved while the matching backup is still live.
|
|
cleanup = operationCleanupInspection{}
|
|
}
|
|
return buildOperationInspection(record, resolution, abort, cleanup), inspectErr
|
|
}
|
|
|
|
func storedOperationInspectionAvailable(record operation.Record) bool {
|
|
if record.Tombstone {
|
|
return true
|
|
}
|
|
if record.State == operation.StateSucceeded && strings.HasPrefix(record.Action, "create-backup:") {
|
|
// A successful backup result is immutable proof that the remote object
|
|
// committed, even when its optional cleanup_pending marker reports only
|
|
// local residue. Never discard the sole object SHA/size projection by
|
|
// replacing it with a weaker current-host observation.
|
|
return true
|
|
}
|
|
if record.State == operation.StateSucceeded && !recordHasCleanupPending(record) {
|
|
return true
|
|
}
|
|
if record.State == operation.StateFailed && record.EffectMayHaveApplied {
|
|
// A failed Journal row describes the response observed by the original
|
|
// caller, not necessarily the durable host effect. Reuse would make the
|
|
// operation look definitively failed forever and bypass the read-only
|
|
// expected-state reconcilers below.
|
|
return false
|
|
}
|
|
return record.State == operation.StateFailed && !operation.IsCreateLikeAction(record.Action) &&
|
|
!operation.RequiresExplicitForceDelete(record)
|
|
}
|
|
|
|
// inspectSuccessfulBackupCleanupPending preserves the immutable applied
|
|
// object projection while adding an optional, read-only cleanup plan for an
|
|
// exact operation-owned local snapshot residue. Host read failures never
|
|
// downgrade the already verified remote object identity.
|
|
func (l *Libvirt) inspectSuccessfulBackupCleanupPending(ctx context.Context,
|
|
record operation.Record) OperationInspection {
|
|
inspection := inspectionFromRecord(record)
|
|
domain, err := l.domainByUUID(ctx, record.ResourceID)
|
|
if err != nil {
|
|
return inspection
|
|
}
|
|
defer domain.Free()
|
|
|
|
_, cleanup := l.inspectSnapshotOperation(record, domain)
|
|
return appliedBackupInspectionWithCleanup(record, inspection, cleanup)
|
|
}
|
|
|
|
func appliedBackupInspectionWithCleanup(record operation.Record, inspection OperationInspection,
|
|
cleanup operationCleanupInspection) OperationInspection {
|
|
if len(cleanup.steps) == 0 || len(cleanup.evidence) == 0 {
|
|
return inspection
|
|
}
|
|
inspection.SummaryCode = InspectionSummaryCleanupRequired
|
|
inspection.Cleanup = OperationCleanupPlan{
|
|
Available: true,
|
|
PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()),
|
|
Steps: append([]string(nil), cleanup.steps...),
|
|
}
|
|
return inspection
|
|
}
|
|
|
|
func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
|
|
var resolution operation.Resolution
|
|
var cleanup operationCleanupInspection
|
|
switch {
|
|
case operation.IsCreateLikeAction(record.Action):
|
|
absent, err := l.deleteArtifactsAbsent(record.ResourceID)
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, absent, l.traffic.contains(record.ResourceID))
|
|
case record.Action == "delete-vm" || record.Action == "force-delete-vm":
|
|
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil)
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
cleanup = plan
|
|
resolution = inspectAbsentDeleteResolution(plan, record.Action == "force-delete-vm")
|
|
default:
|
|
if record.ExpectedState != nil {
|
|
resolution = expectedStateResourceAbsent(record)
|
|
}
|
|
}
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, cleanup), nil
|
|
}
|
|
|
|
func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePresent bool) (operation.Resolution, operationCleanupInspection) {
|
|
if !managedArtifactsAbsent {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
resolution := operation.Resolution{
|
|
Resolved: true, Succeeded: false,
|
|
Result: []byte(`{"creation_absent":true,"managed_artifacts_absent":true}`),
|
|
ErrorCode: "creation_absent",
|
|
ErrorMessage: "host evidence confirms the VM and managed artifacts are absent",
|
|
}
|
|
if trafficStatePresent {
|
|
return resolution, newOperationCleanupInspection(
|
|
[]string{CleanupStepRemoveTrafficState}, "traffic-state:"+uuid,
|
|
)
|
|
}
|
|
return resolution, operationCleanupInspection{}
|
|
}
|
|
|
|
func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection, error) {
|
|
if record.Action == "delete-vm" || record.Action == "force-delete-vm" {
|
|
_, managed, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, err
|
|
}
|
|
if err := l.validateDeleteVMArtifacts(ctx, record.ResourceID, managed); err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, err
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
|
|
}
|
|
if resolution, handled := inspectActiveDeleteResolution(record, active); handled {
|
|
return resolution, operationCleanupInspection{}, nil
|
|
}
|
|
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, domain, managed)
|
|
return operation.Resolution{}, plan, err
|
|
}
|
|
if record.ExpectedState != nil {
|
|
switch record.ExpectedState.Kind {
|
|
case operation.ExpectedUpdateSettings, operation.ExpectedResizeResources, operation.ExpectedChangeNetwork,
|
|
operation.ExpectedNetworkState, operation.ExpectedServiceState,
|
|
operation.ExpectedFirewall:
|
|
resolution, _, err := l.inspectExpectedStateOperation(ctx, record, domain)
|
|
return resolution, operationCleanupInspection{}, err
|
|
}
|
|
}
|
|
if record.Action == "reinstall-vm" {
|
|
resolution, steps := l.inspectReinstallOperation(record, domain)
|
|
return resolution, steps, nil
|
|
}
|
|
if strings.HasPrefix(record.Action, "create-snapshot:") ||
|
|
strings.HasPrefix(record.Action, "create-backup:") ||
|
|
strings.HasPrefix(record.Action, "delete-snapshot:") ||
|
|
strings.HasPrefix(record.Action, "restore-snapshot:") {
|
|
resolution, steps := l.inspectSnapshotOperation(record, domain)
|
|
return resolution, steps, nil
|
|
}
|
|
if record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
|
|
strings.HasPrefix(record.Action, "exit-rescue:") {
|
|
return l.reconcileRescue(ctx, record, domain), operationCleanupInspection{}, nil
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect operation domain state", err)
|
|
}
|
|
mapped := mapDomainState(state)
|
|
switch {
|
|
case operation.IsCreateLikeAction(record.Action):
|
|
resolution, err := l.reconcileCreatedVM(domain)
|
|
return resolution, operationCleanupInspection{}, err
|
|
case record.Action == "power-start":
|
|
if mapped == VMStateRunning || mapped == VMStateBlocked {
|
|
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
|
|
}
|
|
case record.Action == "power-shutdown" || record.Action == "power-destroy":
|
|
if mapped == VMStateShutoff {
|
|
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
|
|
}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}, nil
|
|
}
|
|
|
|
func inspectActiveDeleteResolution(record operation.Record, active bool) (operation.Resolution, bool) {
|
|
if !active {
|
|
return operation.Resolution{}, false
|
|
}
|
|
if record.Action == "delete-vm" && operation.RequiresExplicitForceDelete(record) {
|
|
// Preserve an authoritative force_required outcome written by the
|
|
// original graceful DeleteVM call after its bounded wait.
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "force_required", ErrorMessage: "guest remains active and requires an explicitly authorized force delete"}, true
|
|
}
|
|
if record.Action == "delete-vm" &&
|
|
(record.CauseCode == "graceful_shutdown_ambiguous" ||
|
|
recoveredGracefulAttemptRequiresRetry(record)) {
|
|
// The prior ShutdownFlags response was lost and activity could not be
|
|
// observed conclusively, or the Agent restarted at the dispatch
|
|
// boundary before it persisted any cause. A later active observation
|
|
// can classify the already-terminal operation as failed, but cannot grant
|
|
// force authorization or dispatch another mutation.
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "graceful_shutdown_unconfirmed",
|
|
ErrorMessage: "graceful shutdown result was not confirmed; the resource remains blocked for manual remediation"}, true
|
|
}
|
|
if record.Action == "force-delete-vm" && forceDeleteCanBeConfirmedForRetry(record) {
|
|
// Never replay DestroyFlags automatically. Once no native call can still
|
|
// hold Journal admission, a token-confirmed administrator inspection may
|
|
// close an old force attempt which still observes an active Domain. The
|
|
// original graceful force_required record remains the authorization
|
|
// anchor, so a retry still needs a new operation ID and data-loss ack.
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "force_delete_unconfirmed_active",
|
|
ErrorMessage: "force delete did not reach an inactive domain; inspect and explicitly authorize a new force operation"}, true
|
|
}
|
|
// A force operation may already have crossed Destroy before its result was
|
|
// lost. Read-only inspection must not turn that ambiguity into a cleanup
|
|
// plan which would replay Destroy.
|
|
return operation.Resolution{}, true
|
|
}
|
|
|
|
func forceDeleteCanBeConfirmedForRetry(record operation.Record) bool {
|
|
return record.State == operation.StateFailed && record.EffectMayHaveApplied
|
|
}
|
|
|
|
func recoveredGracefulAttemptRequiresRetry(record operation.Record) bool {
|
|
if record.CauseCode != "" {
|
|
return false
|
|
}
|
|
return record.State == operation.StateFailed && record.EffectMayHaveApplied
|
|
}
|
|
|
|
func inspectAbsentDeleteResolution(cleanup operationCleanupInspection, forced bool) operation.Resolution {
|
|
if len(cleanup.steps) != 0 {
|
|
return operation.Resolution{}
|
|
}
|
|
return resolvedDeleteSuccess(DeleteVMResult{
|
|
Deleted: true, ManagedArtifactsAbsent: true, Forced: forced,
|
|
})
|
|
}
|
|
|
|
func (l *Libvirt) inspectDeleteCleanupSteps(ctx context.Context, uuid string,
|
|
domain *libvirt.Domain, managed *domainMetadataXML) (operationCleanupInspection, error) {
|
|
steps := make([]string, 0, 7)
|
|
evidence := make([]string, 0, 8)
|
|
if domain != nil {
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}, nil
|
|
}
|
|
evidence = append(evidence, jobEvidence...)
|
|
deleteEvidence, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain)
|
|
if err != nil {
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
evidence = append(evidence, deleteEvidence...)
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
|
|
}
|
|
if active {
|
|
return operationCleanupInspection{}, nil
|
|
}
|
|
evidence = append(evidence, "domain:"+uuid+":inactive")
|
|
steps = append(steps, CleanupStepUndefineDomain)
|
|
}
|
|
|
|
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed storage pool", err)
|
|
}
|
|
inventory, err := pool.ListStorageVolumes()
|
|
if err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed volumes", err)
|
|
}
|
|
names, err := collectVMManagedVolumeNames(uuid, managed, inventory)
|
|
if err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
managedVolumesExist := false
|
|
for _, name := range names {
|
|
if err := ctx.Err(); err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
volume, lookupErr := pool.LookupStorageVolByName(name)
|
|
if isNoStorageVolume(lookupErr) {
|
|
continue
|
|
}
|
|
if lookupErr != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", lookupErr)
|
|
}
|
|
managedVolumesExist = true
|
|
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
|
|
volume.Free()
|
|
pool.Free()
|
|
return operationCleanupInspection{}, validateErr
|
|
}
|
|
volumeDigest, digestErr := storageVolumeInspectionDigest(volume)
|
|
if digestErr != nil {
|
|
volume.Free()
|
|
pool.Free()
|
|
return operationCleanupInspection{}, digestErr
|
|
}
|
|
path, pathErr := volume.GetPath()
|
|
volume.Free()
|
|
if pathErr != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", pathErr)
|
|
}
|
|
var referenced bool
|
|
if domain != nil {
|
|
referenced, err = l.otherDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), name, uuid,
|
|
)
|
|
} else {
|
|
referenced, err = l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), name,
|
|
)
|
|
}
|
|
if err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
if referenced {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed cleanup volume is referenced by another domain", nil)
|
|
}
|
|
evidence = append(evidence, "managed-volume:"+name+":"+volumeDigest)
|
|
}
|
|
pool.Free()
|
|
if managedVolumesExist {
|
|
steps = append(steps, CleanupStepDeleteManagedVolumes)
|
|
}
|
|
|
|
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
|
|
if err != nil {
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
if seed != nil {
|
|
seedDigest, digestErr := storageVolumeInspectionDigest(seed)
|
|
if digestErr != nil {
|
|
seed.Free()
|
|
return operationCleanupInspection{}, digestErr
|
|
}
|
|
path, pathErr := seed.GetPath()
|
|
seed.Free()
|
|
if pathErr != nil {
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed seed cleanup volume", pathErr)
|
|
}
|
|
referenced, referenceErr := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
|
|
)
|
|
if domain != nil {
|
|
referenced, referenceErr = l.otherDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid,
|
|
)
|
|
}
|
|
if referenceErr != nil {
|
|
return operationCleanupInspection{}, referenceErr
|
|
}
|
|
if referenced {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed seed cleanup volume is referenced by another domain", nil)
|
|
}
|
|
steps = append(steps, CleanupStepDeleteSeedVolume)
|
|
evidence = append(evidence, "seed-volume:"+seedVolumeName(uuid)+":"+seedDigest)
|
|
}
|
|
|
|
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
|
|
if err != nil {
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
if filter != nil {
|
|
document, xmlErr := filter.GetXMLDesc(0)
|
|
filter.Free()
|
|
if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_invalid", "network filter cannot be confirmed as panel managed", xmlErr)
|
|
}
|
|
excludedUUID := ""
|
|
if domain != nil {
|
|
excludedUUID = uuid
|
|
}
|
|
referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), excludedUUID)
|
|
if referenceErr != nil {
|
|
return operationCleanupInspection{}, referenceErr
|
|
}
|
|
if referenced {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil)
|
|
}
|
|
steps = append(steps, CleanupStepUndefineNetworkFilter)
|
|
digest := sha256.Sum256([]byte(document))
|
|
evidence = append(evidence, "network-filter:"+networkFilterName(uuid)+":"+hex.EncodeToString(digest[:]))
|
|
}
|
|
|
|
nvramExists := false
|
|
for _, name := range managedNVRAMArtifactNames(uuid) {
|
|
path := filepath.Join(l.local.NVRAMDirectory, name)
|
|
info, statErr := os.Lstat(path)
|
|
if errors.Is(statErr, os.ErrNotExist) {
|
|
continue
|
|
}
|
|
if statErr != nil {
|
|
return operationCleanupInspection{}, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr)
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil)
|
|
}
|
|
referenced, referenceErr := l.anyDomainUsesPath(path)
|
|
if domain != nil {
|
|
referenced, referenceErr = l.otherDomainUsesPath(path, uuid)
|
|
}
|
|
if referenceErr != nil {
|
|
return operationCleanupInspection{}, referenceErr
|
|
}
|
|
if referenced {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil)
|
|
}
|
|
nvramExists = true
|
|
evidence = append(evidence, "nvram:"+name+":"+strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10))
|
|
}
|
|
if nvramExists {
|
|
steps = append(steps, CleanupStepDeleteNVRAM)
|
|
}
|
|
trafficPresent := l.traffic.contains(uuid)
|
|
if trafficPresent || len(steps) != 0 {
|
|
// cleanupDeletedVMArtifacts durably rewrites traffic state even if an
|
|
// earlier failed fsync left only an on-disk entry. Make that mutation
|
|
// explicit in every delete cleanup plan.
|
|
steps = append(steps, CleanupStepRemoveTrafficState)
|
|
if trafficPresent {
|
|
evidence = append(evidence, "traffic-state:"+uuid+":present")
|
|
} else {
|
|
evidence = append(evidence, "traffic-state:"+uuid+":rewrite")
|
|
}
|
|
}
|
|
return newOperationCleanupInspection(steps, evidence...), nil
|
|
}
|
|
|
|
func managedFilterDocument(document, expectedName string) bool {
|
|
var filter libvirtxml.NWFilter
|
|
if filter.Unmarshal(document) != nil || filter.Name != expectedName || filter.Chain != "root" {
|
|
return false
|
|
}
|
|
baselineReferences := 0
|
|
for _, entry := range filter.Entries {
|
|
if entry.Ref == nil {
|
|
continue
|
|
}
|
|
if entry.Ref.Filter != "clean-traffic" {
|
|
return false
|
|
}
|
|
baselineReferences++
|
|
}
|
|
return baselineReferences == 1
|
|
}
|
|
|
|
func storageVolumeInspectionDigest(volume *libvirt.StorageVol) (string, error) {
|
|
document, err := volume.GetXMLDesc(0)
|
|
if err != nil {
|
|
return "", translateLibvirtError("read cleanup volume XML", err)
|
|
}
|
|
digest := sha256.Sum256([]byte(document))
|
|
return hex.EncodeToString(digest[:]), nil
|
|
}
|
|
|
|
func (l *Libvirt) inspectReinstallOperation(record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementName := reinstallVolumeName(record.ResourceID, record.OperationID)
|
|
replacement, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementExists := replacement != nil
|
|
if replacement != nil {
|
|
replacement.Free()
|
|
}
|
|
if diskTransitionCommitted(metadata, replacementName, "reinstall", record.OperationID) && replacementExists {
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "reinstall", record.OperationID) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
|
|
}
|
|
result, parseErr := metadataFromXML(document)
|
|
if parseErr != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
result.Status = mapDomainState(state)
|
|
result.ReinstallCommitted = true
|
|
result.CleanupPending = false
|
|
encoded, _ := json.Marshal(result)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
|
|
}
|
|
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "reinstall_not_committed", ErrorMessage: "host evidence confirms the reinstall did not switch the root volume"}, operationCleanupInspection{}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
|
|
func (l *Libvirt) inspectSnapshotOperation(record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
key := parts[1]
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
var stored *domainSnapshotMetadataXML
|
|
for index := range metadata.Snapshots {
|
|
if metadata.Snapshots[index].Key == key {
|
|
stored = &metadata.Snapshots[index]
|
|
break
|
|
}
|
|
}
|
|
volume, err := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, key),
|
|
)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
volumeExists := volume != nil
|
|
if volume != nil {
|
|
defer volume.Free()
|
|
}
|
|
|
|
switch parts[0] {
|
|
case "create-backup":
|
|
if !backupResidueCleanupAllowed(record, stored) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
return operation.Resolution{}, l.backupResidueCleanupInspection(
|
|
domain, volume, snapshotVolumeName(record.ResourceID, key), key, stored,
|
|
)
|
|
case "create-snapshot":
|
|
if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && volumeExists && snapshotConsistencyMatchesRequest(parts[2], stored.Consistency) {
|
|
info, validateErr := validateQCOW2Volume(volume, stored.SizeBytes)
|
|
if validateErr != nil || info.Capacity != stored.SizeBytes {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
result := Snapshot{ID: key, SnapshotKey: key, Name: stored.Name,
|
|
Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(),
|
|
State: "ready", Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, Created: true}
|
|
encoded, _ := json.Marshal(result)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && !volumeExists {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_created", ErrorMessage: "host evidence confirms the snapshot is absent"}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && volumeExists {
|
|
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
|
|
domain, volume, snapshotVolumeName(record.ResourceID, key),
|
|
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
|
|
)
|
|
}
|
|
if stored != nil && !volumeExists {
|
|
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
|
|
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
|
|
)
|
|
}
|
|
case "delete-snapshot":
|
|
if len(parts) != 2 {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && !volumeExists {
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && volumeExists {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_deleted", ErrorMessage: "host evidence confirms the snapshot remains present"}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && !volumeExists {
|
|
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
|
|
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
|
|
)
|
|
}
|
|
if stored == nil && volumeExists {
|
|
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
|
|
domain, volume, snapshotVolumeName(record.ResourceID, key),
|
|
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
|
|
)
|
|
}
|
|
case "restore-snapshot":
|
|
if len(parts) != 2 || stored == nil || !volumeExists {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID)
|
|
replacement, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
|
|
if lookupErr != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementExists := replacement != nil
|
|
if replacement != nil {
|
|
replacement.Free()
|
|
}
|
|
if diskTransitionCommitted(metadata, replacementName, "snapshot_restore", record.OperationID) && replacementExists {
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "snapshot_restore", record.OperationID) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
|
|
}
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Restored: true,
|
|
PowerState: mapDomainState(state), ActiveVolumeName: replacementName, CleanupPending: false})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
|
|
}
|
|
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_restored", ErrorMessage: "host evidence confirms the root volume was not switched"}, operationCleanupInspection{}
|
|
}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
|
|
func backupResidueCleanupAllowed(record operation.Record, stored *domainSnapshotMetadataXML) bool {
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) != 3 || parts[0] != "create-backup" ||
|
|
!catalogIDPattern.MatchString(parts[1]) ||
|
|
(parts[2] != "filesystem" && parts[2] != "crash") {
|
|
return false
|
|
}
|
|
if record.State == operation.StateSucceeded {
|
|
if !successfulBackupCleanupPending(record) {
|
|
return false
|
|
}
|
|
} else if record.State != operation.StateFailed || !record.EffectMayHaveApplied || record.ExecutionFenced {
|
|
return false
|
|
}
|
|
return stored == nil || (stored.Key == parts[1] && stored.SizeBytes > 0 &&
|
|
snapshotConsistencyMatchesRequest(parts[2], stored.Consistency))
|
|
}
|
|
|
|
func successfulBackupCleanupPending(record operation.Record) bool {
|
|
if record.State != operation.StateSucceeded || !recordHasCleanupPending(record) ||
|
|
len(operationEffectProjection(record, record.Result)) == 0 {
|
|
return false
|
|
}
|
|
return strings.HasPrefix(record.Action, "create-backup:")
|
|
}
|
|
|
|
// backupResidueCleanupInspection exposes only an evidence-bound cleanup plan
|
|
// for a local immutable copy left by an already-terminal backup worker. It
|
|
// never asserts whether the remote multipart object committed: failed records
|
|
// therefore remain partial, while successful records retain their immutable
|
|
// applied object projection through appliedBackupInspectionWithCleanup.
|
|
func (l *Libvirt) backupResidueCleanupInspection(domain cleanupJobDomain,
|
|
volume *libvirt.StorageVol, volumeName, key string,
|
|
stored *domainSnapshotMetadataXML) operationCleanupInspection {
|
|
if volume == nil && stored == nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if volume != nil {
|
|
expectedCapacity := uint64(0)
|
|
if stored != nil {
|
|
expectedCapacity = stored.SizeBytes
|
|
}
|
|
if _, err := validateStandaloneQCOW2Volume(volume, expectedCapacity); err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
plan := l.snapshotVolumeCleanupInspection(
|
|
domain, volume, volumeName, CleanupStepDeleteSnapshotVolume,
|
|
"backup-volume:"+volumeName,
|
|
)
|
|
if len(plan.steps) == 0 {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if stored != nil {
|
|
plan.steps = append(plan.steps, CleanupStepRemoveSnapshotMetadata)
|
|
plan.evidence = append(plan.evidence, "backup-metadata:"+key+":"+
|
|
stored.Consistency+":"+strconv.FormatUint(stored.SizeBytes, 10))
|
|
sort.Strings(plan.evidence)
|
|
}
|
|
return plan
|
|
}
|
|
return cleanupInspectionWithIdleDomain(
|
|
domain, []string{CleanupStepRemoveSnapshotMetadata},
|
|
"backup-metadata:"+key+":"+stored.Consistency+":"+
|
|
strconv.FormatUint(stored.SizeBytes, 10),
|
|
)
|
|
}
|
|
|
|
func diskTransitionPowerStateMatches(metadata *domainMetadataXML, state libvirt.DomainState,
|
|
kind, operationID string) bool {
|
|
if !diskTransitionMarkerMatches(metadata, kind, operationID) {
|
|
return true
|
|
}
|
|
if metadata.DiskTransitionRunning {
|
|
return state == libvirt.DOMAIN_RUNNING || state == libvirt.DOMAIN_BLOCKED
|
|
}
|
|
return state == libvirt.DOMAIN_SHUTOFF || state == libvirt.DOMAIN_SHUTDOWN
|
|
}
|
|
|
|
func (l *Libvirt) inspectDiskTransitionCleanupSteps(domain *libvirt.Domain, document string,
|
|
metadata *domainMetadataXML) operationCleanupInspection {
|
|
pending := metadata.PendingCleanupVolume
|
|
if pending == "" || !isVMOwnedRootVolumeName(metadataUUID(document), pending) ||
|
|
pending == metadata.RootVolumeName {
|
|
return operationCleanupInspection{}
|
|
}
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}
|
|
}
|
|
steps := make([]string, 0, 2)
|
|
evidence := append([]string(nil), jobEvidence...)
|
|
evidence = append(evidence, "pending-root:"+pending, "active-root:"+metadata.RootVolumeName)
|
|
old, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), pending)
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if old != nil {
|
|
if _, validateErr := validateQCOW2Volume(old, 0); validateErr != nil {
|
|
old.Free()
|
|
return operationCleanupInspection{}
|
|
}
|
|
oldDigest, digestErr := storageVolumeInspectionDigest(old)
|
|
if digestErr != nil {
|
|
old.Free()
|
|
return operationCleanupInspection{}
|
|
}
|
|
path, pathErr := old.GetPath()
|
|
old.Free()
|
|
if pathErr != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
used, referenceErr := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), pending,
|
|
)
|
|
if referenceErr != nil || used {
|
|
return operationCleanupInspection{}
|
|
}
|
|
steps = append(steps, CleanupStepDeletePreviousRoot)
|
|
evidence = append(evidence, "pending-root-state:present:"+oldDigest)
|
|
} else {
|
|
evidence = append(evidence, "pending-root-state:absent")
|
|
}
|
|
steps = append(steps, CleanupStepPersistCleanupMetadata)
|
|
return newOperationCleanupInspection(steps, evidence...)
|
|
}
|
|
|
|
type cleanupJobDomain interface {
|
|
IsActive() (bool, error)
|
|
GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error)
|
|
GetXMLDesc(libvirt.DomainXMLFlags) (string, error)
|
|
GetBlockJobInfo(string, libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error)
|
|
}
|
|
|
|
func inspectCleanupJobEvidence(domain cleanupJobDomain) ([]string, bool) {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
flags := libvirt.DOMAIN_XML_INACTIVE
|
|
evidence := []string{"domain-runtime:inactive"}
|
|
if active {
|
|
flags = 0
|
|
job, jobErr := domain.GetJobStats(0)
|
|
if jobErr != nil || job == nil || job.Type != libvirt.DOMAIN_JOB_NONE {
|
|
return nil, false
|
|
}
|
|
evidence[0] = "domain-runtime:active:job-none"
|
|
}
|
|
document, err := domain.GetXMLDesc(flags)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
documentDigest := sha256.Sum256([]byte(document))
|
|
evidence = append(evidence, "domain-xml:"+hex.EncodeToString(documentDigest[:]))
|
|
if active {
|
|
persistentDocument, persistentErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if persistentErr != nil {
|
|
return nil, false
|
|
}
|
|
persistentDigest := sha256.Sum256([]byte(persistentDocument))
|
|
evidence = append(evidence, "persistent-domain-xml:"+hex.EncodeToString(persistentDigest[:]))
|
|
}
|
|
targets, err := cleanupDiskTargets(document)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
for _, target := range targets {
|
|
evidence = append(evidence, "disk-target:"+target)
|
|
if !active {
|
|
continue
|
|
}
|
|
info, infoErr := domain.GetBlockJobInfo(target, 0)
|
|
if infoErr != nil || !blockJobInfoAbsent(info) {
|
|
return nil, false
|
|
}
|
|
}
|
|
return evidence, true
|
|
}
|
|
|
|
func cleanupDiskTargets(document string) ([]string, error) {
|
|
var definition libvirtxml.Domain
|
|
if err := definition.Unmarshal(document); err != nil || definition.Devices == nil {
|
|
return nil, errors.New("domain disk evidence is invalid")
|
|
}
|
|
seen := make(map[string]struct{}, len(definition.Devices.Disks))
|
|
for _, disk := range definition.Devices.Disks {
|
|
if disk.Target == nil || disk.Target.Dev == "" {
|
|
return nil, errors.New("domain disk target evidence is incomplete")
|
|
}
|
|
seen[disk.Target.Dev] = struct{}{}
|
|
}
|
|
targets := make([]string, 0, len(seen))
|
|
for target := range seen {
|
|
targets = append(targets, target)
|
|
}
|
|
sort.Strings(targets)
|
|
return targets, nil
|
|
}
|
|
|
|
func blockJobInfoAbsent(info *libvirt.DomainBlockJobInfo) bool {
|
|
// The Go binding does not expose virDomainGetBlockJobInfo's 0/1 return
|
|
// value. A 0 (no job) is represented by the zero-valued info struct.
|
|
return info != nil && info.Type == libvirt.DOMAIN_BLOCK_JOB_TYPE_UNKNOWN &&
|
|
info.Bandwidth == 0 && info.Cur == 0 && info.End == 0
|
|
}
|
|
|
|
func cleanupInspectionWithIdleDomain(domain cleanupJobDomain, steps []string,
|
|
evidence ...string) operationCleanupInspection {
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}
|
|
}
|
|
evidence = append(evidence, jobEvidence...)
|
|
return newOperationCleanupInspection(steps, evidence...)
|
|
}
|
|
|
|
func ensureNoCleanupBlockingJobs(domain cleanupJobDomain) error {
|
|
if _, safe := inspectCleanupJobEvidence(domain); !safe {
|
|
return NewError(ErrConflict, "cleanup_job_not_idle", "domain cleanup is blocked by active or unreadable job state", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) snapshotVolumeCleanupInspection(domain cleanupJobDomain, volume *libvirt.StorageVol,
|
|
volumeName, step, targetEvidence string) operationCleanupInspection {
|
|
if volume == nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if _, err := validateQCOW2Volume(volume, 0); err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
volumeDigest, err := storageVolumeInspectionDigest(volume)
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
path, err := volume.GetPath()
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
referenced, err := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), volumeName,
|
|
)
|
|
if err != nil || referenced {
|
|
return operationCleanupInspection{}
|
|
}
|
|
evidence := append([]string{targetEvidence + ":" + volumeDigest}, jobEvidence...)
|
|
return newOperationCleanupInspection([]string{step}, evidence...)
|
|
}
|
|
|
|
func (l *Libvirt) inspectMatchingBackupJob(record operation.Record, domain *libvirt.Domain) OperationAbortInspection {
|
|
if record.State != operation.StateDispatching &&
|
|
(record.State != operation.StateFailed || !record.EffectMayHaveApplied) {
|
|
return OperationAbortInspection{}
|
|
}
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) != 3 || (parts[0] != "create-snapshot" && parts[0] != "create-backup") ||
|
|
!catalogIDPattern.MatchString(parts[1]) || (parts[2] != "filesystem" && parts[2] != "crash") {
|
|
return OperationAbortInspection{}
|
|
}
|
|
job, err := domain.GetJobStats(0)
|
|
if err != nil || job == nil ||
|
|
(job.Type != libvirt.DOMAIN_JOB_BOUNDED && job.Type != libvirt.DOMAIN_JOB_UNBOUNDED) ||
|
|
!job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
|
|
return OperationAbortInspection{}
|
|
}
|
|
backupXML, err := domain.BackupGetXMLDesc(0)
|
|
if err != nil {
|
|
return OperationAbortInspection{}
|
|
}
|
|
volume, err := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), snapshotVolumeName(record.ResourceID, parts[1]),
|
|
)
|
|
if err != nil || volume == nil {
|
|
return OperationAbortInspection{}
|
|
}
|
|
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
|
|
volume.Free()
|
|
return OperationAbortInspection{}
|
|
}
|
|
targetPath, pathErr := volume.GetPath()
|
|
volume.Free()
|
|
if pathErr != nil || !backupPushMatchesTarget(backupXML, targetPath) {
|
|
return OperationAbortInspection{}
|
|
}
|
|
return OperationAbortInspection{
|
|
Available: true, Kind: "backup",
|
|
InspectionToken: operationInspectionToken(record, "backup", backupXML),
|
|
}
|
|
}
|
|
|
|
func backupPushMatchesTarget(backupXML, targetPath string) bool {
|
|
var backup libvirtxml.DomainBackup
|
|
if backup.Unmarshal(backupXML) != nil || backup.Pull != nil || backup.Push == nil ||
|
|
backup.Push.Disks == nil || len(backup.Push.Disks.Disks) != 1 {
|
|
return false
|
|
}
|
|
disk := backup.Push.Disks.Disks[0]
|
|
return disk.Name == "vda" && disk.Backup == "yes" && disk.Target != nil &&
|
|
disk.Target.File != nil && filepath.Clean(disk.Target.File.File) == filepath.Clean(targetPath)
|
|
}
|
|
|
|
func buildOperationInspection(record operation.Record, resolution operation.Resolution,
|
|
abort OperationAbortInspection, cleanup operationCleanupInspection) OperationInspection {
|
|
result := OperationInspection{
|
|
OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID,
|
|
RequestFingerprint: record.Fingerprint, OperationStatus: record.State,
|
|
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
|
|
SummaryCode: InspectionSummaryIncomplete,
|
|
EffectResolution: InspectionEffectInconclusive,
|
|
AbortJob: abort, Cleanup: OperationCleanupPlan{},
|
|
}
|
|
if abort.Available {
|
|
result.SummaryCode = InspectionSummaryAbortableJob
|
|
return result
|
|
}
|
|
if len(cleanup.steps) != 0 && len(cleanup.evidence) != 0 {
|
|
result.SummaryCode = InspectionSummaryCleanupRequired
|
|
result.EffectResolution = InspectionEffectPartial
|
|
result.Cleanup = OperationCleanupPlan{
|
|
Available: true,
|
|
PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()),
|
|
Steps: append([]string(nil), cleanup.steps...),
|
|
}
|
|
return result
|
|
}
|
|
if !resolution.Resolved {
|
|
return result
|
|
}
|
|
if record.Action == "resize-vm" && !resolution.Succeeded && resolution.ErrorCode == "resources_partially_resized" {
|
|
result.ErrorCode = resolution.ErrorCode
|
|
result.EffectResolution = InspectionEffectPartial
|
|
return result
|
|
}
|
|
if resolution.Succeeded {
|
|
result.Outcome = InspectionOutcomeSucceeded
|
|
result.SummaryCode = InspectionSummarySucceeded
|
|
projection := operationEffectProjection(record, resolution.Result)
|
|
result.EffectResolution = inspectedEffectResolution(record, resolution, projection)
|
|
if result.EffectResolution == InspectionEffectApplied {
|
|
result.EffectProjection = projection
|
|
} else {
|
|
result.Outcome = InspectionOutcomeUncertain
|
|
result.SummaryCode = InspectionSummaryIncomplete
|
|
}
|
|
} else {
|
|
result.Outcome = InspectionOutcomeFailed
|
|
result.SummaryCode = InspectionSummaryFailed
|
|
projection := failedOperationEffectProjection(record, resolution)
|
|
result.EffectResolution = inspectedEffectResolution(record, resolution, projection)
|
|
if result.EffectResolution == InspectionEffectNotApplied {
|
|
result.EffectProjection = projection
|
|
} else {
|
|
result.Outcome = InspectionOutcomeUncertain
|
|
result.SummaryCode = InspectionSummaryIncomplete
|
|
}
|
|
if inspectionErrorCodePattern.MatchString(resolution.ErrorCode) {
|
|
result.ErrorCode = resolution.ErrorCode
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// inspectedEffectResolution separates immutable operation evidence from a
|
|
// read-only observation that merely happens to match the requested target.
|
|
// Power, service, network-link, catalog and storage state may be changed out
|
|
// of band, so those observations remain inconclusive even when the target is
|
|
// currently reached. Only a Journal terminal record or a mutation-specific
|
|
// durable marker may authorize automated ledger convergence.
|
|
func inspectedEffectResolution(record operation.Record, resolution operation.Resolution,
|
|
projection json.RawMessage) string {
|
|
if record.State == operation.StateSucceeded {
|
|
return InspectionEffectApplied
|
|
}
|
|
if record.State == operation.StateFailed && !record.EffectMayHaveApplied {
|
|
return InspectionEffectNotApplied
|
|
}
|
|
if record.State != operation.StateFailed || !record.EffectMayHaveApplied || !resolution.Resolved {
|
|
return InspectionEffectInconclusive
|
|
}
|
|
if resolution.Succeeded {
|
|
if len(projection) == 0 {
|
|
return InspectionEffectInconclusive
|
|
}
|
|
switch {
|
|
case record.Action == "update-settings",
|
|
record.Action == "resize-vm",
|
|
record.Action == "change-network",
|
|
record.Action == "reinstall-vm",
|
|
record.Action == "apply-firewall",
|
|
strings.HasPrefix(record.Action, "restore-snapshot:"):
|
|
return InspectionEffectApplied
|
|
default:
|
|
return InspectionEffectInconclusive
|
|
}
|
|
}
|
|
switch {
|
|
case operation.IsCreateLikeAction(record.Action) && resolution.ErrorCode == "creation_absent" && len(projection) != 0,
|
|
record.Action == "update-settings" && resolution.ErrorCode == "settings_not_committed",
|
|
record.Action == "change-network" && resolution.ErrorCode == "network_not_committed",
|
|
record.Action == "reinstall-vm" && resolution.ErrorCode == "reinstall_not_committed",
|
|
record.Action == "apply-firewall" && resolution.ErrorCode == "firewall_not_committed",
|
|
strings.HasPrefix(record.Action, "restore-snapshot:") && resolution.ErrorCode == "snapshot_not_restored":
|
|
return InspectionEffectNotApplied
|
|
default:
|
|
return InspectionEffectInconclusive
|
|
}
|
|
}
|
|
|
|
func inspectionFromRecord(record operation.Record) OperationInspection {
|
|
switch record.State {
|
|
case operation.StateSucceeded:
|
|
return buildOperationInspection(record, operation.Resolution{
|
|
Resolved: true, Succeeded: true, Result: append([]byte(nil), record.Result...),
|
|
}, OperationAbortInspection{}, operationCleanupInspection{})
|
|
case operation.StateFailed:
|
|
resolution := operation.Resolution{Resolved: true, Succeeded: false, Result: append([]byte(nil), record.Result...)}
|
|
if record.Error != nil {
|
|
resolution.ErrorCode = record.Error.Code
|
|
resolution.ErrorMessage = record.Error.Message
|
|
}
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{})
|
|
}
|
|
return inconclusiveInspectionFromRecord(record)
|
|
}
|
|
|
|
func inconclusiveInspectionFromRecord(record operation.Record) OperationInspection {
|
|
return OperationInspection{
|
|
OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID,
|
|
RequestFingerprint: record.Fingerprint, OperationStatus: record.State,
|
|
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
|
|
SummaryCode: InspectionSummaryIncomplete, EffectResolution: InspectionEffectInconclusive,
|
|
}
|
|
}
|
|
|
|
func failedOperationEffectProjection(record operation.Record, resolution operation.Resolution) json.RawMessage {
|
|
if !operation.IsCreateLikeAction(record.Action) || resolution.ErrorCode != "creation_absent" {
|
|
return nil
|
|
}
|
|
var evidence struct {
|
|
CreationAbsent bool `json:"creation_absent"`
|
|
ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"`
|
|
}
|
|
if !decodeOperationEffectResult(resolution.Result, &evidence) ||
|
|
!evidence.CreationAbsent || !evidence.ManagedArtifactsAbsent {
|
|
return nil
|
|
}
|
|
encoded, err := json.Marshal(evidence)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return encoded
|
|
}
|
|
|
|
const maxOperationEffectProjectionBytes = 64 << 10
|
|
|
|
// operationEffectProjection never forwards Journal bytes. It first decodes a
|
|
// result into the exact response type allowed for the recorded action and then
|
|
// rebuilds a deliberately smaller projection. Unknown actions, malformed or
|
|
// oversized results, and identity mismatches simply omit the optional
|
|
// projection while keeping the independently derived effect resolution.
|
|
func operationEffectProjection(record operation.Record, raw []byte) json.RawMessage {
|
|
if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes || !json.Valid(raw) {
|
|
return nil
|
|
}
|
|
var projection any
|
|
switch {
|
|
case record.Action == "configure-storage":
|
|
var value StorageConfigurationResult
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
Configured bool `json:"configured"`
|
|
}{Configured: value.Configured}
|
|
case strings.HasPrefix(record.Action, "create-backup:"):
|
|
parts := strings.Split(record.Action, ":")
|
|
var value BackupResult
|
|
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
(parts[2] != "filesystem" && parts[2] != "crash") ||
|
|
!decodeOperationEffectResult(raw, &value) || value.BackupKey != parts[1] ||
|
|
value.SizeBytes <= 0 || value.VirtualSizeBytes == 0 ||
|
|
!lowerHexDigest(value.SHA256) ||
|
|
!snapshotConsistencyMatchesRequest(parts[2], value.Consistency) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case strings.HasPrefix(record.Action, "restore-backup:"):
|
|
parts := strings.Split(record.Action, ":")
|
|
var value RestoreBackupResult
|
|
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
!decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID ||
|
|
value.BackupKey != parts[1] || !lowerHexDigest(value.SHA256) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
operationVMEffectProjection
|
|
BackupKey string `json:"backup_key"`
|
|
SHA256 string `json:"sha256"`
|
|
}{
|
|
operationVMEffectProjection: vmEffectProjection(value.VMMetadata),
|
|
BackupKey: value.BackupKey,
|
|
SHA256: value.SHA256,
|
|
}
|
|
case operation.IsCreateLikeAction(record.Action) || record.Action == "update-settings" ||
|
|
record.Action == "change-network" || record.Action == "reinstall-vm":
|
|
var value VMMetadata
|
|
if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID {
|
|
return nil
|
|
}
|
|
projection = vmEffectProjection(value)
|
|
case record.Action == "resize-vm":
|
|
var value ResizeResourcesResult
|
|
expected := record.ExpectedState
|
|
if !decodeOperationEffectResult(raw, &value) || expected == nil || expected.Kind != operation.ExpectedResizeResources ||
|
|
expected.TransitionID != record.OperationID || value.UUID != record.ResourceID || !value.ResourcesResized ||
|
|
value.FromVCPUs != expected.FromVCPUs || value.TargetVCPUs != expected.TargetVCPUs ||
|
|
value.FromMemoryMiB != expected.FromMemoryMiB || value.TargetMemoryMiB != expected.TargetMemoryMiB ||
|
|
value.FromDiskGiB != expected.FromDiskGiB || value.TargetDiskGiB != expected.TargetDiskGiB ||
|
|
value.FromBandwidthMbps != expected.FromBandwidthMbps || value.TargetBandwidthMbps != expected.TargetBandwidthMbps ||
|
|
value.AppliedVCPUs != value.TargetVCPUs || value.AppliedMemoryMiB != value.TargetMemoryMiB ||
|
|
value.AppliedDiskGiB != value.TargetDiskGiB || value.AppliedBandwidthMbps != value.TargetBandwidthMbps ||
|
|
!value.PowerStateRestored ||
|
|
(value.OriginallyRunning && value.Status != VMStateRunning && value.Status != VMStateBlocked) ||
|
|
(!value.OriginallyRunning && value.Status != VMStateShutoff && value.Status != VMStateCrashed) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case record.Action == "delete-vm" || record.Action == "force-delete-vm":
|
|
var value DeleteVMResult
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case strings.HasPrefix(record.Action, "power-") || strings.HasPrefix(record.Action, "service-"):
|
|
var value VMState
|
|
if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
UUID string `json:"uuid"`
|
|
State VMStateName `json:"state"`
|
|
}{UUID: value.UUID, State: value.State}
|
|
case record.Action == "network-suspend" || record.Action == "network-resume":
|
|
var value struct {
|
|
NetworkEnabled bool `json:"network_enabled"`
|
|
}
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case strings.HasPrefix(record.Action, "create-snapshot:"):
|
|
var value Snapshot
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = operationSnapshotEffectProjection{
|
|
ID: value.ID, SnapshotKey: value.SnapshotKey, Name: value.Name,
|
|
CreatedAt: value.CreatedAt, State: value.State, Consistency: value.Consistency,
|
|
SizeBytes: value.SizeBytes, Created: value.Created,
|
|
}
|
|
case strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:"):
|
|
var value SnapshotMutationResult
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
SnapshotKey string `json:"snapshot_key"`
|
|
Restored bool `json:"restored,omitempty"`
|
|
Deleted bool `json:"deleted,omitempty"`
|
|
PowerState VMStateName `json:"power_state,omitempty"`
|
|
CleanupPending bool `json:"cleanup_pending,omitempty"`
|
|
}{value.SnapshotKey, value.Restored, value.Deleted, value.PowerState, value.CleanupPending}
|
|
case record.Action == "apply-firewall":
|
|
var value FirewallPolicy
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
Revision uint64 `json:"revision"`
|
|
DefaultIn string `json:"default_in"`
|
|
DefaultOut string `json:"default_out"`
|
|
PolicyDigest string `json:"policy_digest,omitempty"`
|
|
Rules []FirewallRule `json:"rules"`
|
|
}{value.Revision, value.DefaultIn, value.DefaultOut, value.PolicyDigest, append([]FirewallRule(nil), value.Rules...)}
|
|
case record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
|
|
strings.HasPrefix(record.Action, "exit-rescue:"):
|
|
var value RescueState
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
Active bool `json:"active"`
|
|
Staged bool `json:"staged"`
|
|
PendingManualRestart bool `json:"pending_manual_restart,omitempty"`
|
|
TemplateID string `json:"template_id,omitempty"`
|
|
NetworkSuspended bool `json:"network_suspended"`
|
|
}{value.Active, value.Staged, value.PendingManualRestart, value.TemplateID, value.NetworkSuspended}
|
|
case record.Action == "import-template" || record.Action == "update-template":
|
|
var value ImageTemplate
|
|
if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "template:"+value.ID {
|
|
return nil
|
|
}
|
|
projection = operationCatalogImportProjection{
|
|
ID: value.ID, Name: value.Name, OSFamily: value.OSFamily, Version: value.Version,
|
|
Architecture: value.Architecture, SizeBytes: value.SizeBytes,
|
|
VirtualSizeBytes: value.VirtualSizeBytes, SHA256: value.SHA256,
|
|
Revision: value.Revision, GenerationToken: value.GenerationToken, CreatedAt: value.CreatedAt,
|
|
}
|
|
case record.Action == "import-rescue-template":
|
|
var value RescueTemplate
|
|
if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "rescue-template:"+value.ID {
|
|
return nil
|
|
}
|
|
projection = operationCatalogImportProjection{
|
|
ID: value.ID, Name: value.Name, SizeBytes: value.SizeBytes,
|
|
SHA256: value.SHA256, CreatedAt: value.CreatedAt,
|
|
}
|
|
case record.Action == "import-user-iso":
|
|
var value UserISO
|
|
if !decodeOperationEffectResult(raw, &value) ||
|
|
record.ResourceID != "user-iso:"+strconv.FormatInt(value.OwnerID, 10)+":"+value.ID {
|
|
return nil
|
|
}
|
|
projection = operationCatalogImportProjection{
|
|
ID: value.ID, OwnerID: value.OwnerID, Name: value.Name,
|
|
SizeBytes: value.SizeBytes, SHA256: value.SHA256, CreatedAt: value.CreatedAt,
|
|
}
|
|
case record.Action == "delete-template" || record.Action == "delete-rescue-template" ||
|
|
record.Action == "delete-user-iso":
|
|
var value CatalogDeleteResult
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
default:
|
|
return nil
|
|
}
|
|
encoded, err := json.Marshal(projection)
|
|
if err != nil || len(encoded) == 0 || len(encoded) > maxOperationEffectProjectionBytes {
|
|
return nil
|
|
}
|
|
return encoded
|
|
}
|
|
|
|
type operationVMEffectProjection struct {
|
|
UUID string `json:"uuid"`
|
|
Name string `json:"name,omitempty"`
|
|
Status VMStateName `json:"status,omitempty"`
|
|
VCPUs uint `json:"vcpu,omitempty"`
|
|
MemoryMiB uint64 `json:"memory_mib,omitempty"`
|
|
DiskGiB uint64 `json:"disk_gib,omitempty"`
|
|
BandwidthMbps uint64 `json:"bandwidth_mbps,omitempty"`
|
|
Hostname string `json:"hostname,omitempty"`
|
|
PublicIPCIDR string `json:"public_ip_cidr,omitempty"`
|
|
PublicPoolKey string `json:"public_pool_key,omitempty"`
|
|
PublicPoolCIDR string `json:"public_pool_cidr,omitempty"`
|
|
Gateway string `json:"gateway,omitempty"`
|
|
DNSServers []string `json:"dns_servers,omitempty"`
|
|
NetworkSelector string `json:"network,omitempty"`
|
|
TemplateID string `json:"template_id,omitempty"`
|
|
Machine string `json:"machine,omitempty"`
|
|
Firmware string `json:"firmware,omitempty"`
|
|
SecureBoot bool `json:"secure_boot,omitempty"`
|
|
SettingsChanged bool `json:"settings_changed,omitempty"`
|
|
NetworkChanged bool `json:"network_changed,omitempty"`
|
|
ReinstallCommitted bool `json:"reinstall_committed,omitempty"`
|
|
CleanupPending bool `json:"cleanup_pending,omitempty"`
|
|
}
|
|
|
|
func vmEffectProjection(value VMMetadata) operationVMEffectProjection {
|
|
return operationVMEffectProjection{
|
|
UUID: value.UUID, Name: value.Name, Status: value.Status,
|
|
VCPUs: value.VCPUs, MemoryMiB: value.MemoryMiB, DiskGiB: value.DiskGiB,
|
|
BandwidthMbps: value.BandwidthMbps, Hostname: value.Hostname,
|
|
PublicIPCIDR: value.PublicIPCIDR, PublicPoolKey: value.PublicPoolKey,
|
|
PublicPoolCIDR: value.PublicPoolCIDR, Gateway: value.Gateway,
|
|
DNSServers: append([]string(nil), value.DNSServers...),
|
|
NetworkSelector: value.NetworkSelector, TemplateID: value.TemplateID,
|
|
Machine: value.Machine, Firmware: value.Firmware, SecureBoot: value.SecureBoot,
|
|
SettingsChanged: value.SettingsChanged, NetworkChanged: value.NetworkChanged,
|
|
ReinstallCommitted: value.ReinstallCommitted, CleanupPending: value.CleanupPending,
|
|
}
|
|
}
|
|
|
|
type operationSnapshotEffectProjection struct {
|
|
ID string `json:"id"`
|
|
SnapshotKey string `json:"snapshot_key"`
|
|
Name string `json:"name,omitempty"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
State string `json:"state"`
|
|
Consistency string `json:"consistency"`
|
|
SizeBytes uint64 `json:"size_bytes,omitempty"`
|
|
Created bool `json:"created,omitempty"`
|
|
}
|
|
|
|
type operationCatalogImportProjection struct {
|
|
ID string `json:"id"`
|
|
OwnerID int64 `json:"owner_id,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
OSFamily string `json:"os_family,omitempty"`
|
|
Version string `json:"version,omitempty"`
|
|
Architecture string `json:"architecture,omitempty"`
|
|
SizeBytes uint64 `json:"size_bytes,omitempty"`
|
|
VirtualSizeBytes uint64 `json:"virtual_size_bytes,omitempty"`
|
|
SHA256 string `json:"sha256,omitempty"`
|
|
Revision uint64 `json:"revision,omitempty"`
|
|
GenerationToken string `json:"generation_token,omitempty"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
}
|
|
|
|
func decodeOperationEffectResult(raw []byte, target any) bool {
|
|
if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes {
|
|
return false
|
|
}
|
|
trimmed := strings.TrimSpace(string(raw))
|
|
return len(trimmed) >= 2 && trimmed[0] == '{' && trimmed[len(trimmed)-1] == '}' &&
|
|
json.Unmarshal(raw, target) == nil
|
|
}
|
|
|
|
func recordHasCleanupPending(record operation.Record) bool {
|
|
if len(record.Result) == 0 {
|
|
return false
|
|
}
|
|
var result struct {
|
|
CleanupPending bool `json:"cleanup_pending"`
|
|
}
|
|
return json.Unmarshal(record.Result, &result) == nil && result.CleanupPending
|
|
}
|
|
|
|
func operationInspectionToken(record operation.Record, kind, evidence string) string {
|
|
digest := sha256.New()
|
|
for _, part := range []string{
|
|
"virt-cloud-operation-inspection-v1", record.OperationID, record.ResourceID,
|
|
record.Action, record.Fingerprint, kind, evidence,
|
|
} {
|
|
digest.Write([]byte(part))
|
|
digest.Write([]byte{0})
|
|
}
|
|
return hex.EncodeToString(digest.Sum(nil))
|
|
}
|
|
|
|
func lowerHexDigest(value string) bool {
|
|
if len(value) != sha256.Size*2 {
|
|
return false
|
|
}
|
|
for _, current := range value {
|
|
if (current < '0' || current > '9') && (current < 'a' || current > 'f') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|