You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2025 lines
79 KiB
Go
2025 lines
79 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"git.luoli.live/root/virt/agent/peermigration"
|
|
"git.luoli.live/root/virt/agent/publicnetwork"
|
|
"git.luoli.live/root/virt/agent/vpcrouting"
|
|
"libvirt.org/go/libvirt"
|
|
"libvirt.org/go/libvirtxml"
|
|
)
|
|
|
|
var inspectionErrorCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
|
|
|
|
// operationCleanupInspection carries the exact, private host evidence used to
|
|
// bind a cleanup plan. Only the stable step codes are serialized; resource
|
|
// names, XML digests, and file attributes stay inside the Agent and feed the
|
|
// one-way plan identifier.
|
|
type operationCleanupInspection struct {
|
|
steps []string
|
|
evidence []string
|
|
}
|
|
|
|
func newOperationCleanupInspection(steps []string, evidence ...string) operationCleanupInspection {
|
|
if len(evidence) == 0 {
|
|
return operationCleanupInspection{}
|
|
}
|
|
normalizedEvidence := append([]string(nil), evidence...)
|
|
sort.Strings(normalizedEvidence)
|
|
return operationCleanupInspection{
|
|
steps: append([]string(nil), steps...), evidence: normalizedEvidence,
|
|
}
|
|
}
|
|
|
|
func (inspection operationCleanupInspection) tokenEvidence() string {
|
|
encoded, _ := json.Marshal(struct {
|
|
Steps []string `json:"steps"`
|
|
Evidence []string `json:"evidence"`
|
|
}{Steps: inspection.steps, Evidence: inspection.evidence})
|
|
return string(encoded)
|
|
}
|
|
|
|
// InspectOperation deliberately bypasses Journal admission. A dispatched
|
|
// libvirt call retains its resource lease until the real call returns, but an
|
|
// administrator must still be able to read its state and identify an owned
|
|
// cancellable backup job while that call is stuck. Every operation below is a
|
|
// libvirt/filesystem read. It may return a cleanup plan, but inspection never
|
|
// executes that plan or changes a host resource.
|
|
func (l *Libvirt) InspectOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
if successfulBackupCleanupPending(record) {
|
|
if record.InspectionEvidenceUnavailable {
|
|
return inconclusiveInspectionFromRecord(record), nil
|
|
}
|
|
return l.inspectSuccessfulBackupCleanupPending(ctx, record), nil
|
|
}
|
|
if storedOperationInspectionAvailable(record) {
|
|
if record.InspectionEvidenceUnavailable {
|
|
return inconclusiveInspectionFromRecord(record), nil
|
|
}
|
|
inspection := inspectionFromRecord(record)
|
|
// A recovered successful receipt is useful for convergence only when its
|
|
// action-specific result can be reduced to the bounded public projection.
|
|
// Receipt status alone must not turn an unprojectable or damaged result
|
|
// into an "applied" verdict after Journal detail retention.
|
|
if record.InspectionReceiptRecovered && record.State == operation.StateSucceeded &&
|
|
len(inspection.EffectProjection) == 0 {
|
|
return inconclusiveInspectionFromRecord(record), nil
|
|
}
|
|
return inspection, nil
|
|
}
|
|
|
|
if resolution, handled, err := l.reconcileStorageConfiguration(record); handled {
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
|
|
}
|
|
if resolution, handled, err := l.reconcileCatalogOperation(ctx, record); handled {
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
|
|
}
|
|
if resolution, handled, err := l.reconcileVPCNetwork(record); handled {
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{}), err
|
|
}
|
|
|
|
domain, err := l.domainByUUID(ctx, record.ResourceID)
|
|
if errors.Is(err, ErrNotFound) {
|
|
return l.inspectAbsentDomainOperation(ctx, record)
|
|
}
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
defer domain.Free()
|
|
|
|
abort := l.inspectMatchingBackupJob(record, domain)
|
|
resolution, cleanup, inspectErr := l.inspectExistingDomainOperation(ctx, record, domain)
|
|
if abort.Available {
|
|
// No cleanup may be approved while the matching backup is still live.
|
|
cleanup = operationCleanupInspection{}
|
|
}
|
|
return buildOperationInspection(record, resolution, abort, cleanup), inspectErr
|
|
}
|
|
|
|
func storedOperationInspectionAvailable(record operation.Record) bool {
|
|
if record.Tombstone {
|
|
return true
|
|
}
|
|
if record.State == operation.StateSucceeded && strings.HasPrefix(record.Action, "create-backup:") {
|
|
// A successful backup result is immutable proof that the remote object
|
|
// committed, even when its optional cleanup_pending marker reports only
|
|
// local residue. Never discard the sole object SHA/size projection by
|
|
// replacing it with a weaker current-host observation.
|
|
return true
|
|
}
|
|
if record.State == operation.StateSucceeded && !recordHasCleanupPending(record) {
|
|
return true
|
|
}
|
|
if record.State == operation.StateFailed && record.EffectMayHaveApplied {
|
|
// A failed Journal row describes the response observed by the original
|
|
// caller, not necessarily the durable host effect. Reuse would make the
|
|
// operation look definitively failed forever and bypass the read-only
|
|
// expected-state reconcilers below.
|
|
return false
|
|
}
|
|
return record.State == operation.StateFailed && !operation.IsCreateLikeAction(record.Action) &&
|
|
!operation.RequiresExplicitForceDelete(record)
|
|
}
|
|
|
|
// inspectSuccessfulBackupCleanupPending preserves the immutable applied
|
|
// object projection while adding an optional, read-only cleanup plan for an
|
|
// exact operation-owned local workspace residue. Host read failures never
|
|
// downgrade the already verified remote object identity.
|
|
func (l *Libvirt) inspectSuccessfulBackupCleanupPending(ctx context.Context,
|
|
record operation.Record) OperationInspection {
|
|
inspection := inspectionFromRecord(record)
|
|
domain, err := l.domainByUUID(ctx, record.ResourceID)
|
|
if errors.Is(err, ErrNotFound) {
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) {
|
|
return inspection
|
|
}
|
|
volumeName, volume, _, residueErr := l.backupWorkspaceResidue(
|
|
record.ResourceID, parts[1],
|
|
)
|
|
if residueErr != nil || volume == nil {
|
|
return inspection
|
|
}
|
|
defer volume.Free()
|
|
cleanup := l.absentDomainBackupWorkspaceCleanupInspection(
|
|
volume, volumeName,
|
|
)
|
|
return appliedBackupInspectionWithCleanup(
|
|
record, inspection, cleanup,
|
|
)
|
|
}
|
|
if err != nil {
|
|
return inspection
|
|
}
|
|
defer domain.Free()
|
|
|
|
_, cleanup := l.inspectSnapshotOperation(ctx, record, domain)
|
|
return appliedBackupInspectionWithCleanup(record, inspection, cleanup)
|
|
}
|
|
|
|
func appliedBackupInspectionWithCleanup(record operation.Record, inspection OperationInspection,
|
|
cleanup operationCleanupInspection) OperationInspection {
|
|
if len(cleanup.steps) == 0 || len(cleanup.evidence) == 0 {
|
|
return inspection
|
|
}
|
|
inspection.SummaryCode = InspectionSummaryCleanupRequired
|
|
inspection.Cleanup = OperationCleanupPlan{
|
|
Available: true,
|
|
PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()),
|
|
Steps: append([]string(nil), cleanup.steps...),
|
|
}
|
|
return inspection
|
|
}
|
|
|
|
func (l *Libvirt) inspectAbsentDomainOperation(ctx context.Context, record operation.Record) (OperationInspection, error) {
|
|
var resolution operation.Resolution
|
|
var cleanup operationCleanupInspection
|
|
switch {
|
|
case strings.HasPrefix(record.Action, "receive-peer-migration:"):
|
|
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil)
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
if len(plan.steps) != 0 {
|
|
cleanup = plan
|
|
break
|
|
}
|
|
resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, true, false)
|
|
case operation.IsCreateLikeAction(record.Action):
|
|
absent, err := l.deleteArtifactsAbsent(record.ResourceID)
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
resolution, cleanup = inspectAbsentCreateVM(record.ResourceID, absent, l.traffic.contains(record.ResourceID))
|
|
case record.Action == "delete-vm" || record.Action == "force-delete-vm" ||
|
|
strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"):
|
|
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, nil, nil)
|
|
if err != nil {
|
|
return OperationInspection{}, err
|
|
}
|
|
cleanup = plan
|
|
resolution = inspectAbsentDeleteResolution(plan, record.Action == "force-delete-vm")
|
|
case strings.HasPrefix(record.Action, "cleanup-backup-workspace:"):
|
|
resolution = l.reconcileAbsentDomainBackupWorkspaceCleanup(record)
|
|
default:
|
|
if record.ExpectedState != nil {
|
|
resolution = expectedStateResourceAbsent(record)
|
|
}
|
|
}
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, cleanup), nil
|
|
}
|
|
|
|
func (l *Libvirt) reconcileAbsentDomainBackupWorkspaceCleanup(
|
|
record operation.Record,
|
|
) operation.Resolution {
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) {
|
|
return operation.Resolution{}
|
|
}
|
|
_, volume, _, err := l.backupWorkspaceResidue(
|
|
record.ResourceID, parts[1],
|
|
)
|
|
if err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
if volume == nil {
|
|
result, _ := json.Marshal(CleanupBackupWorkspaceResult{
|
|
BackupKey: parts[1], Cleaned: true,
|
|
WorkspaceAbsent: true, MetadataAbsent: true,
|
|
})
|
|
return operation.Resolution{
|
|
Resolved: true, Succeeded: true, Result: result,
|
|
}
|
|
}
|
|
defer volume.Free()
|
|
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
|
|
return operation.Resolution{}
|
|
}
|
|
return operation.Resolution{
|
|
Resolved: true, Succeeded: false,
|
|
ErrorCode: "backup_workspace_cleanup_not_applied",
|
|
ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied",
|
|
}
|
|
}
|
|
|
|
func inspectAbsentCreateVM(uuid string, managedArtifactsAbsent, trafficStatePresent bool) (operation.Resolution, operationCleanupInspection) {
|
|
if !managedArtifactsAbsent {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
resolution := operation.Resolution{
|
|
Resolved: true, Succeeded: false,
|
|
Result: []byte(`{"creation_absent":true,"managed_artifacts_absent":true}`),
|
|
ErrorCode: "creation_absent",
|
|
ErrorMessage: "host evidence confirms the VM and managed artifacts are absent",
|
|
}
|
|
if trafficStatePresent {
|
|
return resolution, newOperationCleanupInspection(
|
|
[]string{CleanupStepRemoveTrafficState}, "traffic-state:"+uuid,
|
|
)
|
|
}
|
|
return resolution, operationCleanupInspection{}
|
|
}
|
|
|
|
func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection, error) {
|
|
if strings.HasPrefix(record.Action, "cleanup-peer-migration-target:") {
|
|
// This successor is authorized only to clean an absent target. A domain
|
|
// appearing before or after dispatch is never converted into a delete
|
|
// plan by reconciliation.
|
|
return operation.Resolution{}, operationCleanupInspection{}, nil
|
|
}
|
|
if record.Action == "delete-vm" || record.Action == "force-delete-vm" {
|
|
_, managed, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, err
|
|
}
|
|
if err := l.validateDeleteVMArtifacts(ctx, record.ResourceID, managed); err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, err
|
|
}
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
|
|
}
|
|
if resolution, handled := inspectActiveDeleteResolution(record, active); handled {
|
|
return resolution, operationCleanupInspection{}, nil
|
|
}
|
|
plan, err := l.inspectDeleteCleanupSteps(ctx, record.ResourceID, domain, managed)
|
|
return operation.Resolution{}, plan, err
|
|
}
|
|
if record.ExpectedState != nil {
|
|
resolution, handled, err := l.inspectExpectedStateOperation(ctx, record, domain)
|
|
if handled {
|
|
return resolution, operationCleanupInspection{}, err
|
|
}
|
|
}
|
|
if strings.HasPrefix(record.Action, "prepare-peer-migration:") {
|
|
resolution, err := l.reconcilePreparedPeerMigration(record, domain)
|
|
return resolution, operationCleanupInspection{}, err
|
|
}
|
|
if strings.HasPrefix(record.Action, "receive-peer-migration:") {
|
|
resolution, err := l.reconcileReceivedPeerMigration(ctx, record, domain)
|
|
return resolution, operationCleanupInspection{}, err
|
|
}
|
|
if record.Action == "reinstall-vm" {
|
|
resolution, steps := l.inspectReinstallOperation(record, domain)
|
|
return resolution, steps, nil
|
|
}
|
|
if strings.HasPrefix(record.Action, "create-snapshot:") ||
|
|
strings.HasPrefix(record.Action, "create-backup:") ||
|
|
strings.HasPrefix(record.Action, "cleanup-backup-workspace:") ||
|
|
strings.HasPrefix(record.Action, "delete-snapshot:") ||
|
|
strings.HasPrefix(record.Action, "restore-snapshot:") ||
|
|
strings.HasPrefix(record.Action, "recover-backup:") {
|
|
resolution, steps := l.inspectSnapshotOperation(ctx, record, domain)
|
|
return resolution, steps, nil
|
|
}
|
|
if record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
|
|
strings.HasPrefix(record.Action, "exit-rescue:") {
|
|
return l.reconcileRescue(ctx, record, domain), operationCleanupInspection{}, nil
|
|
}
|
|
state, _, err := domain.GetState()
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}, translateLibvirtError("inspect operation domain state", err)
|
|
}
|
|
mapped := mapDomainState(state)
|
|
switch {
|
|
case operation.IsCreateLikeAction(record.Action):
|
|
resolution, err := l.reconcileCreatedVM(domain)
|
|
return resolution, operationCleanupInspection{}, err
|
|
case record.Action == "power-start":
|
|
if mapped == VMStateRunning || mapped == VMStateBlocked {
|
|
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
|
|
}
|
|
case record.Action == "power-shutdown" || record.Action == "power-destroy":
|
|
if mapped == VMStateShutoff {
|
|
return operation.Resolution{Resolved: true, Succeeded: true}, operationCleanupInspection{}, nil
|
|
}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}, nil
|
|
}
|
|
|
|
func peerMigrationActionID(action, prefix string) (string, bool) {
|
|
parts := strings.Split(action, ":")
|
|
returnValue := ""
|
|
if len(parts) == 2 && parts[0] == prefix && catalogIDPattern.MatchString(parts[1]) {
|
|
returnValue = parts[1]
|
|
}
|
|
return returnValue, returnValue != ""
|
|
}
|
|
|
|
func (l *Libvirt) reconcilePreparedPeerMigration(record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, error) {
|
|
migrationID, ok := peerMigrationActionID(record.Action, "prepare-peer-migration")
|
|
if !ok {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
store, err := peermigration.NewStore(
|
|
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
|
|
l.local.PeerMigrationStagingPath,
|
|
)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
manifest, manifestErr := store.LoadSource(migrationID)
|
|
if manifestErr == nil {
|
|
autostart, autoErr := domain.GetAutostart()
|
|
active, activeErr := domain.IsActive()
|
|
if autoErr != nil || activeErr != nil {
|
|
return operation.Resolution{}, errors.Join(autoErr, activeErr)
|
|
}
|
|
if autostart || active || !strings.EqualFold(manifest.SourceUUID, record.ResourceID) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
digest, err := manifest.Public().Digest()
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
result, err := json.Marshal(PreparePeerMigrationResult{
|
|
MigrationID: migrationID, ManifestDigest: digest,
|
|
Artifacts: manifest.Public().Artifacts, SourceStopped: true,
|
|
SourceWasRunning: manifest.SourceWasRunning,
|
|
SourceAutostartEnabled: manifest.SourceAutostartEnabled,
|
|
SourceAutostartDisabled: true,
|
|
})
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, nil
|
|
}
|
|
if !errors.Is(manifestErr, os.ErrNotExist) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
intent, intentErr := store.LoadSourceIntent(migrationID)
|
|
if errors.Is(intentErr, os.ErrNotExist) {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "peer_migration_prepare_not_applied",
|
|
ErrorMessage: "source intent was not persisted before the mutation boundary"}, nil
|
|
}
|
|
if intentErr != nil || !strings.EqualFold(intent.SourceUUID, record.ResourceID) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
autostart, autoErr := domain.GetAutostart()
|
|
active, activeErr := domain.IsActive()
|
|
if autoErr != nil || activeErr != nil {
|
|
return operation.Resolution{}, errors.Join(autoErr, activeErr)
|
|
}
|
|
if autostart == intent.SourceAutostartEnabled && active == intent.SourceWasRunning {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "peer_migration_prepare_not_applied",
|
|
ErrorMessage: "source state is restored and no ready manifest exists"}, nil
|
|
}
|
|
return operation.Resolution{}, nil
|
|
}
|
|
|
|
func (l *Libvirt) reconcileReceivedPeerMigration(_ context.Context,
|
|
record operation.Record, domain *libvirt.Domain) (operation.Resolution, error) {
|
|
migrationID, ok := peerMigrationActionID(record.Action, "receive-peer-migration")
|
|
if !ok {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
store, err := peermigration.NewStore(
|
|
l.local.PeerMigrationStatePath, l.local.StoragePathRoots,
|
|
l.local.PeerMigrationStagingPath,
|
|
)
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
receipt, err := store.LoadTargetReceipt(migrationID)
|
|
if err != nil || !strings.EqualFold(receipt.TargetUUID, record.ResourceID) {
|
|
return operation.Resolution{}, nil
|
|
}
|
|
autostart, autoErr := domain.GetAutostart()
|
|
active, activeErr := domain.IsActive()
|
|
if autoErr != nil || activeErr != nil || autostart || active {
|
|
return operation.Resolution{}, errors.Join(autoErr, activeErr)
|
|
}
|
|
vm, complete, err := l.reconcileStagedPeerMigrationVM(domain, receipt)
|
|
if err != nil || !complete {
|
|
return operation.Resolution{}, err
|
|
}
|
|
result, err := json.Marshal(ReceivePeerMigrationResult{
|
|
VMMetadata: vm, MigrationID: migrationID,
|
|
ManifestDigest: receipt.ManifestDigest,
|
|
Artifacts: receipt.Artifacts, StagedPoweredOff: true,
|
|
})
|
|
if err != nil {
|
|
return operation.Resolution{}, err
|
|
}
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, nil
|
|
}
|
|
|
|
func (l *Libvirt) reconcileStagedPeerMigrationVM(domain *libvirt.Domain,
|
|
receipt peermigration.TargetReceipt) (VMMetadata, bool, error) {
|
|
document, definition, metadata, pool, root, err := l.managedRootContext(domain)
|
|
if err != nil {
|
|
return VMMetadata{}, false, nil
|
|
}
|
|
defer pool.Free()
|
|
defer root.Free()
|
|
autostart, autostartErr := domain.GetAutostart()
|
|
active, activeErr := domain.IsActive()
|
|
state, _, stateErr := domain.GetState()
|
|
if autostartErr != nil || activeErr != nil || stateErr != nil {
|
|
return VMMetadata{}, false, errors.Join(autostartErr, activeErr, stateErr)
|
|
}
|
|
if autostart || active || state != libvirt.DOMAIN_SHUTOFF {
|
|
return VMMetadata{}, false, nil
|
|
}
|
|
rootInfo, err := validateStandaloneQCOW2Volume(root, 0)
|
|
if err != nil {
|
|
return VMMetadata{}, false, nil
|
|
}
|
|
var rootArtifact *peermigration.Artifact
|
|
snapshotArtifacts := make(map[string]peermigration.Artifact)
|
|
for index := range receipt.Artifacts {
|
|
artifact := &receipt.Artifacts[index]
|
|
if artifact.Role == "root" {
|
|
rootArtifact = artifact
|
|
continue
|
|
}
|
|
snapshotArtifacts[artifact.SnapshotKey] = *artifact
|
|
}
|
|
if rootArtifact == nil || rootInfo.Capacity < rootArtifact.VirtualSizeBytes ||
|
|
len(metadata.Snapshots) != len(snapshotArtifacts) {
|
|
return VMMetadata{}, false, nil
|
|
}
|
|
for _, stored := range metadata.Snapshots {
|
|
artifact, ok := snapshotArtifacts[stored.Key]
|
|
if !ok || stored.Name != artifact.Name || stored.Consistency != artifact.Consistency ||
|
|
stored.SizeBytes != artifact.VirtualSizeBytes || stored.CreatedAt != artifact.CreatedAt {
|
|
return VMMetadata{}, false, nil
|
|
}
|
|
volume, lookupErr := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged),
|
|
snapshotVolumeName(definition.UUID, stored.Key),
|
|
)
|
|
if lookupErr != nil {
|
|
return VMMetadata{}, false, lookupErr
|
|
}
|
|
if volume == nil {
|
|
return VMMetadata{}, false, nil
|
|
}
|
|
_, inspectErr := validateStandaloneQCOW2Volume(volume, artifact.VirtualSizeBytes)
|
|
volume.Free()
|
|
if inspectErr != nil {
|
|
return VMMetadata{}, false, nil
|
|
}
|
|
}
|
|
seed, err := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleSeed), seedVolumeName(definition.UUID),
|
|
)
|
|
if err != nil || seed == nil {
|
|
return VMMetadata{}, false, err
|
|
}
|
|
seedPath, seedErr := seed.GetPath()
|
|
seed.Free()
|
|
if seedErr != nil || definition.Devices == nil {
|
|
return VMMetadata{}, false, seedErr
|
|
}
|
|
seedAttached := false
|
|
filterName := networkFilterName(definition.UUID)
|
|
filterAttached := managedNetworkLayoutMatches(definition, metadata)
|
|
for _, disk := range definition.Devices.Disks {
|
|
if disk.Device == "cdrom" && disk.Target != nil && disk.Target.Dev == "sda" &&
|
|
disk.Source != nil && disk.Source.File != nil &&
|
|
disk.Source.File.File == seedPath && disk.ReadOnly != nil {
|
|
seedAttached = true
|
|
}
|
|
}
|
|
var policy PlatformNetworkPolicySnapshot
|
|
if !seedAttached || !filterAttached ||
|
|
json.Unmarshal([]byte(metadata.PlatformPolicyJSON), &policy) != nil ||
|
|
ValidatePlatformNetworkPolicy(policy) != nil ||
|
|
l.verifyManagedFilter(filterName, policy) != nil {
|
|
return VMMetadata{}, false, nil
|
|
}
|
|
vm, err := metadataFromXML(document)
|
|
if err != nil {
|
|
return VMMetadata{}, false, nil
|
|
}
|
|
vm.Status = VMStateShutoff
|
|
return vm, true, nil
|
|
}
|
|
|
|
func inspectActiveDeleteResolution(record operation.Record, active bool) (operation.Resolution, bool) {
|
|
if !active {
|
|
return operation.Resolution{}, false
|
|
}
|
|
if record.Action == "delete-vm" && operation.RequiresExplicitForceDelete(record) {
|
|
// Preserve an authoritative force_required outcome written by the
|
|
// original graceful DeleteVM call after its bounded wait.
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "force_required", ErrorMessage: "guest remains active and requires an explicitly authorized force delete"}, true
|
|
}
|
|
if record.Action == "delete-vm" &&
|
|
(record.CauseCode == "graceful_shutdown_ambiguous" ||
|
|
recoveredGracefulAttemptRequiresRetry(record)) {
|
|
// The prior ShutdownFlags response was lost and activity could not be
|
|
// observed conclusively, or the Agent restarted at the dispatch
|
|
// boundary before it persisted any cause. A later active observation
|
|
// can classify the already-terminal operation as failed, but cannot grant
|
|
// force authorization or dispatch another mutation.
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "graceful_shutdown_unconfirmed",
|
|
ErrorMessage: "graceful shutdown result was not confirmed; the resource remains blocked for manual remediation"}, true
|
|
}
|
|
if record.Action == "force-delete-vm" && forceDeleteCanBeConfirmedForRetry(record) {
|
|
// Never replay DestroyFlags automatically. Once no native call can still
|
|
// hold Journal admission, a token-confirmed administrator inspection may
|
|
// close an old force attempt which still observes an active Domain. The
|
|
// original graceful force_required record remains the authorization
|
|
// anchor, so a retry still needs a new operation ID and data-loss ack.
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "force_delete_unconfirmed_active",
|
|
ErrorMessage: "force delete did not reach an inactive domain; inspect and explicitly authorize a new force operation"}, true
|
|
}
|
|
// A force operation may already have crossed Destroy before its result was
|
|
// lost. Read-only inspection must not turn that ambiguity into a cleanup
|
|
// plan which would replay Destroy.
|
|
return operation.Resolution{}, true
|
|
}
|
|
|
|
func forceDeleteCanBeConfirmedForRetry(record operation.Record) bool {
|
|
return record.State == operation.StateFailed && record.EffectMayHaveApplied
|
|
}
|
|
|
|
func recoveredGracefulAttemptRequiresRetry(record operation.Record) bool {
|
|
if record.CauseCode != "" {
|
|
return false
|
|
}
|
|
return record.State == operation.StateFailed && record.EffectMayHaveApplied
|
|
}
|
|
|
|
func inspectAbsentDeleteResolution(cleanup operationCleanupInspection, forced bool) operation.Resolution {
|
|
if len(cleanup.steps) != 0 {
|
|
return operation.Resolution{}
|
|
}
|
|
return resolvedDeleteSuccess(DeleteVMResult{
|
|
Deleted: true, ManagedArtifactsAbsent: true, Forced: forced,
|
|
})
|
|
}
|
|
|
|
func (l *Libvirt) inspectDeleteCleanupSteps(ctx context.Context, uuid string,
|
|
domain *libvirt.Domain, managed *domainMetadataXML) (operationCleanupInspection, error) {
|
|
steps := make([]string, 0, 7)
|
|
evidence := make([]string, 0, 8)
|
|
if domain != nil {
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}, nil
|
|
}
|
|
evidence = append(evidence, jobEvidence...)
|
|
deleteEvidence, err := l.inspectDeleteDomainEvidence(ctx, uuid, domain)
|
|
if err != nil {
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
evidence = append(evidence, deleteEvidence...)
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect delete domain activity", err)
|
|
}
|
|
if active {
|
|
return operationCleanupInspection{}, nil
|
|
}
|
|
evidence = append(evidence, "domain:"+uuid+":inactive")
|
|
steps = append(steps, CleanupStepUndefineDomain)
|
|
}
|
|
|
|
pool, err := l.conn.LookupStoragePoolByName(l.storagePoolName(StorageRoleManaged))
|
|
if err != nil {
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed storage pool", err)
|
|
}
|
|
inventory, err := pool.ListStorageVolumes()
|
|
if err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed volumes", err)
|
|
}
|
|
names, err := collectVMManagedVolumeNames(uuid, managed, inventory)
|
|
if err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
managedVolumesExist := false
|
|
for _, name := range names {
|
|
if err := ctx.Err(); err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
volume, lookupErr := pool.LookupStorageVolByName(name)
|
|
if isNoStorageVolume(lookupErr) {
|
|
continue
|
|
}
|
|
if lookupErr != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", lookupErr)
|
|
}
|
|
managedVolumesExist = true
|
|
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
|
|
volume.Free()
|
|
pool.Free()
|
|
return operationCleanupInspection{}, validateErr
|
|
}
|
|
volumeDigest, digestErr := storageVolumeInspectionDigest(volume)
|
|
if digestErr != nil {
|
|
volume.Free()
|
|
pool.Free()
|
|
return operationCleanupInspection{}, digestErr
|
|
}
|
|
path, pathErr := volume.GetPath()
|
|
volume.Free()
|
|
if pathErr != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed cleanup volume", pathErr)
|
|
}
|
|
var referenced bool
|
|
if domain != nil {
|
|
referenced, err = l.otherDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), name, uuid,
|
|
)
|
|
} else {
|
|
referenced, err = l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), name,
|
|
)
|
|
}
|
|
if err != nil {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
if referenced {
|
|
pool.Free()
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed cleanup volume is referenced by another domain", nil)
|
|
}
|
|
evidence = append(evidence, "managed-volume:"+name+":"+volumeDigest)
|
|
}
|
|
pool.Free()
|
|
if managedVolumesExist {
|
|
steps = append(steps, CleanupStepDeleteManagedVolumes)
|
|
}
|
|
|
|
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
|
|
if err != nil {
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
if seed != nil {
|
|
seedDigest, digestErr := storageVolumeInspectionDigest(seed)
|
|
if digestErr != nil {
|
|
seed.Free()
|
|
return operationCleanupInspection{}, digestErr
|
|
}
|
|
path, pathErr := seed.GetPath()
|
|
seed.Free()
|
|
if pathErr != nil {
|
|
return operationCleanupInspection{}, translateLibvirtError("inspect managed seed cleanup volume", pathErr)
|
|
}
|
|
referenced, referenceErr := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
|
|
)
|
|
if domain != nil {
|
|
referenced, referenceErr = l.otherDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), uuid,
|
|
)
|
|
}
|
|
if referenceErr != nil {
|
|
return operationCleanupInspection{}, referenceErr
|
|
}
|
|
if referenced {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_disk_shared", "managed seed cleanup volume is referenced by another domain", nil)
|
|
}
|
|
steps = append(steps, CleanupStepDeleteSeedVolume)
|
|
evidence = append(evidence, "seed-volume:"+seedVolumeName(uuid)+":"+seedDigest)
|
|
}
|
|
|
|
filter, err := l.lookupOptionalNetworkFilter(networkFilterName(uuid))
|
|
if err != nil {
|
|
return operationCleanupInspection{}, err
|
|
}
|
|
if filter != nil {
|
|
document, xmlErr := filter.GetXMLDesc(0)
|
|
filter.Free()
|
|
if xmlErr != nil || !managedFilterDocument(document, networkFilterName(uuid)) {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_invalid", "network filter cannot be confirmed as panel managed", xmlErr)
|
|
}
|
|
excludedUUID := ""
|
|
if domain != nil {
|
|
excludedUUID = uuid
|
|
}
|
|
referenced, referenceErr := l.otherDomainUsesFilter(networkFilterName(uuid), excludedUUID)
|
|
if referenceErr != nil {
|
|
return operationCleanupInspection{}, referenceErr
|
|
}
|
|
if referenced {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_filter_shared", "managed VM network filter is referenced by another domain", nil)
|
|
}
|
|
steps = append(steps, CleanupStepUndefineNetworkFilter)
|
|
digest := sha256.Sum256([]byte(document))
|
|
evidence = append(evidence, "network-filter:"+networkFilterName(uuid)+":"+hex.EncodeToString(digest[:]))
|
|
}
|
|
|
|
nvramExists := false
|
|
for _, name := range managedNVRAMArtifactNames(uuid) {
|
|
path := filepath.Join(l.local.NVRAMDirectory, name)
|
|
info, statErr := os.Lstat(path)
|
|
if errors.Is(statErr, os.ErrNotExist) {
|
|
continue
|
|
}
|
|
if statErr != nil {
|
|
return operationCleanupInspection{}, NewError(ErrUnavailable, "nvram_state_unavailable", "managed firmware state could not be inspected", statErr)
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_invalid", "managed firmware cleanup target is invalid", nil)
|
|
}
|
|
referenced, referenceErr := l.anyDomainUsesPath(path)
|
|
if domain != nil {
|
|
referenced, referenceErr = l.otherDomainUsesPath(path, uuid)
|
|
}
|
|
if referenceErr != nil {
|
|
return operationCleanupInspection{}, referenceErr
|
|
}
|
|
if referenced {
|
|
return operationCleanupInspection{}, NewError(ErrConflict, "managed_nvram_shared", "managed firmware state is referenced by another domain", nil)
|
|
}
|
|
nvramExists = true
|
|
evidence = append(evidence, "nvram:"+name+":"+strconv.FormatInt(info.Size(), 10)+":"+strconv.FormatInt(info.ModTime().UnixNano(), 10))
|
|
}
|
|
if nvramExists {
|
|
steps = append(steps, CleanupStepDeleteNVRAM)
|
|
}
|
|
trafficPresent := l.traffic.contains(uuid)
|
|
if trafficPresent || len(steps) != 0 {
|
|
// cleanupDeletedVMArtifacts durably rewrites traffic state even if an
|
|
// earlier failed fsync left only an on-disk entry. Make that mutation
|
|
// explicit in every delete cleanup plan.
|
|
steps = append(steps, CleanupStepRemoveTrafficState)
|
|
if trafficPresent {
|
|
evidence = append(evidence, "traffic-state:"+uuid+":present")
|
|
} else {
|
|
evidence = append(evidence, "traffic-state:"+uuid+":rewrite")
|
|
}
|
|
}
|
|
return newOperationCleanupInspection(steps, evidence...), nil
|
|
}
|
|
|
|
func managedFilterDocument(document, expectedName string) bool {
|
|
var filter libvirtxml.NWFilter
|
|
if filter.Unmarshal(document) != nil || filter.Name != expectedName || filter.Chain != "root" {
|
|
return false
|
|
}
|
|
return managedFilterBaselineValid(&filter)
|
|
}
|
|
|
|
func storageVolumeInspectionDigest(volume *libvirt.StorageVol) (string, error) {
|
|
document, err := volume.GetXMLDesc(0)
|
|
if err != nil {
|
|
return "", translateLibvirtError("read cleanup volume XML", err)
|
|
}
|
|
digest := sha256.Sum256([]byte(document))
|
|
return hex.EncodeToString(digest[:]), nil
|
|
}
|
|
|
|
func (l *Libvirt) inspectReinstallOperation(record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementName := reinstallVolumeName(record.ResourceID, record.OperationID)
|
|
replacement, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementExists := replacement != nil
|
|
if replacement != nil {
|
|
replacement.Free()
|
|
}
|
|
if diskTransitionCommitted(metadata, replacementName, "reinstall", record.OperationID) && replacementExists {
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "reinstall", record.OperationID) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
|
|
}
|
|
result, parseErr := metadataFromXML(document)
|
|
if parseErr != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
result.Status = mapDomainState(state)
|
|
result.ReinstallCommitted = true
|
|
result.CleanupPending = false
|
|
encoded, _ := json.Marshal(result)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
|
|
}
|
|
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "reinstall_not_committed", ErrorMessage: "host evidence confirms the reinstall did not switch the root volume"}, operationCleanupInspection{}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
|
|
func (l *Libvirt) inspectSnapshotOperation(ctx context.Context, record operation.Record,
|
|
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
key := parts[1]
|
|
document, metadata, err := requireManagedDomain(domain)
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
var stored *domainSnapshotMetadataXML
|
|
for index := range metadata.Snapshots {
|
|
if metadata.Snapshots[index].Key == key {
|
|
stored = &metadata.Snapshots[index]
|
|
break
|
|
}
|
|
}
|
|
volumeName := snapshotVolumeName(record.ResourceID, key)
|
|
var volume *libvirt.StorageVol
|
|
legacyBackupVolume := false
|
|
if parts[0] == "create-backup" || parts[0] == "cleanup-backup-workspace" {
|
|
volumeName, volume, legacyBackupVolume, err = l.backupWorkspaceResidue(
|
|
record.ResourceID, key,
|
|
)
|
|
} else {
|
|
volume, err = l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), volumeName,
|
|
)
|
|
}
|
|
if err != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
volumeExists := volume != nil
|
|
if volume != nil {
|
|
defer volume.Free()
|
|
}
|
|
|
|
switch parts[0] {
|
|
case "create-backup":
|
|
if stored != nil && volume != nil &&
|
|
volumeName != snapshotVolumeName(record.ResourceID, key) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if !backupResidueCleanupAllowed(record, stored) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
return operation.Resolution{}, l.backupResidueCleanupInspection(
|
|
domain, volume, volumeName, key, stored,
|
|
)
|
|
case "cleanup-backup-workspace":
|
|
if len(parts) != 2 {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && volume != nil &&
|
|
volumeName != snapshotVolumeName(record.ResourceID, key) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && !volumeExists {
|
|
result, _ := json.Marshal(CleanupBackupWorkspaceResult{
|
|
BackupKey: key, Cleaned: true,
|
|
WorkspaceAbsent: true, MetadataAbsent: true,
|
|
})
|
|
return operation.Resolution{
|
|
Resolved: true, Succeeded: true, Result: result,
|
|
}, operationCleanupInspection{}
|
|
}
|
|
if (stored == nil && volumeExists && !legacyBackupVolume) ||
|
|
(stored != nil && volumeExists && legacyBackupVolume) {
|
|
return operation.Resolution{
|
|
Resolved: true, Succeeded: false,
|
|
ErrorCode: "backup_workspace_cleanup_not_applied",
|
|
ErrorMessage: "host evidence confirms the backup workspace cleanup was not applied",
|
|
}, operationCleanupInspection{}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
case "create-snapshot":
|
|
if len(parts) != 3 || (parts[2] != "filesystem" && parts[2] != "crash") {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && volumeExists && snapshotConsistencyMatchesRequest(parts[2], stored.Consistency) {
|
|
info, validateErr := validateQCOW2Volume(volume, stored.SizeBytes)
|
|
if validateErr != nil || info.Capacity != stored.SizeBytes {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
result := Snapshot{ID: key, SnapshotKey: key, Name: stored.Name,
|
|
Description: stored.Description, CreatedAt: time.Unix(stored.CreatedAt, 0).UTC(),
|
|
State: "ready", Consistency: stored.Consistency, SizeBytes: stored.SizeBytes, Created: true}
|
|
encoded, _ := json.Marshal(result)
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: encoded}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && !volumeExists {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_created", ErrorMessage: "host evidence confirms the snapshot is absent"}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && volumeExists {
|
|
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
|
|
domain, volume, snapshotVolumeName(record.ResourceID, key),
|
|
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
|
|
)
|
|
}
|
|
if stored != nil && !volumeExists {
|
|
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
|
|
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
|
|
)
|
|
}
|
|
case "delete-snapshot":
|
|
if len(parts) != 2 {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if stored == nil && !volumeExists {
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Deleted: true})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && volumeExists {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_deleted", ErrorMessage: "host evidence confirms the snapshot remains present"}, operationCleanupInspection{}
|
|
}
|
|
if stored != nil && !volumeExists {
|
|
return operation.Resolution{}, cleanupInspectionWithIdleDomain(
|
|
domain, []string{CleanupStepRemoveSnapshotMetadata}, "snapshot-metadata:"+key,
|
|
)
|
|
}
|
|
if stored == nil && volumeExists {
|
|
return operation.Resolution{}, l.snapshotVolumeCleanupInspection(
|
|
domain, volume, snapshotVolumeName(record.ResourceID, key),
|
|
CleanupStepDeleteSnapshotVolume, "snapshot-volume:"+snapshotVolumeName(record.ResourceID, key),
|
|
)
|
|
}
|
|
case "restore-snapshot":
|
|
if len(parts) != 2 || stored == nil || !volumeExists {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID)
|
|
replacement, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
|
|
if lookupErr != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementExists := replacement != nil
|
|
if replacement != nil {
|
|
replacement.Free()
|
|
}
|
|
if diskTransitionCommitted(metadata, replacementName, "snapshot_restore", record.OperationID) && replacementExists {
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil || !diskTransitionPowerStateMatches(metadata, state, "snapshot_restore", record.OperationID) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
return operation.Resolution{}, l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
|
|
}
|
|
result, _ := json.Marshal(SnapshotMutationResult{SnapshotKey: key, Restored: true,
|
|
PowerState: mapDomainState(state), ActiveVolumeName: replacementName, CleanupPending: false})
|
|
return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{}
|
|
}
|
|
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "snapshot_not_restored", ErrorMessage: "host evidence confirms the root volume was not switched"}, operationCleanupInspection{}
|
|
}
|
|
case "recover-backup":
|
|
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
!sha256Pattern.MatchString(parts[2]) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
key, sourceSHA256 := parts[1], parts[2]
|
|
replacementName := snapshotRestoreVolumeName(record.ResourceID, record.OperationID)
|
|
replacement, lookupErr := l.lookupOptionalManagedVolume(
|
|
l.storagePoolName(StorageRoleManaged), replacementName)
|
|
if lookupErr != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
replacementExists := replacement != nil
|
|
if replacement != nil {
|
|
replacement.Free()
|
|
}
|
|
seedContents, seedErr := l.managedSeedContents(ctx, record.ResourceID)
|
|
if seedErr != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
seedOperation, seedErr := seedMetadataMarker(
|
|
seedContents, backupRecoveryOperationMetadata,
|
|
)
|
|
if seedErr != nil {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
seedCommitted := seedOperation == record.OperationID
|
|
diskCommitted := diskTransitionCommitted(metadata, replacementName,
|
|
"backup_recover", record.OperationID)
|
|
evidence := classifyBackupRecoveryEvidence(
|
|
replacementExists, diskCommitted,
|
|
metadata.DiskTransitionID == record.OperationID, seedCommitted,
|
|
)
|
|
if evidence == backupRecoveryEvidenceApplied {
|
|
state, _, stateErr := domain.GetState()
|
|
if stateErr != nil || !diskTransitionPowerStateMatches(
|
|
metadata, state, "backup_recover", record.OperationID) {
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
if metadata.PendingCleanupVolume != "" {
|
|
return operation.Resolution{},
|
|
l.inspectDiskTransitionCleanupSteps(domain, document, metadata)
|
|
}
|
|
result, _ := json.Marshal(RecoverBackupResult{
|
|
BackupKey: key, SHA256: sourceSHA256, Restored: true,
|
|
PowerState: mapDomainState(state), ActiveVolumeName: replacementName,
|
|
})
|
|
return operation.Resolution{Resolved: true, Succeeded: true,
|
|
Result: result}, operationCleanupInspection{}
|
|
}
|
|
if evidence == backupRecoveryEvidenceNotApplied {
|
|
return operation.Resolution{Resolved: true, Succeeded: false,
|
|
ErrorCode: "backup_not_recovered",
|
|
ErrorMessage: "host evidence confirms the root volume was not switched"},
|
|
operationCleanupInspection{}
|
|
}
|
|
}
|
|
return operation.Resolution{}, operationCleanupInspection{}
|
|
}
|
|
|
|
type backupRecoveryEvidence uint8
|
|
|
|
const (
|
|
backupRecoveryEvidenceUnknown backupRecoveryEvidence = iota
|
|
backupRecoveryEvidenceApplied
|
|
backupRecoveryEvidenceNotApplied
|
|
)
|
|
|
|
// classifyBackupRecoveryEvidence is deliberately fail-closed. Recovery is
|
|
// applied only when both the root-disk transition and the seed transition are
|
|
// visible. Any partial evidence remains unresolved and fenced.
|
|
func classifyBackupRecoveryEvidence(replacementExists, diskCommitted,
|
|
diskTransitionClaimed, seedCommitted bool) backupRecoveryEvidence {
|
|
if replacementExists && diskCommitted && seedCommitted {
|
|
return backupRecoveryEvidenceApplied
|
|
}
|
|
if !replacementExists && !diskTransitionClaimed && !seedCommitted {
|
|
return backupRecoveryEvidenceNotApplied
|
|
}
|
|
return backupRecoveryEvidenceUnknown
|
|
}
|
|
|
|
func backupResidueCleanupAllowed(record operation.Record, stored *domainSnapshotMetadataXML) bool {
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) != 3 || parts[0] != "create-backup" ||
|
|
!catalogIDPattern.MatchString(parts[1]) ||
|
|
(parts[2] != "filesystem" && parts[2] != "crash") {
|
|
return false
|
|
}
|
|
if record.State == operation.StateSucceeded {
|
|
if !successfulBackupCleanupPending(record) {
|
|
return false
|
|
}
|
|
} else if record.State != operation.StateFailed || !record.EffectMayHaveApplied || record.ExecutionFenced {
|
|
return false
|
|
}
|
|
return stored == nil || (stored.Key == parts[1] && stored.SizeBytes > 0 &&
|
|
snapshotConsistencyMatchesRequest(parts[2], stored.Consistency))
|
|
}
|
|
|
|
func successfulBackupCleanupPending(record operation.Record) bool {
|
|
if record.State != operation.StateSucceeded || !recordHasCleanupPending(record) ||
|
|
len(operationEffectProjection(record, record.Result)) == 0 {
|
|
return false
|
|
}
|
|
return strings.HasPrefix(record.Action, "create-backup:")
|
|
}
|
|
|
|
// backupResidueCleanupInspection exposes only an evidence-bound cleanup plan
|
|
// for a local immutable copy left by an already-terminal backup worker. It
|
|
// never asserts whether the remote multipart object committed: failed records
|
|
// therefore remain partial, while successful records retain their immutable
|
|
// applied object projection through appliedBackupInspectionWithCleanup.
|
|
func (l *Libvirt) backupResidueCleanupInspection(domain cleanupJobDomain,
|
|
volume *libvirt.StorageVol, volumeName, key string,
|
|
stored *domainSnapshotMetadataXML) operationCleanupInspection {
|
|
if volume == nil && stored == nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if volume != nil {
|
|
expectedCapacity := uint64(0)
|
|
if stored != nil {
|
|
expectedCapacity = stored.SizeBytes
|
|
}
|
|
if _, err := validateStandaloneQCOW2Volume(volume, expectedCapacity); err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
plan := l.snapshotVolumeCleanupInspection(
|
|
domain, volume, volumeName, CleanupStepDeleteSnapshotVolume,
|
|
"backup-volume:"+volumeName,
|
|
)
|
|
if len(plan.steps) == 0 {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if stored != nil {
|
|
plan.steps = append(plan.steps, CleanupStepRemoveSnapshotMetadata)
|
|
plan.evidence = append(plan.evidence, "backup-metadata:"+key+":"+
|
|
stored.Consistency+":"+strconv.FormatUint(stored.SizeBytes, 10))
|
|
sort.Strings(plan.evidence)
|
|
}
|
|
return plan
|
|
}
|
|
return cleanupInspectionWithIdleDomain(
|
|
domain, []string{CleanupStepRemoveSnapshotMetadata},
|
|
"backup-metadata:"+key+":"+stored.Consistency+":"+
|
|
strconv.FormatUint(stored.SizeBytes, 10),
|
|
)
|
|
}
|
|
|
|
func (l *Libvirt) absentDomainBackupWorkspaceCleanupInspection(
|
|
volume *libvirt.StorageVol, volumeName string,
|
|
) operationCleanupInspection {
|
|
if volume == nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if _, err := validateStandaloneQCOW2Volume(volume, 0); err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
digest, err := storageVolumeInspectionDigest(volume)
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
path, err := volume.GetPath()
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
referenced, err := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), volumeName,
|
|
)
|
|
if err != nil || referenced {
|
|
return operationCleanupInspection{}
|
|
}
|
|
return newOperationCleanupInspection(
|
|
[]string{CleanupStepDeleteSnapshotVolume},
|
|
"domain-state:absent", "backup-volume:"+volumeName+":"+digest,
|
|
)
|
|
}
|
|
|
|
func diskTransitionPowerStateMatches(metadata *domainMetadataXML, state libvirt.DomainState,
|
|
kind, operationID string) bool {
|
|
if !diskTransitionMarkerMatches(metadata, kind, operationID) {
|
|
return true
|
|
}
|
|
if metadata.DiskTransitionRunning {
|
|
return state == libvirt.DOMAIN_RUNNING || state == libvirt.DOMAIN_BLOCKED
|
|
}
|
|
return state == libvirt.DOMAIN_SHUTOFF || state == libvirt.DOMAIN_SHUTDOWN
|
|
}
|
|
|
|
func (l *Libvirt) inspectDiskTransitionCleanupSteps(domain *libvirt.Domain, document string,
|
|
metadata *domainMetadataXML) operationCleanupInspection {
|
|
pending := metadata.PendingCleanupVolume
|
|
if pending == "" || !isVMOwnedRootVolumeName(metadataUUID(document), pending) ||
|
|
pending == metadata.RootVolumeName {
|
|
return operationCleanupInspection{}
|
|
}
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}
|
|
}
|
|
steps := make([]string, 0, 2)
|
|
evidence := append([]string(nil), jobEvidence...)
|
|
evidence = append(evidence, "pending-root:"+pending, "active-root:"+metadata.RootVolumeName)
|
|
old, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), pending)
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if old != nil {
|
|
if _, validateErr := validateQCOW2Volume(old, 0); validateErr != nil {
|
|
old.Free()
|
|
return operationCleanupInspection{}
|
|
}
|
|
oldDigest, digestErr := storageVolumeInspectionDigest(old)
|
|
if digestErr != nil {
|
|
old.Free()
|
|
return operationCleanupInspection{}
|
|
}
|
|
path, pathErr := old.GetPath()
|
|
old.Free()
|
|
if pathErr != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
used, referenceErr := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), pending,
|
|
)
|
|
if referenceErr != nil || used {
|
|
return operationCleanupInspection{}
|
|
}
|
|
steps = append(steps, CleanupStepDeletePreviousRoot)
|
|
evidence = append(evidence, "pending-root-state:present:"+oldDigest)
|
|
} else {
|
|
evidence = append(evidence, "pending-root-state:absent")
|
|
}
|
|
steps = append(steps, CleanupStepPersistCleanupMetadata)
|
|
return newOperationCleanupInspection(steps, evidence...)
|
|
}
|
|
|
|
type cleanupJobDomain interface {
|
|
IsActive() (bool, error)
|
|
GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error)
|
|
GetXMLDesc(libvirt.DomainXMLFlags) (string, error)
|
|
GetBlockJobInfo(string, libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error)
|
|
}
|
|
|
|
func inspectCleanupJobEvidence(domain cleanupJobDomain) ([]string, bool) {
|
|
active, err := domain.IsActive()
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
flags := libvirt.DOMAIN_XML_INACTIVE
|
|
evidence := []string{"domain-runtime:inactive"}
|
|
if active {
|
|
flags = 0
|
|
job, jobErr := domain.GetJobStats(0)
|
|
if jobErr != nil || job == nil || job.Type != libvirt.DOMAIN_JOB_NONE {
|
|
return nil, false
|
|
}
|
|
evidence[0] = "domain-runtime:active:job-none"
|
|
}
|
|
document, err := domain.GetXMLDesc(flags)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
documentDigest := sha256.Sum256([]byte(document))
|
|
evidence = append(evidence, "domain-xml:"+hex.EncodeToString(documentDigest[:]))
|
|
if active {
|
|
persistentDocument, persistentErr := domain.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
|
|
if persistentErr != nil {
|
|
return nil, false
|
|
}
|
|
persistentDigest := sha256.Sum256([]byte(persistentDocument))
|
|
evidence = append(evidence, "persistent-domain-xml:"+hex.EncodeToString(persistentDigest[:]))
|
|
}
|
|
targets, err := cleanupDiskTargets(document)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
for _, target := range targets {
|
|
evidence = append(evidence, "disk-target:"+target)
|
|
if !active {
|
|
continue
|
|
}
|
|
info, infoErr := domain.GetBlockJobInfo(target, 0)
|
|
if infoErr != nil || !blockJobInfoAbsent(info) {
|
|
return nil, false
|
|
}
|
|
}
|
|
return evidence, true
|
|
}
|
|
|
|
func cleanupDiskTargets(document string) ([]string, error) {
|
|
var definition libvirtxml.Domain
|
|
if err := definition.Unmarshal(document); err != nil || definition.Devices == nil {
|
|
return nil, errors.New("domain disk evidence is invalid")
|
|
}
|
|
seen := make(map[string]struct{}, len(definition.Devices.Disks))
|
|
for _, disk := range definition.Devices.Disks {
|
|
if disk.Target == nil || disk.Target.Dev == "" {
|
|
return nil, errors.New("domain disk target evidence is incomplete")
|
|
}
|
|
seen[disk.Target.Dev] = struct{}{}
|
|
}
|
|
targets := make([]string, 0, len(seen))
|
|
for target := range seen {
|
|
targets = append(targets, target)
|
|
}
|
|
sort.Strings(targets)
|
|
return targets, nil
|
|
}
|
|
|
|
func blockJobInfoAbsent(info *libvirt.DomainBlockJobInfo) bool {
|
|
// The Go binding does not expose virDomainGetBlockJobInfo's 0/1 return
|
|
// value. A 0 (no job) is represented by the zero-valued info struct.
|
|
return info != nil && info.Type == libvirt.DOMAIN_BLOCK_JOB_TYPE_UNKNOWN &&
|
|
info.Bandwidth == 0 && info.Cur == 0 && info.End == 0
|
|
}
|
|
|
|
func cleanupInspectionWithIdleDomain(domain cleanupJobDomain, steps []string,
|
|
evidence ...string) operationCleanupInspection {
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}
|
|
}
|
|
evidence = append(evidence, jobEvidence...)
|
|
return newOperationCleanupInspection(steps, evidence...)
|
|
}
|
|
|
|
func ensureNoCleanupBlockingJobs(domain cleanupJobDomain) error {
|
|
if _, safe := inspectCleanupJobEvidence(domain); !safe {
|
|
return NewError(ErrConflict, "cleanup_job_not_idle", "domain cleanup is blocked by active or unreadable job state", nil)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *Libvirt) snapshotVolumeCleanupInspection(domain cleanupJobDomain, volume *libvirt.StorageVol,
|
|
volumeName, step, targetEvidence string) operationCleanupInspection {
|
|
if volume == nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
jobEvidence, safe := inspectCleanupJobEvidence(domain)
|
|
if !safe {
|
|
return operationCleanupInspection{}
|
|
}
|
|
if _, err := validateQCOW2Volume(volume, 0); err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
volumeDigest, err := storageVolumeInspectionDigest(volume)
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
path, err := volume.GetPath()
|
|
if err != nil {
|
|
return operationCleanupInspection{}
|
|
}
|
|
referenced, err := l.anyDomainUsesVolume(
|
|
path, l.storagePoolName(StorageRoleManaged), volumeName,
|
|
)
|
|
if err != nil || referenced {
|
|
return operationCleanupInspection{}
|
|
}
|
|
evidence := append([]string{targetEvidence + ":" + volumeDigest}, jobEvidence...)
|
|
return newOperationCleanupInspection([]string{step}, evidence...)
|
|
}
|
|
|
|
func (l *Libvirt) inspectMatchingBackupJob(record operation.Record, domain *libvirt.Domain) OperationAbortInspection {
|
|
if record.State != operation.StateDispatching &&
|
|
(record.State != operation.StateFailed || !record.EffectMayHaveApplied) {
|
|
return OperationAbortInspection{}
|
|
}
|
|
parts := strings.Split(record.Action, ":")
|
|
if len(parts) != 3 || (parts[0] != "create-snapshot" && parts[0] != "create-backup") ||
|
|
!catalogIDPattern.MatchString(parts[1]) || (parts[2] != "filesystem" && parts[2] != "crash") {
|
|
return OperationAbortInspection{}
|
|
}
|
|
job, err := domain.GetJobStats(0)
|
|
if err != nil || job == nil ||
|
|
(job.Type != libvirt.DOMAIN_JOB_BOUNDED && job.Type != libvirt.DOMAIN_JOB_UNBOUNDED) ||
|
|
!job.OperationSet || job.Operation != libvirt.DOMAIN_JOB_OPERATION_BACKUP {
|
|
return OperationAbortInspection{}
|
|
}
|
|
backupXML, err := domain.BackupGetXMLDesc(0)
|
|
if err != nil {
|
|
return OperationAbortInspection{}
|
|
}
|
|
_, volume, _, err := l.backupWorkspaceResidue(record.ResourceID, parts[1])
|
|
if err != nil || volume == nil {
|
|
return OperationAbortInspection{}
|
|
}
|
|
if _, validateErr := validateQCOW2Volume(volume, 0); validateErr != nil {
|
|
volume.Free()
|
|
return OperationAbortInspection{}
|
|
}
|
|
targetPath, pathErr := volume.GetPath()
|
|
volume.Free()
|
|
if pathErr != nil || !backupPushMatchesTarget(backupXML, targetPath) {
|
|
return OperationAbortInspection{}
|
|
}
|
|
return OperationAbortInspection{
|
|
Available: true, Kind: "backup",
|
|
InspectionToken: operationInspectionToken(record, "backup", backupXML),
|
|
}
|
|
}
|
|
|
|
func backupPushMatchesTarget(backupXML, targetPath string) bool {
|
|
var backup libvirtxml.DomainBackup
|
|
if backup.Unmarshal(backupXML) != nil || backup.Pull != nil || backup.Push == nil ||
|
|
backup.Push.Disks == nil || len(backup.Push.Disks.Disks) != 1 {
|
|
return false
|
|
}
|
|
disk := backup.Push.Disks.Disks[0]
|
|
return disk.Name == "vda" && disk.Backup == "yes" && disk.Target != nil &&
|
|
disk.Target.File != nil && filepath.Clean(disk.Target.File.File) == filepath.Clean(targetPath)
|
|
}
|
|
|
|
func buildOperationInspection(record operation.Record, resolution operation.Resolution,
|
|
abort OperationAbortInspection, cleanup operationCleanupInspection) OperationInspection {
|
|
result := OperationInspection{
|
|
OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID,
|
|
RequestFingerprint: record.Fingerprint, OperationStatus: record.State,
|
|
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
|
|
SummaryCode: InspectionSummaryIncomplete,
|
|
EffectResolution: InspectionEffectInconclusive,
|
|
AbortJob: abort, Cleanup: OperationCleanupPlan{},
|
|
}
|
|
if abort.Available {
|
|
result.SummaryCode = InspectionSummaryAbortableJob
|
|
return result
|
|
}
|
|
if len(cleanup.steps) != 0 && len(cleanup.evidence) != 0 {
|
|
result.SummaryCode = InspectionSummaryCleanupRequired
|
|
result.EffectResolution = InspectionEffectPartial
|
|
result.Cleanup = OperationCleanupPlan{
|
|
Available: true,
|
|
PlanID: operationInspectionToken(record, "cleanup", cleanup.tokenEvidence()),
|
|
Steps: append([]string(nil), cleanup.steps...),
|
|
}
|
|
return result
|
|
}
|
|
if !resolution.Resolved {
|
|
return result
|
|
}
|
|
if record.Action == "resize-vm" && !resolution.Succeeded && resolution.ErrorCode == "resources_partially_resized" {
|
|
result.ErrorCode = resolution.ErrorCode
|
|
result.EffectResolution = InspectionEffectPartial
|
|
return result
|
|
}
|
|
if resolution.Succeeded {
|
|
result.Outcome = InspectionOutcomeSucceeded
|
|
result.SummaryCode = InspectionSummarySucceeded
|
|
projection := operationEffectProjection(record, resolution.Result)
|
|
result.EffectResolution = inspectedEffectResolution(record, resolution, projection)
|
|
if result.EffectResolution == InspectionEffectApplied {
|
|
result.EffectProjection = projection
|
|
} else {
|
|
result.Outcome = InspectionOutcomeUncertain
|
|
result.SummaryCode = InspectionSummaryIncomplete
|
|
}
|
|
} else {
|
|
result.Outcome = InspectionOutcomeFailed
|
|
result.SummaryCode = InspectionSummaryFailed
|
|
projection := failedOperationEffectProjection(record, resolution)
|
|
result.EffectResolution = inspectedEffectResolution(record, resolution, projection)
|
|
if result.EffectResolution == InspectionEffectNotApplied {
|
|
result.EffectProjection = projection
|
|
} else {
|
|
result.Outcome = InspectionOutcomeUncertain
|
|
result.SummaryCode = InspectionSummaryIncomplete
|
|
}
|
|
if inspectionErrorCodePattern.MatchString(resolution.ErrorCode) {
|
|
result.ErrorCode = resolution.ErrorCode
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// inspectedEffectResolution separates immutable operation evidence from a
|
|
// read-only observation that merely happens to match the requested target.
|
|
// Power, service, network-link, catalog and storage state may be changed out
|
|
// of band, so those observations remain inconclusive even when the target is
|
|
// currently reached. Only a Journal terminal record or a mutation-specific
|
|
// durable marker may authorize automated ledger convergence.
|
|
func inspectedEffectResolution(record operation.Record, resolution operation.Resolution,
|
|
projection json.RawMessage) string {
|
|
if record.State == operation.StateSucceeded {
|
|
return InspectionEffectApplied
|
|
}
|
|
if record.State == operation.StateFailed && !record.EffectMayHaveApplied {
|
|
return InspectionEffectNotApplied
|
|
}
|
|
if record.State != operation.StateFailed || !record.EffectMayHaveApplied || !resolution.Resolved {
|
|
return InspectionEffectInconclusive
|
|
}
|
|
if resolution.Succeeded {
|
|
if len(projection) == 0 {
|
|
return InspectionEffectInconclusive
|
|
}
|
|
switch {
|
|
case record.Action == "update-settings",
|
|
record.Action == "resize-vm",
|
|
record.Action == "change-network",
|
|
record.Action == "reinstall-vm",
|
|
record.Action == "apply-firewall",
|
|
strings.HasPrefix(record.Action, "apply-vpc-network:"),
|
|
strings.HasPrefix(record.Action, "remove-vpc-network:"),
|
|
record.Action == "apply-vpc-port",
|
|
record.Action == "remove-vpc-port",
|
|
strings.HasPrefix(record.Action, "restore-snapshot:"),
|
|
strings.HasPrefix(record.Action, "recover-backup:"):
|
|
return InspectionEffectApplied
|
|
default:
|
|
return InspectionEffectInconclusive
|
|
}
|
|
}
|
|
switch {
|
|
case operation.IsCreateLikeAction(record.Action) && resolution.ErrorCode == "creation_absent" && len(projection) != 0,
|
|
record.Action == "update-settings" && resolution.ErrorCode == "settings_not_committed",
|
|
record.Action == "change-network" && resolution.ErrorCode == "network_not_committed",
|
|
record.Action == "reinstall-vm" && resolution.ErrorCode == "reinstall_not_committed",
|
|
record.Action == "apply-firewall" && resolution.ErrorCode == "firewall_not_committed",
|
|
strings.HasPrefix(record.Action, "restore-snapshot:") && resolution.ErrorCode == "snapshot_not_restored",
|
|
strings.HasPrefix(record.Action, "recover-backup:") && resolution.ErrorCode == "backup_not_recovered":
|
|
return InspectionEffectNotApplied
|
|
default:
|
|
return InspectionEffectInconclusive
|
|
}
|
|
}
|
|
|
|
func inspectionFromRecord(record operation.Record) OperationInspection {
|
|
switch record.State {
|
|
case operation.StateSucceeded:
|
|
return buildOperationInspection(record, operation.Resolution{
|
|
Resolved: true, Succeeded: true, Result: append([]byte(nil), record.Result...),
|
|
}, OperationAbortInspection{}, operationCleanupInspection{})
|
|
case operation.StateFailed:
|
|
resolution := operation.Resolution{Resolved: true, Succeeded: false, Result: append([]byte(nil), record.Result...)}
|
|
if record.Error != nil {
|
|
resolution.ErrorCode = record.Error.Code
|
|
resolution.ErrorMessage = record.Error.Message
|
|
}
|
|
return buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{})
|
|
}
|
|
return inconclusiveInspectionFromRecord(record)
|
|
}
|
|
|
|
func inconclusiveInspectionFromRecord(record operation.Record) OperationInspection {
|
|
return OperationInspection{
|
|
OperationID: record.OperationID, Action: record.Action, ResourceID: record.ResourceID,
|
|
RequestFingerprint: record.Fingerprint, OperationStatus: record.State,
|
|
CheckedAt: time.Now().UTC().Unix(), Outcome: InspectionOutcomeUncertain,
|
|
SummaryCode: InspectionSummaryIncomplete, EffectResolution: InspectionEffectInconclusive,
|
|
}
|
|
}
|
|
|
|
func failedOperationEffectProjection(record operation.Record, resolution operation.Resolution) json.RawMessage {
|
|
if !operation.IsCreateLikeAction(record.Action) || resolution.ErrorCode != "creation_absent" {
|
|
return nil
|
|
}
|
|
var evidence struct {
|
|
CreationAbsent bool `json:"creation_absent"`
|
|
ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"`
|
|
}
|
|
if !decodeOperationEffectResult(resolution.Result, &evidence) ||
|
|
!evidence.CreationAbsent || !evidence.ManagedArtifactsAbsent {
|
|
return nil
|
|
}
|
|
encoded, err := json.Marshal(evidence)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return encoded
|
|
}
|
|
|
|
const maxOperationEffectProjectionBytes = 64 << 10
|
|
|
|
// operationEffectProjection never forwards Journal bytes. It first decodes a
|
|
// result into the exact response type allowed for the recorded action and then
|
|
// rebuilds a deliberately smaller projection. Unknown actions, malformed or
|
|
// oversized results, and identity mismatches simply omit the optional
|
|
// projection while keeping the independently derived effect resolution.
|
|
func operationEffectProjection(record operation.Record, raw []byte) json.RawMessage {
|
|
if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes || !json.Valid(raw) {
|
|
return nil
|
|
}
|
|
var projection any
|
|
switch {
|
|
case record.Action == "configure-storage":
|
|
var value StorageConfigurationResult
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
Configured bool `json:"configured"`
|
|
}{Configured: value.Configured}
|
|
case strings.HasPrefix(record.Action, "apply-vpc-network:") ||
|
|
strings.HasPrefix(record.Action, "remove-vpc-network:"):
|
|
var value vpcrouting.Result
|
|
expected := record.ExpectedState
|
|
bridge := ""
|
|
if expected != nil {
|
|
bridge, _ = vpcrouting.BridgeName(expected.VPCKey)
|
|
}
|
|
if !decodeOperationEffectResult(raw, &value) || expected == nil ||
|
|
expected.Kind != operation.ExpectedVPCNetwork || value.VPCKey != expected.VPCKey ||
|
|
value.DesiredRevision != expected.VPCRevision || value.DesiredDigest != expected.VPCDigest ||
|
|
(expected.VPCPresent && (!value.Applied || value.Removed || value.Bridge != bridge)) ||
|
|
(!expected.VPCPresent && (!value.Removed || value.Applied || value.Bridge != "")) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case record.Action == "apply-vpc-port" || record.Action == "remove-vpc-port":
|
|
var value VPCPortResult
|
|
expected := record.ExpectedState
|
|
if !decodeOperationEffectResult(raw, &value) || expected == nil ||
|
|
expected.Kind != operation.ExpectedVPCPort || value.UUID != record.ResourceID ||
|
|
value.PortKey != expected.VPCPortKey || value.VPCKey != expected.VPCKey ||
|
|
(expected.VPCPresent && (!value.Attached || value.Removed || value.VPCPort == nil ||
|
|
*value.VPCPort != expectedVPCPortMetadata(*expected))) ||
|
|
(!expected.VPCPresent && (!value.Removed || value.Attached || value.VPCPort != nil)) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case strings.HasPrefix(record.Action, "create-backup:"):
|
|
parts := strings.Split(record.Action, ":")
|
|
var value BackupResult
|
|
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
(parts[2] != "filesystem" && parts[2] != "crash") ||
|
|
!decodeOperationEffectResult(raw, &value) || value.BackupKey != parts[1] ||
|
|
value.SizeBytes <= 0 || value.VirtualSizeBytes == 0 ||
|
|
!lowerHexDigest(value.SHA256) ||
|
|
!snapshotConsistencyMatchesRequest(parts[2], value.Consistency) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case strings.HasPrefix(record.Action, "prepare-peer-migration:"):
|
|
parts := strings.Split(record.Action, ":")
|
|
var value PreparePeerMigrationResult
|
|
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
!decodeOperationEffectResult(raw, &value) || value.MigrationID != parts[1] ||
|
|
!lowerHexDigest(value.ManifestDigest) || !value.SourceStopped ||
|
|
!value.SourceAutostartDisabled || len(value.Artifacts) < 1 ||
|
|
len(value.Artifacts) > peermigration.MaxArtifacts ||
|
|
!validPeerMigrationProjectionArtifacts(parts[1], record.ResourceID, value.Artifacts) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case strings.HasPrefix(record.Action, "receive-peer-migration:"):
|
|
parts := strings.Split(record.Action, ":")
|
|
var value ReceivePeerMigrationResult
|
|
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
!decodeOperationEffectResult(raw, &value) || value.MigrationID != parts[1] ||
|
|
value.UUID != record.ResourceID || !lowerHexDigest(value.ManifestDigest) ||
|
|
!value.StagedPoweredOff || len(value.Artifacts) < 1 ||
|
|
len(value.Artifacts) > peermigration.MaxArtifacts ||
|
|
!validPeerMigrationProjectionArtifacts(parts[1], record.ResourceID, value.Artifacts) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
operationVMEffectProjection
|
|
MigrationID string `json:"migration_id"`
|
|
ManifestDigest string `json:"manifest_digest"`
|
|
Artifacts []peermigration.Artifact `json:"artifacts"`
|
|
StagedPoweredOff bool `json:"staged_powered_off"`
|
|
}{
|
|
operationVMEffectProjection: vmEffectProjection(value.VMMetadata),
|
|
MigrationID: value.MigrationID, ManifestDigest: value.ManifestDigest,
|
|
Artifacts: value.Artifacts, StagedPoweredOff: value.StagedPoweredOff,
|
|
}
|
|
case strings.HasPrefix(record.Action, "restore-backup:"):
|
|
parts := strings.Split(record.Action, ":")
|
|
var value RestoreBackupResult
|
|
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
!decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID ||
|
|
value.BackupKey != parts[1] || !lowerHexDigest(value.SHA256) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
operationVMEffectProjection
|
|
BackupKey string `json:"backup_key"`
|
|
SHA256 string `json:"sha256"`
|
|
}{
|
|
operationVMEffectProjection: vmEffectProjection(value.VMMetadata),
|
|
BackupKey: value.BackupKey, SHA256: value.SHA256,
|
|
}
|
|
case strings.HasPrefix(record.Action, "recover-backup:"):
|
|
parts := strings.Split(record.Action, ":")
|
|
var value RecoverBackupResult
|
|
if len(parts) != 3 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
!sha256Pattern.MatchString(parts[2]) ||
|
|
!decodeOperationEffectResult(raw, &value) || !value.Restored ||
|
|
value.BackupKey != parts[1] || value.SHA256 != parts[2] {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
BackupKey string `json:"backup_key"`
|
|
SHA256 string `json:"sha256,omitempty"`
|
|
Restored bool `json:"restored"`
|
|
PowerState VMStateName `json:"power_state"`
|
|
}{value.BackupKey, value.SHA256, value.Restored, value.PowerState}
|
|
case operation.IsCreateLikeAction(record.Action) || record.Action == "update-settings" ||
|
|
record.Action == "change-network" || record.Action == "reinstall-vm":
|
|
var value VMMetadata
|
|
if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID {
|
|
return nil
|
|
}
|
|
projection = vmEffectProjection(value)
|
|
case record.Action == "resize-vm":
|
|
var value ResizeResourcesResult
|
|
expected := record.ExpectedState
|
|
if !decodeOperationEffectResult(raw, &value) || expected == nil || expected.Kind != operation.ExpectedResizeResources ||
|
|
expected.TransitionID != record.OperationID || value.UUID != record.ResourceID || !value.ResourcesResized ||
|
|
value.FromVCPUs != expected.FromVCPUs || value.TargetVCPUs != expected.TargetVCPUs ||
|
|
value.FromMemoryMiB != expected.FromMemoryMiB || value.TargetMemoryMiB != expected.TargetMemoryMiB ||
|
|
value.FromDiskGiB != expected.FromDiskGiB || value.TargetDiskGiB != expected.TargetDiskGiB ||
|
|
value.FromBandwidthMbps != expected.FromBandwidthMbps || value.TargetBandwidthMbps != expected.TargetBandwidthMbps ||
|
|
value.AppliedVCPUs != value.TargetVCPUs || value.AppliedMemoryMiB != value.TargetMemoryMiB ||
|
|
value.AppliedDiskGiB != value.TargetDiskGiB || value.AppliedBandwidthMbps != value.TargetBandwidthMbps ||
|
|
!value.PowerStateRestored ||
|
|
(value.OriginallyRunning && value.Status != VMStateRunning && value.Status != VMStateBlocked) ||
|
|
(!value.OriginallyRunning && value.Status != VMStateShutoff && value.Status != VMStateCrashed) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case record.Action == "delete-vm" || record.Action == "force-delete-vm":
|
|
var value DeleteVMResult
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case strings.HasPrefix(record.Action, "cleanup-peer-migration-target:"):
|
|
parts := strings.Split(record.Action, ":")
|
|
var value DeleteVMResult
|
|
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
!decodeOperationEffectResult(raw, &value) || !value.Deleted ||
|
|
!value.ManagedArtifactsAbsent || value.Forced {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case strings.HasPrefix(record.Action, "activate-peer-migration:") ||
|
|
strings.HasPrefix(record.Action, "rollback-peer-migration:"):
|
|
parts := strings.Split(record.Action, ":")
|
|
var value PeerMigrationStateResult
|
|
expected := record.ExpectedState
|
|
if len(parts) != 2 || !catalogIDPattern.MatchString(parts[1]) ||
|
|
!decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID ||
|
|
expected == nil || expected.Kind != operation.ExpectedPeerMigrationState ||
|
|
value.Running != expected.StartIfSuspended ||
|
|
value.AutostartEnabled != expected.Enabled {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
UUID string `json:"uuid"`
|
|
State VMStateName `json:"state"`
|
|
Running bool `json:"running"`
|
|
AutostartEnabled bool `json:"autostart_enabled"`
|
|
}{value.UUID, value.State, value.Running, value.AutostartEnabled}
|
|
case strings.HasPrefix(record.Action, "power-") || strings.HasPrefix(record.Action, "service-"):
|
|
var value VMState
|
|
if !decodeOperationEffectResult(raw, &value) || value.UUID != record.ResourceID {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
UUID string `json:"uuid"`
|
|
State VMStateName `json:"state"`
|
|
}{UUID: value.UUID, State: value.State}
|
|
case record.Action == "network-suspend" || record.Action == "network-resume":
|
|
var value struct {
|
|
NetworkEnabled bool `json:"network_enabled"`
|
|
}
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
case strings.HasPrefix(record.Action, "create-snapshot:"):
|
|
var value Snapshot
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = operationSnapshotEffectProjection{
|
|
ID: value.ID, SnapshotKey: value.SnapshotKey, Name: value.Name,
|
|
CreatedAt: value.CreatedAt, State: value.State, Consistency: value.Consistency,
|
|
SizeBytes: value.SizeBytes, Created: value.Created,
|
|
}
|
|
case strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:"):
|
|
var value SnapshotMutationResult
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
SnapshotKey string `json:"snapshot_key"`
|
|
Restored bool `json:"restored,omitempty"`
|
|
Deleted bool `json:"deleted,omitempty"`
|
|
PowerState VMStateName `json:"power_state,omitempty"`
|
|
CleanupPending bool `json:"cleanup_pending,omitempty"`
|
|
}{value.SnapshotKey, value.Restored, value.Deleted, value.PowerState, value.CleanupPending}
|
|
case record.Action == "apply-firewall":
|
|
var value FirewallPolicy
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
Revision uint64 `json:"revision"`
|
|
DefaultIn string `json:"default_in"`
|
|
DefaultOut string `json:"default_out"`
|
|
PolicyDigest string `json:"policy_digest,omitempty"`
|
|
Rules []FirewallRule `json:"rules"`
|
|
}{value.Revision, value.DefaultIn, value.DefaultOut, value.PolicyDigest, append([]FirewallRule(nil), value.Rules...)}
|
|
case record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
|
|
strings.HasPrefix(record.Action, "exit-rescue:"):
|
|
var value RescueState
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = struct {
|
|
Active bool `json:"active"`
|
|
Staged bool `json:"staged"`
|
|
PendingManualRestart bool `json:"pending_manual_restart,omitempty"`
|
|
TemplateID string `json:"template_id,omitempty"`
|
|
Revision uint64 `json:"revision,omitempty"`
|
|
GenerationToken string `json:"generation_token,omitempty"`
|
|
NetworkSuspended bool `json:"network_suspended"`
|
|
}{
|
|
value.Active,
|
|
value.Staged,
|
|
value.PendingManualRestart,
|
|
value.TemplateID,
|
|
value.Revision,
|
|
value.GenerationToken,
|
|
value.NetworkSuspended,
|
|
}
|
|
case record.Action == "import-template" || record.Action == "update-template":
|
|
var value ImageTemplate
|
|
if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "template:"+value.ID {
|
|
return nil
|
|
}
|
|
projection = operationCatalogImportProjection{
|
|
ID: value.ID, Name: value.Name, OSFamily: value.OSFamily, Version: value.Version,
|
|
Architecture: value.Architecture, SizeBytes: value.SizeBytes,
|
|
VirtualSizeBytes: value.VirtualSizeBytes, SHA256: value.SHA256,
|
|
Revision: value.Revision, GenerationToken: value.GenerationToken, CreatedAt: value.CreatedAt,
|
|
}
|
|
case record.Action == "import-rescue-template":
|
|
var value RescueTemplate
|
|
if !decodeOperationEffectResult(raw, &value) || record.ResourceID != "rescue-template:"+value.ID {
|
|
return nil
|
|
}
|
|
projection = operationCatalogImportProjection{
|
|
ID: value.ID, Name: value.Name, SizeBytes: value.SizeBytes,
|
|
SHA256: value.SHA256, CreatedAt: value.CreatedAt,
|
|
}
|
|
case record.Action == "import-user-iso":
|
|
var value UserISO
|
|
if !decodeOperationEffectResult(raw, &value) ||
|
|
record.ResourceID != "user-iso:"+strconv.FormatInt(value.OwnerID, 10)+":"+value.ID {
|
|
return nil
|
|
}
|
|
projection = operationCatalogImportProjection{
|
|
ID: value.ID, OwnerID: value.OwnerID, Name: value.Name,
|
|
SizeBytes: value.SizeBytes, SHA256: value.SHA256, CreatedAt: value.CreatedAt,
|
|
}
|
|
case record.Action == "delete-template" || record.Action == "delete-rescue-template" ||
|
|
record.Action == "delete-user-iso":
|
|
var value CatalogDeleteResult
|
|
if !decodeOperationEffectResult(raw, &value) {
|
|
return nil
|
|
}
|
|
projection = value
|
|
default:
|
|
return nil
|
|
}
|
|
encoded, err := json.Marshal(projection)
|
|
if err != nil || len(encoded) == 0 || len(encoded) > maxOperationEffectProjectionBytes {
|
|
return nil
|
|
}
|
|
return encoded
|
|
}
|
|
|
|
func validPeerMigrationProjectionArtifacts(migrationID, uuid string,
|
|
artifacts []peermigration.Artifact) bool {
|
|
for _, artifact := range artifacts {
|
|
if artifact.Path != "" {
|
|
return false
|
|
}
|
|
}
|
|
return peermigration.ValidateManifest(peermigration.Manifest{
|
|
Version: peermigration.ProtocolVersion, MigrationID: migrationID,
|
|
SourceUUID: strings.ToLower(uuid), ExpiresAt: time.Now().Add(time.Hour).Unix(),
|
|
Artifacts: artifacts,
|
|
}, time.Now()) == nil
|
|
}
|
|
|
|
type operationVMEffectProjection struct {
|
|
UUID string `json:"uuid"`
|
|
Name string `json:"name,omitempty"`
|
|
Status VMStateName `json:"status,omitempty"`
|
|
VCPUs uint `json:"vcpu,omitempty"`
|
|
MemoryMiB uint64 `json:"memory_mib,omitempty"`
|
|
DiskGiB uint64 `json:"disk_gib,omitempty"`
|
|
BandwidthMbps uint64 `json:"bandwidth_mbps,omitempty"`
|
|
Hostname string `json:"hostname,omitempty"`
|
|
PublicNetwork publicnetwork.Intent `json:"public_network"`
|
|
TemplateID string `json:"template_id,omitempty"`
|
|
Machine string `json:"machine,omitempty"`
|
|
Firmware string `json:"firmware,omitempty"`
|
|
SecureBoot bool `json:"secure_boot,omitempty"`
|
|
SettingsChanged bool `json:"settings_changed,omitempty"`
|
|
NetworkChanged bool `json:"network_changed,omitempty"`
|
|
ReinstallCommitted bool `json:"reinstall_committed,omitempty"`
|
|
CleanupPending bool `json:"cleanup_pending,omitempty"`
|
|
}
|
|
|
|
func vmEffectProjection(value VMMetadata) operationVMEffectProjection {
|
|
return operationVMEffectProjection{
|
|
UUID: value.UUID, Name: value.Name, Status: value.Status,
|
|
VCPUs: value.VCPUs, MemoryMiB: value.MemoryMiB, DiskGiB: value.DiskGiB,
|
|
BandwidthMbps: value.BandwidthMbps, Hostname: value.Hostname,
|
|
PublicNetwork: value.PublicNetwork, TemplateID: value.TemplateID,
|
|
Machine: value.Machine, Firmware: value.Firmware, SecureBoot: value.SecureBoot,
|
|
SettingsChanged: value.SettingsChanged, NetworkChanged: value.NetworkChanged,
|
|
ReinstallCommitted: value.ReinstallCommitted, CleanupPending: value.CleanupPending,
|
|
}
|
|
}
|
|
|
|
type operationSnapshotEffectProjection struct {
|
|
ID string `json:"id"`
|
|
SnapshotKey string `json:"snapshot_key"`
|
|
Name string `json:"name,omitempty"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
State string `json:"state"`
|
|
Consistency string `json:"consistency"`
|
|
SizeBytes uint64 `json:"size_bytes,omitempty"`
|
|
Created bool `json:"created,omitempty"`
|
|
}
|
|
|
|
type operationCatalogImportProjection struct {
|
|
ID string `json:"id"`
|
|
OwnerID int64 `json:"owner_id,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
OSFamily string `json:"os_family,omitempty"`
|
|
Version string `json:"version,omitempty"`
|
|
Architecture string `json:"architecture,omitempty"`
|
|
SizeBytes uint64 `json:"size_bytes,omitempty"`
|
|
VirtualSizeBytes uint64 `json:"virtual_size_bytes,omitempty"`
|
|
SHA256 string `json:"sha256,omitempty"`
|
|
Revision uint64 `json:"revision,omitempty"`
|
|
GenerationToken string `json:"generation_token,omitempty"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
}
|
|
|
|
func decodeOperationEffectResult(raw []byte, target any) bool {
|
|
if len(raw) == 0 || len(raw) > maxOperationEffectProjectionBytes {
|
|
return false
|
|
}
|
|
trimmed := strings.TrimSpace(string(raw))
|
|
return len(trimmed) >= 2 && trimmed[0] == '{' && trimmed[len(trimmed)-1] == '}' &&
|
|
json.Unmarshal(raw, target) == nil
|
|
}
|
|
|
|
func recordHasCleanupPending(record operation.Record) bool {
|
|
if len(record.Result) == 0 {
|
|
return false
|
|
}
|
|
var result struct {
|
|
CleanupPending bool `json:"cleanup_pending"`
|
|
}
|
|
return json.Unmarshal(record.Result, &result) == nil && result.CleanupPending
|
|
}
|
|
|
|
func operationInspectionToken(record operation.Record, kind, evidence string) string {
|
|
digest := sha256.New()
|
|
for _, part := range []string{
|
|
"virt-cloud-operation-inspection-v1", record.OperationID, record.ResourceID,
|
|
record.Action, record.Fingerprint, kind, evidence,
|
|
} {
|
|
digest.Write([]byte(part))
|
|
digest.Write([]byte{0})
|
|
}
|
|
return hex.EncodeToString(digest.Sum(nil))
|
|
}
|
|
|
|
func lowerHexDigest(value string) bool {
|
|
if len(value) != sha256.Size*2 {
|
|
return false
|
|
}
|
|
for _, current := range value {
|
|
if (current < '0' || current > '9') && (current < 'a' || current > 'f') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|