You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_operation_admin_tes...

691 lines
31 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"encoding/json"
"errors"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
)
type cleanupJobDomainFake struct {
active bool
activeErr error
job *libvirt.DomainJobInfo
jobErr error
document string
inactiveXML string
xmlErr error
blockInfo map[string]*libvirt.DomainBlockJobInfo
blockErr map[string]error
blockQueries []string
}
func (f *cleanupJobDomainFake) IsActive() (bool, error) { return f.active, f.activeErr }
func (f *cleanupJobDomainFake) GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) {
return f.job, f.jobErr
}
func (f *cleanupJobDomainFake) GetXMLDesc(flags libvirt.DomainXMLFlags) (string, error) {
if flags == libvirt.DOMAIN_XML_INACTIVE && f.inactiveXML != "" {
return f.inactiveXML, f.xmlErr
}
return f.document, f.xmlErr
}
func (f *cleanupJobDomainFake) GetBlockJobInfo(target string, _ libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error) {
f.blockQueries = append(f.blockQueries, target)
if err := f.blockErr[target]; err != nil {
return nil, err
}
if info, ok := f.blockInfo[target]; ok {
return info, nil
}
return &libvirt.DomainBlockJobInfo{}, nil
}
func TestCleanupJobEvidenceRequiresIdleDomainAndBlockJobs(t *testing.T) {
document := `<domain><devices><disk device="disk"><target dev="vdb"/></disk><disk device="disk"><target dev="vda"/></disk></devices></domain>`
idle := &cleanupJobDomainFake{
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document,
}
evidence, safe := inspectCleanupJobEvidence(idle)
joinedEvidence := strings.Join(evidence, "\n")
if !safe || !reflect.DeepEqual(idle.blockQueries, []string{"vda", "vdb"}) ||
!strings.Contains(joinedEvidence, "domain-runtime:active:job-none") ||
!strings.Contains(joinedEvidence, "disk-target:vda") ||
!strings.Contains(joinedEvidence, "disk-target:vdb") ||
!strings.Contains(joinedEvidence, "persistent-domain-xml:") {
t.Fatalf("idle job evidence=%v safe=%v queries=%v", evidence, safe, idle.blockQueries)
}
inactive := &cleanupJobDomainFake{document: document}
evidence, safe = inspectCleanupJobEvidence(inactive)
if !safe || len(inactive.blockQueries) != 0 || evidence[0] != "domain-runtime:inactive" {
t.Fatalf("inactive job evidence=%v safe=%v queries=%v", evidence, safe, inactive.blockQueries)
}
for name, domain := range map[string]*cleanupJobDomainFake{
"active domain job": {
active: true, job: activeBackupJob(), document: document,
},
"active block job": {
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document,
blockInfo: map[string]*libvirt.DomainBlockJobInfo{"vda": {Type: libvirt.DOMAIN_BLOCK_JOB_TYPE_COPY, End: 10}},
},
"block state unreadable": {
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document,
blockErr: map[string]error{"vda": errors.New("query failed")},
},
"disk evidence incomplete": {
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE},
document: `<domain><devices><disk device="disk"/></devices></domain>`,
},
} {
t.Run(name, func(t *testing.T) {
if evidence, safe := inspectCleanupJobEvidence(domain); safe || len(evidence) != 0 {
t.Fatalf("unsafe job evidence=%v safe=%v", evidence, safe)
}
})
}
}
func TestCleanupPlanIdentifierBindsPrivateExactEvidence(t *testing.T) {
record := operation.Record{
OperationID: "operation-exact-cleanup-plan", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "delete-vm",
Fingerprint: operation.Fingerprint(
"DELETE", "/v1/vms/123e4567-e89b-42d3-a456-426614174000", []byte(`{}`),
),
State: operation.StateFailed,
EffectMayHaveApplied: true,
}
first := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{},
newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:first"))
second := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{},
newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:second"))
if first.Cleanup.PlanID == second.Cleanup.PlanID {
t.Fatal("cleanup plan id did not change with the exact resource set")
}
encoded, err := json.Marshal(first)
if err != nil {
t.Fatal(err)
}
if string(encoded) == "" || containsAny(string(encoded), "managed-volume:first") ||
!strings.Contains(string(encoded), record.Fingerprint) {
t.Fatalf("private cleanup evidence leaked: %s", encoded)
}
}
func TestForceRequiredOperationsAlwaysUseFreshReadOnlyInspection(t *testing.T) {
for _, action := range []string{"delete-vm", "service-suspend"} {
record := operation.Record{
OperationID: "operation-fresh-" + strings.ReplaceAll(action, "-", ""),
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: action,
State: operation.StateFailed,
Error: &operation.PublicError{Code: "force_required", Message: "explicit force required"},
}
if storedOperationInspectionAvailable(record) {
t.Fatalf("%s force-required record reused cached inspection", action)
}
}
ordinaryFailure := operation.Record{
Action: "power-start", State: operation.StateFailed,
Error: &operation.PublicError{Code: "operation_failed"},
}
if !storedOperationInspectionAvailable(ordinaryFailure) {
t.Fatal("ordinary terminal failure unexpectedly required host inspection")
}
ordinaryFailure.EffectMayHaveApplied = true
if storedOperationInspectionAvailable(ordinaryFailure) {
t.Fatal("effectful terminal failure reused its stored failure instead of live host inspection")
}
}
func TestOperationInspectionPublishesBoundedEffectEvidence(t *testing.T) {
record := operation.Record{
OperationID: "operation-settings-effect-evidence",
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "update-settings",
Fingerprint: operation.Fingerprint("POST", "/v1/vms/123e4567-e89b-42d3-a456-426614174000/settings", []byte(`{}`)),
State: operation.StateFailed,
EffectMayHaveApplied: true,
}
resolution := operation.Resolution{
Resolved: true,
Succeeded: true,
Result: []byte(`{
"uuid":"123e4567-e89b-42d3-a456-426614174000",
"name":"vm-123e4567",
"status":"running",
"vcpu":2,
"memory_mib":2048,
"hostname":"safe.example",
"settings_changed":true,
"root_password":"must-not-leak",
"credential":"must-not-leak",
"private_key":"must-not-leak",
"public_keys":["must-not-leak"],
"seed":"must-not-leak",
"user-data":"must-not-leak"
}`),
}
inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{})
if inspection.Action != record.Action || inspection.ResourceID != record.ResourceID ||
inspection.RequestFingerprint != record.Fingerprint {
t.Fatalf("inspection omitted immutable operation identity: %#v", inspection)
}
if inspection.EffectResolution != InspectionEffectApplied {
t.Fatalf("effect resolution = %q, want %q", inspection.EffectResolution, InspectionEffectApplied)
}
if len(inspection.EffectProjection) == 0 {
t.Fatal("applied settings inspection omitted its safe effect projection")
}
encoded, err := json.Marshal(inspection)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{
"must-not-leak", "root_password", "credential", "private_key", "public_keys", "seed", "user-data",
} {
if strings.Contains(string(encoded), forbidden) {
t.Fatalf("inspection leaked forbidden effect data %q: %s", forbidden, encoded)
}
}
if !strings.Contains(string(inspection.EffectProjection), `"settings_changed":true`) ||
!strings.Contains(string(inspection.EffectProjection), `"hostname":"safe.example"`) {
t.Fatalf("settings projection omitted safe convergence evidence: %s", inspection.EffectProjection)
}
failed := buildOperationInspection(record, operation.Resolution{
Resolved: true, Succeeded: false, ErrorCode: "settings_not_committed",
}, OperationAbortInspection{}, operationCleanupInspection{})
if failed.EffectResolution != InspectionEffectNotApplied || len(failed.EffectProjection) != 0 {
t.Fatalf("failed effect inspection = %#v", failed)
}
exactNoEffect := inspectionFromRecord(operation.Record{
OperationID: "operation-settings-exact-no-effect",
ResourceID: record.ResourceID,
Action: "update-settings",
State: operation.StateFailed,
Error: &operation.PublicError{Code: "cloudinit_validation_failed"},
})
if exactNoEffect.EffectResolution != InspectionEffectNotApplied ||
exactNoEffect.FenceResolutionRequired ||
exactNoEffect.ErrorCode != "cloudinit_validation_failed" {
t.Fatalf("exact no-effect Journal inspection = %#v", exactNoEffect)
}
partial := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{},
newOperationCleanupInspection([]string{CleanupStepPersistCleanupMetadata}, "private-host-evidence"))
if partial.EffectResolution != InspectionEffectPartial || len(partial.EffectProjection) != 0 {
t.Fatalf("partial effect inspection = %#v", partial)
}
inconclusive := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, operationCleanupInspection{})
if inconclusive.EffectResolution != InspectionEffectInconclusive || len(inconclusive.EffectProjection) != 0 {
t.Fatalf("inconclusive effect inspection = %#v", inconclusive)
}
}
func TestReceiptBackedTombstoneInspectionRequiresProjectableEvidence(t *testing.T) {
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = journal.Close() })
now := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC)
journal.SetClockForTesting(func() time.Time { return now })
resourceID := "123e4567-e89b-42d3-a456-426614174000"
spec := operation.Spec{
OperationID: "operation-pruned-settings-receipt",
ResourceID: resourceID,
Action: "update-settings",
Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+resourceID+"/settings", []byte(`{}`)),
Destructive: true,
}
result := json.RawMessage(`{
"uuid":"123e4567-e89b-42d3-a456-426614174000",
"name":"vm-safe",
"status":"running",
"hostname":"safe.example",
"settings_changed":true,
"internal_note":"must-not-leak"
}`)
if _, _, err := journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
return result, nil
}); err != nil {
t.Fatal(err)
}
claimed, err := journal.ClaimPendingCompletions(1)
if err != nil || len(claimed) != 1 {
t.Fatalf("completion claim = %#v err=%v", claimed, err)
}
if err := journal.AcknowledgeCompletion(claimed[0].EventID); err != nil {
t.Fatal(err)
}
now = now.Add(91 * 24 * time.Hour)
if accepted, err := journal.UseNonce("nonce-project-pruned-receipt", now, now.Add(time.Minute)); err != nil || !accepted {
t.Fatalf("trigger detail pruning accepted=%v err=%v", accepted, err)
}
record, err := journal.GetForInspection(spec.OperationID)
if err != nil || !record.Tombstone || !record.InspectionReceiptRecovered ||
record.InspectionEvidenceUnavailable {
t.Fatalf("receipt-backed record = %#v err=%v", record, err)
}
inspection, err := (&Libvirt{}).InspectOperation(context.Background(), record)
if err != nil || inspection.Outcome != InspectionOutcomeSucceeded ||
inspection.EffectResolution != InspectionEffectApplied || len(inspection.EffectProjection) == 0 {
t.Fatalf("receipt-backed tombstone inspection = %#v err=%v", inspection, err)
}
encoded, err := json.Marshal(inspection)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(encoded), "must-not-leak") || strings.Contains(string(encoded), "internal_note") ||
!strings.Contains(string(inspection.EffectProjection), `"settings_changed":true`) {
t.Fatalf("unsafe or incomplete receipt projection: %s", encoded)
}
t.Run("tampered receipt marker", func(t *testing.T) {
tampered := record
tampered.Result = nil
tampered.InspectionReceiptRecovered = false
tampered.InspectionEvidenceUnavailable = true
inspection, err := (&Libvirt{}).InspectOperation(context.Background(), tampered)
if err != nil || inspection.Outcome != InspectionOutcomeUncertain ||
inspection.SummaryCode != InspectionSummaryIncomplete ||
inspection.EffectResolution != InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 {
t.Fatalf("tampered receipt inspection = %#v err=%v", inspection, err)
}
})
t.Run("unprojectable recovered result", func(t *testing.T) {
unprojectable := record
unprojectable.Result = json.RawMessage(`{"uuid":"123e4567-e89b-42d3-a456-426614174999","settings_changed":true}`)
inspection, err := (&Libvirt{}).InspectOperation(context.Background(), unprojectable)
if err != nil || inspection.Outcome != InspectionOutcomeUncertain ||
inspection.SummaryCode != InspectionSummaryIncomplete ||
inspection.EffectResolution != InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 {
t.Fatalf("unprojectable receipt inspection = %#v err=%v", inspection, err)
}
})
}
func TestTemplateOperationProjectionPreservesFinalizerIdentity(t *testing.T) {
raw := []byte(`{
"id":"template-safe",
"name":"Debian 13",
"os_family":"linux",
"version":"13",
"architecture":"amd64",
"size_bytes":1048576,
"virtual_size_bytes":2147483648,
"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"revision":7,
"generation_token":"0123456789abcdef0123456789abcdef",
"created_at":"2026-10-20T01:00:00Z",
"source_token":"must-not-leak"
}`)
for _, action := range []string{"import-template", "update-template"} {
t.Run(action, func(t *testing.T) {
inspection := inspectionFromRecord(operation.Record{
OperationID: "operation-template-projection-" + strings.ReplaceAll(action, "-", ""),
ResourceID: "template:template-safe",
Action: action,
State: operation.StateSucceeded,
Result: raw,
})
var projection struct {
ID string `json:"id"`
Name string `json:"name"`
Revision uint64 `json:"revision"`
GenerationToken string `json:"generation_token"`
}
if err := json.Unmarshal(inspection.EffectProjection, &projection); err != nil {
t.Fatalf("decode template projection: %v (%s)", err, inspection.EffectProjection)
}
if projection.ID != "template-safe" || projection.Name != "Debian 13" || projection.Revision != 7 ||
projection.GenerationToken != "0123456789abcdef0123456789abcdef" {
t.Fatalf("template projection omitted finalizer identity: %#v", projection)
}
if strings.Contains(string(inspection.EffectProjection), "must-not-leak") ||
strings.Contains(string(inspection.EffectProjection), "source_token") {
t.Fatalf("template projection leaked source credential: %s", inspection.EffectProjection)
}
})
}
mismatched := operationEffectProjection(operation.Record{
ResourceID: "template:different", Action: "import-template",
}, raw)
if len(mismatched) != 0 {
t.Fatalf("template identity mismatch produced projection: %s", mismatched)
}
}
func TestUncertainTemplateUpdateDoesNotUseCurrentCatalogStateAsOperationEvidence(t *testing.T) {
resolution, handled, err := (&Libvirt{}).reconcileCatalogOperation(context.Background(), operation.Record{
OperationID: "operation-template-update-inspect",
ResourceID: "template:template-safe",
Action: "update-template",
State: operation.StateFailed,
EffectMayHaveApplied: true,
})
if err != nil || !handled || resolution.Resolved {
t.Fatalf("template update inspection = %#v handled=%v err=%v", resolution, handled, err)
}
}
func TestOperationInspectionDoesNotPromoteTargetStateToExactEffect(t *testing.T) {
for name, record := range map[string]operation.Record{
"power target state": {
OperationID: "operation-power-target-state", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "power-start", State: operation.StateFailed, EffectMayHaveApplied: true,
},
"network link target state": {
OperationID: "operation-network-target-state", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "network-suspend", State: operation.StateFailed, EffectMayHaveApplied: true,
},
"storage partially configured": {
OperationID: "operation-storage-target-state", ResourceID: "host-storage:7",
Action: "configure-storage", State: operation.StateFailed, EffectMayHaveApplied: true,
},
} {
t.Run(name, func(t *testing.T) {
for _, resolution := range []operation.Resolution{
{Resolved: true, Succeeded: true, Result: []byte(`{"network_enabled":false}`)},
{Resolved: true, Succeeded: false, ErrorCode: "network_state_not_applied"},
} {
inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{})
if inspection.EffectResolution != InspectionEffectInconclusive ||
inspection.Outcome != InspectionOutcomeUncertain ||
inspection.SummaryCode != InspectionSummaryIncomplete {
t.Fatalf("target-state observation became exact effect verdict: %#v", inspection)
}
}
})
}
stored := operation.Record{
OperationID: "operation-power-journal-success", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "power-start", State: operation.StateSucceeded,
}
if inspection := inspectionFromRecord(stored); inspection.EffectResolution != InspectionEffectApplied {
t.Fatalf("immutable Journal success lost its exact effect verdict: %#v", inspection)
} else if inspection.FenceResolutionRequired {
t.Fatalf("immutable Journal success incorrectly requires a fence resolution: %#v", inspection)
}
}
func TestOperationInspectionNeverProjectsUnknownOrMalformedResults(t *testing.T) {
for name, record := range map[string]operation.Record{
"unknown action": {
OperationID: "operation-unknown-projection", ResourceID: "resource-unknown",
Action: "future-secret-operation", State: operation.StateSucceeded,
Result: []byte(`{"password":"must-not-leak"}`),
},
"malformed known result": {
OperationID: "operation-malformed-projection", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "update-settings", State: operation.StateSucceeded,
Result: []byte(`{"hostname":"truncated"`),
},
} {
t.Run(name, func(t *testing.T) {
inspection := inspectionFromRecord(record)
if inspection.EffectResolution != InspectionEffectApplied {
t.Fatalf("effect resolution = %q", inspection.EffectResolution)
}
if len(inspection.EffectProjection) != 0 {
t.Fatalf("unsafe projection escaped: %s", inspection.EffectProjection)
}
encoded, err := json.Marshal(inspection)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(encoded), "must-not-leak") || strings.Contains(string(encoded), "truncated") {
t.Fatalf("stored result leaked through inspection: %s", encoded)
}
})
}
}
func TestDeleteInspectionClassifiesActiveInactiveAndAbsentWithoutForceReplay(t *testing.T) {
record := operation.Record{
OperationID: "operation-delete-inspection-states",
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "delete-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed,
Error: &operation.PublicError{Code: "force_required"},
}
activeGraceful, handled := inspectActiveDeleteResolution(record, true)
if !handled || !activeGraceful.Resolved || activeGraceful.Succeeded || activeGraceful.ErrorCode != "force_required" {
t.Fatalf("active graceful inspection = %#v handled=%v", activeGraceful, handled)
}
record.State = operation.StateFailed
record.EffectMayHaveApplied = true
record.Error = nil
crashWindow, handled := inspectActiveDeleteResolution(record, true)
if !handled || !crashWindow.Resolved || crashWindow.Succeeded ||
crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" {
t.Fatalf("active graceful crash-window inspection did not require a safe retry = %#v handled=%v", crashWindow, handled)
}
record.State = operation.StateDispatching
liveDispatch, handled := inspectActiveDeleteResolution(record, true)
if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" {
t.Fatalf("live graceful dispatch was closed before the native call returned = %#v handled=%v", liveDispatch, handled)
}
record.CauseCode = "result_uncertain"
knownCause, handled := inspectActiveDeleteResolution(record, true)
if !handled || knownCause.Resolved || knownCause.ErrorCode != "" {
t.Fatalf("known-cause graceful operation was closed as a restart crash = %#v handled=%v", knownCause, handled)
}
record.CauseCode = ""
if _, handled := inspectActiveDeleteResolution(record, false); handled {
t.Fatal("inactive graceful delete was treated as an active force boundary")
}
record.CauseCode = "graceful_shutdown_ambiguous"
ambiguousGraceful, handled := inspectActiveDeleteResolution(record, true)
if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded ||
ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" {
t.Fatalf("ambiguous graceful delete did not close safely = %#v handled=%v",
ambiguousGraceful, handled)
}
record.CauseCode = ""
record.Action = "force-delete-vm"
record.State = operation.StateFailed
activeForce, handled := inspectActiveDeleteResolution(record, true)
activeForceInspection := buildOperationInspection(record, activeForce, OperationAbortInspection{}, operationCleanupInspection{})
if !handled || !activeForce.Resolved || activeForce.Succeeded ||
activeForce.ErrorCode != "force_delete_unconfirmed_active" ||
activeForceInspection.Cleanup.Available ||
activeForceInspection.Outcome != InspectionOutcomeUncertain ||
activeForceInspection.EffectResolution != InspectionEffectInconclusive ||
activeForceInspection.SummaryCode != InspectionSummaryIncomplete {
t.Fatalf("active force inspection = %#v resolution=%#v handled=%v", activeForceInspection, activeForce, handled)
}
record.State = operation.StateDispatching
liveForce, handled := inspectActiveDeleteResolution(record, true)
if !handled || liveForce.Resolved {
t.Fatalf("live force dispatch was closed before its native call returned = %#v handled=%v", liveForce, handled)
}
record.State = operation.StateFailed
record.EffectMayHaveApplied = true
inactivePlan := newOperationCleanupInspection([]string{CleanupStepUndefineDomain}, "domain:inactive")
inactiveInspection := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, inactivePlan)
if !inactiveInspection.Cleanup.Available || inactiveInspection.Cleanup.Steps[0] != CleanupStepUndefineDomain ||
inactiveInspection.SummaryCode != InspectionSummaryCleanupRequired {
t.Fatalf("inactive delete inspection = %#v", inactiveInspection)
}
absent := inspectAbsentDeleteResolution(operationCleanupInspection{}, true)
absentInspection := buildOperationInspection(record, absent, OperationAbortInspection{}, operationCleanupInspection{})
var absentResult DeleteVMResult
if err := json.Unmarshal(absent.Result, &absentResult); err != nil {
t.Fatalf("decode force delete inspection result: %v", err)
}
if !absent.Resolved || !absent.Succeeded || absentInspection.Outcome != InspectionOutcomeUncertain ||
absentInspection.EffectResolution != InspectionEffectInconclusive ||
absentInspection.Cleanup.Available ||
!absentResult.Deleted || !absentResult.ManagedArtifactsAbsent || !absentResult.Forced {
t.Fatalf("absent delete inspection = %#v resolution=%#v", absentInspection, absent)
}
normalAbsent := inspectAbsentDeleteResolution(operationCleanupInspection{}, false)
var normalResult DeleteVMResult
if err := json.Unmarshal(normalAbsent.Result, &normalResult); err != nil ||
!normalResult.Deleted || !normalResult.ManagedArtifactsAbsent || normalResult.Forced {
t.Fatalf("normal delete inspection result=%#v err=%v", normalResult, err)
}
absentResidue := newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:one")
residueResolution := inspectAbsentDeleteResolution(absentResidue, true)
residueInspection := buildOperationInspection(record, residueResolution, OperationAbortInspection{}, absentResidue)
if residueResolution.Resolved || !residueInspection.Cleanup.Available ||
residueInspection.SummaryCode != InspectionSummaryCleanupRequired {
t.Fatalf("absent delete residue inspection = %#v resolution=%#v", residueInspection, residueResolution)
}
}
func TestServiceSuspendInspectionDoesNotInventForceAuthorizationAfterDispatchCrash(t *testing.T) {
record := operation.Record{
OperationID: "operation-service-suspend-crash-window",
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "service-suspend",
State: operation.StateFailed,
EffectMayHaveApplied: true,
ExpectedState: &operation.ExpectedState{
Kind: operation.ExpectedServiceState,
},
}
crashWindow, handled := inspectActiveServiceSuspension(record, true)
if !handled || !crashWindow.Resolved || crashWindow.Succeeded ||
crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" {
t.Fatalf("dispatch crash did not require a new graceful suspension = %#v handled=%v", crashWindow, handled)
}
record.State = operation.StateDispatching
liveDispatch, handled := inspectActiveServiceSuspension(record, true)
if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" {
t.Fatalf("live graceful suspension was closed before the native call returned = %#v handled=%v", liveDispatch, handled)
}
record.State = operation.StateFailed
record.EffectMayHaveApplied = true
record.CauseCode = "graceful_shutdown_ambiguous"
ambiguousGraceful, handled := inspectActiveServiceSuspension(record, true)
if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded ||
ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" {
t.Fatalf("ambiguous graceful suspension did not close safely = %#v handled=%v",
ambiguousGraceful, handled)
}
record.CauseCode = ""
record.State = operation.StateFailed
record.Error = &operation.PublicError{Code: "force_required", Message: "explicit force required"}
authoritative, handled := inspectActiveServiceSuspension(record, true)
if !handled || !authoritative.Resolved || authoritative.Succeeded || authoritative.ErrorCode != "force_required" {
t.Fatalf("authoritative graceful timeout was lost = %#v handled=%v", authoritative, handled)
}
}
func containsAny(value string, candidates ...string) bool {
for _, candidate := range candidates {
if len(candidate) != 0 && len(value) >= len(candidate) && strings.Contains(value, candidate) {
return true
}
}
return false
}
func TestAbsentCreateTrafficResidueRequiresExplicitCleanup(t *testing.T) {
record := operation.Record{
OperationID: "operation-create-traffic-residue",
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "create-vm",
Fingerprint: "request-fingerprint",
State: operation.StateFailed,
EffectMayHaveApplied: true,
}
resolution, steps := inspectAbsentCreateVM(record.ResourceID, true, true)
inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, steps)
if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryCleanupRequired ||
!inspection.Cleanup.Available || len(inspection.Cleanup.Steps) != 1 ||
inspection.Cleanup.Steps[0] != CleanupStepRemoveTrafficState || !lowerHexDigest(inspection.Cleanup.PlanID) {
t.Fatalf("create traffic residue inspection = %#v", inspection)
}
resolution, steps = inspectAbsentCreateVM(record.ResourceID, true, false)
inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps)
if inspection.Outcome != InspectionOutcomeFailed || inspection.SummaryCode != InspectionSummaryFailed ||
inspection.ErrorCode != "creation_absent" || inspection.EffectResolution != InspectionEffectNotApplied ||
inspection.Cleanup.Available {
t.Fatalf("clean absent create inspection = %#v", inspection)
}
var absentProjection struct {
CreationAbsent bool `json:"creation_absent"`
ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"`
}
if len(inspection.EffectProjection) == 0 ||
json.Unmarshal(inspection.EffectProjection, &absentProjection) != nil ||
!absentProjection.CreationAbsent || !absentProjection.ManagedArtifactsAbsent {
t.Fatalf("clean absent create projection = %s", inspection.EffectProjection)
}
resolution, steps = inspectAbsentCreateVM(record.ResourceID, false, true)
inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps)
if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete ||
inspection.Cleanup.Available {
t.Fatalf("unproven create absence inspection = %#v", inspection)
}
}
func TestFailedCreateAlwaysRequiresLiveInspection(t *testing.T) {
for name, record := range map[string]operation.Record{
"ordinary failure without effect flag": {
OperationID: "operation-create-ordinary-failure", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
Action: "create-vm", State: operation.StateFailed,
Error: &operation.PublicError{Code: "operation_failed", Message: "create failed"},
},
"unknown effect failure": {
OperationID: "operation-create-unknown-effect", ResourceID: "123e4567-e89b-42d3-a456-426614174001",
Action: "create-vm", State: operation.StateFailed, EffectMayHaveApplied: true,
Error: &operation.PublicError{Code: "result_unconfirmed", Message: "create result was not confirmed"},
},
} {
t.Run(name, func(t *testing.T) {
if storedOperationInspectionAvailable(record) {
t.Fatal("failed create reused its original Journal error instead of requiring live host inspection")
}
})
}
}
func TestBackupPushMatchesOnlyExactOwnedTarget(t *testing.T) {
target := "/var/lib/virt-cloud/managed/snapshot.qcow2"
matching := `<domainbackup mode="push"><disks><disk name="vda" backup="yes" type="file"><driver type="qcow2"/><target file="` + target + `"/></disk></disks></domainbackup>`
if !backupPushMatchesTarget(matching, target) {
t.Fatal("exact push backup target was not recognized")
}
for name, document := range map[string]string{
"foreign target": `<domainbackup mode="push"><disks><disk name="vda" backup="yes" type="file"><target file="/tmp/foreign.qcow2"/></disk></disks></domainbackup>`,
"wrong disk": `<domainbackup mode="push"><disks><disk name="vdb" backup="yes" type="file"><target file="` + target + `"/></disk></disks></domainbackup>`,
"pull backup": `<domainbackup mode="pull"><server transport="unix" socket="/tmp/backup.sock"/><disks><disk name="vda" backup="yes" type="file"><scratch file="` + target + `"/></disk></disks></domainbackup>`,
"multiple disks": `<domainbackup mode="push"><disks><disk name="vda" backup="yes" type="file"><target file="` + target + `"/></disk><disk name="vdb" backup="yes" type="file"><target file="` + target + `"/></disk></disks></domainbackup>`,
} {
t.Run(name, func(t *testing.T) {
if backupPushMatchesTarget(document, target) {
t.Fatal("non-matching backup was accepted")
}
})
}
}