You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
337 lines
16 KiB
Go
337 lines
16 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"libvirt.org/go/libvirt"
|
|
)
|
|
|
|
type cleanupJobDomainFake struct {
|
|
active bool
|
|
activeErr error
|
|
job *libvirt.DomainJobInfo
|
|
jobErr error
|
|
document string
|
|
inactiveXML string
|
|
xmlErr error
|
|
blockInfo map[string]*libvirt.DomainBlockJobInfo
|
|
blockErr map[string]error
|
|
blockQueries []string
|
|
}
|
|
|
|
func (f *cleanupJobDomainFake) IsActive() (bool, error) { return f.active, f.activeErr }
|
|
|
|
func (f *cleanupJobDomainFake) GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) {
|
|
return f.job, f.jobErr
|
|
}
|
|
|
|
func (f *cleanupJobDomainFake) GetXMLDesc(flags libvirt.DomainXMLFlags) (string, error) {
|
|
if flags == libvirt.DOMAIN_XML_INACTIVE && f.inactiveXML != "" {
|
|
return f.inactiveXML, f.xmlErr
|
|
}
|
|
return f.document, f.xmlErr
|
|
}
|
|
|
|
func (f *cleanupJobDomainFake) GetBlockJobInfo(target string, _ libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error) {
|
|
f.blockQueries = append(f.blockQueries, target)
|
|
if err := f.blockErr[target]; err != nil {
|
|
return nil, err
|
|
}
|
|
if info, ok := f.blockInfo[target]; ok {
|
|
return info, nil
|
|
}
|
|
return &libvirt.DomainBlockJobInfo{}, nil
|
|
}
|
|
|
|
func TestCleanupJobEvidenceRequiresIdleDomainAndBlockJobs(t *testing.T) {
|
|
document := `<domain><devices><disk device="disk"><target dev="vdb"/></disk><disk device="disk"><target dev="vda"/></disk></devices></domain>`
|
|
idle := &cleanupJobDomainFake{
|
|
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document,
|
|
}
|
|
evidence, safe := inspectCleanupJobEvidence(idle)
|
|
joinedEvidence := strings.Join(evidence, "\n")
|
|
if !safe || !reflect.DeepEqual(idle.blockQueries, []string{"vda", "vdb"}) ||
|
|
!strings.Contains(joinedEvidence, "domain-runtime:active:job-none") ||
|
|
!strings.Contains(joinedEvidence, "disk-target:vda") ||
|
|
!strings.Contains(joinedEvidence, "disk-target:vdb") ||
|
|
!strings.Contains(joinedEvidence, "persistent-domain-xml:") {
|
|
t.Fatalf("idle job evidence=%v safe=%v queries=%v", evidence, safe, idle.blockQueries)
|
|
}
|
|
|
|
inactive := &cleanupJobDomainFake{document: document}
|
|
evidence, safe = inspectCleanupJobEvidence(inactive)
|
|
if !safe || len(inactive.blockQueries) != 0 || evidence[0] != "domain-runtime:inactive" {
|
|
t.Fatalf("inactive job evidence=%v safe=%v queries=%v", evidence, safe, inactive.blockQueries)
|
|
}
|
|
|
|
for name, domain := range map[string]*cleanupJobDomainFake{
|
|
"active domain job": {
|
|
active: true, job: activeBackupJob(), document: document,
|
|
},
|
|
"active block job": {
|
|
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document,
|
|
blockInfo: map[string]*libvirt.DomainBlockJobInfo{"vda": {Type: libvirt.DOMAIN_BLOCK_JOB_TYPE_COPY, End: 10}},
|
|
},
|
|
"block state unreadable": {
|
|
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document,
|
|
blockErr: map[string]error{"vda": errors.New("query failed")},
|
|
},
|
|
"disk evidence incomplete": {
|
|
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE},
|
|
document: `<domain><devices><disk device="disk"/></devices></domain>`,
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if evidence, safe := inspectCleanupJobEvidence(domain); safe || len(evidence) != 0 {
|
|
t.Fatalf("unsafe job evidence=%v safe=%v", evidence, safe)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCleanupPlanIdentifierBindsPrivateExactEvidence(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-exact-cleanup-plan", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "delete-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed,
|
|
EffectMayHaveApplied: true,
|
|
}
|
|
first := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{},
|
|
newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:first"))
|
|
second := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{},
|
|
newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:second"))
|
|
if first.Cleanup.PlanID == second.Cleanup.PlanID {
|
|
t.Fatal("cleanup plan id did not change with the exact resource set")
|
|
}
|
|
encoded, err := json.Marshal(first)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(encoded) == "" || containsAny(string(encoded), "managed-volume:first", "request-fingerprint") {
|
|
t.Fatalf("private cleanup evidence leaked: %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestForceRequiredOperationsAlwaysUseFreshReadOnlyInspection(t *testing.T) {
|
|
for _, action := range []string{"delete-vm", "service-suspend"} {
|
|
record := operation.Record{
|
|
OperationID: "operation-fresh-" + strings.ReplaceAll(action, "-", ""),
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: action,
|
|
State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: "force_required", Message: "explicit force required"},
|
|
}
|
|
if storedOperationInspectionAvailable(record) {
|
|
t.Fatalf("%s force-required record reused cached inspection", action)
|
|
}
|
|
}
|
|
ordinaryFailure := operation.Record{
|
|
Action: "power-start", State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: "operation_failed"},
|
|
}
|
|
if !storedOperationInspectionAvailable(ordinaryFailure) {
|
|
t.Fatal("ordinary terminal failure unexpectedly required host inspection")
|
|
}
|
|
}
|
|
|
|
func TestDeleteInspectionClassifiesActiveInactiveAndAbsentWithoutForceReplay(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-delete-inspection-states",
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "delete-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: "force_required"},
|
|
}
|
|
activeGraceful, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || !activeGraceful.Resolved || activeGraceful.Succeeded || activeGraceful.ErrorCode != "force_required" {
|
|
t.Fatalf("active graceful inspection = %#v handled=%v", activeGraceful, handled)
|
|
}
|
|
record.State = operation.StateFailed
|
|
record.EffectMayHaveApplied = true
|
|
record.Error = nil
|
|
crashWindow, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || !crashWindow.Resolved || crashWindow.Succeeded ||
|
|
crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" {
|
|
t.Fatalf("active graceful crash-window inspection did not require a safe retry = %#v handled=%v", crashWindow, handled)
|
|
}
|
|
record.State = operation.StateDispatching
|
|
liveDispatch, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" {
|
|
t.Fatalf("live graceful dispatch was closed before the native call returned = %#v handled=%v", liveDispatch, handled)
|
|
}
|
|
record.CauseCode = "result_uncertain"
|
|
knownCause, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || knownCause.Resolved || knownCause.ErrorCode != "" {
|
|
t.Fatalf("known-cause graceful operation was closed as a restart crash = %#v handled=%v", knownCause, handled)
|
|
}
|
|
record.CauseCode = ""
|
|
if _, handled := inspectActiveDeleteResolution(record, false); handled {
|
|
t.Fatal("inactive graceful delete was treated as an active force boundary")
|
|
}
|
|
record.CauseCode = "graceful_shutdown_ambiguous"
|
|
ambiguousGraceful, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded ||
|
|
ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" {
|
|
t.Fatalf("ambiguous graceful delete did not close safely = %#v handled=%v",
|
|
ambiguousGraceful, handled)
|
|
}
|
|
record.CauseCode = ""
|
|
|
|
record.Action = "force-delete-vm"
|
|
record.State = operation.StateFailed
|
|
activeForce, handled := inspectActiveDeleteResolution(record, true)
|
|
activeForceInspection := buildOperationInspection(record, activeForce, OperationAbortInspection{}, operationCleanupInspection{})
|
|
if !handled || !activeForce.Resolved || activeForce.Succeeded ||
|
|
activeForce.ErrorCode != "force_delete_unconfirmed_active" ||
|
|
activeForceInspection.Cleanup.Available ||
|
|
activeForceInspection.Outcome != InspectionOutcomeFailed ||
|
|
activeForceInspection.SummaryCode != InspectionSummaryFailed {
|
|
t.Fatalf("active force inspection = %#v resolution=%#v handled=%v", activeForceInspection, activeForce, handled)
|
|
}
|
|
record.State = operation.StateDispatching
|
|
liveForce, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || liveForce.Resolved {
|
|
t.Fatalf("live force dispatch was closed before its native call returned = %#v handled=%v", liveForce, handled)
|
|
}
|
|
record.State = operation.StateFailed
|
|
record.EffectMayHaveApplied = true
|
|
|
|
inactivePlan := newOperationCleanupInspection([]string{CleanupStepUndefineDomain}, "domain:inactive")
|
|
inactiveInspection := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, inactivePlan)
|
|
if !inactiveInspection.Cleanup.Available || inactiveInspection.Cleanup.Steps[0] != CleanupStepUndefineDomain ||
|
|
inactiveInspection.SummaryCode != InspectionSummaryCleanupRequired {
|
|
t.Fatalf("inactive delete inspection = %#v", inactiveInspection)
|
|
}
|
|
|
|
absent := inspectAbsentDeleteResolution(operationCleanupInspection{}, true)
|
|
absentInspection := buildOperationInspection(record, absent, OperationAbortInspection{}, operationCleanupInspection{})
|
|
var absentResult DeleteVMResult
|
|
if err := json.Unmarshal(absent.Result, &absentResult); err != nil {
|
|
t.Fatalf("decode force delete inspection result: %v", err)
|
|
}
|
|
if !absent.Resolved || !absent.Succeeded || absentInspection.Outcome != InspectionOutcomeSucceeded ||
|
|
absentInspection.Cleanup.Available ||
|
|
!absentResult.Deleted || !absentResult.ManagedArtifactsAbsent || !absentResult.Forced {
|
|
t.Fatalf("absent delete inspection = %#v resolution=%#v", absentInspection, absent)
|
|
}
|
|
normalAbsent := inspectAbsentDeleteResolution(operationCleanupInspection{}, false)
|
|
var normalResult DeleteVMResult
|
|
if err := json.Unmarshal(normalAbsent.Result, &normalResult); err != nil ||
|
|
!normalResult.Deleted || !normalResult.ManagedArtifactsAbsent || normalResult.Forced {
|
|
t.Fatalf("normal delete inspection result=%#v err=%v", normalResult, err)
|
|
}
|
|
absentResidue := newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:one")
|
|
residueResolution := inspectAbsentDeleteResolution(absentResidue, true)
|
|
residueInspection := buildOperationInspection(record, residueResolution, OperationAbortInspection{}, absentResidue)
|
|
if residueResolution.Resolved || !residueInspection.Cleanup.Available ||
|
|
residueInspection.SummaryCode != InspectionSummaryCleanupRequired {
|
|
t.Fatalf("absent delete residue inspection = %#v resolution=%#v", residueInspection, residueResolution)
|
|
}
|
|
}
|
|
|
|
func TestServiceSuspendInspectionDoesNotInventForceAuthorizationAfterDispatchCrash(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-service-suspend-crash-window",
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "service-suspend",
|
|
State: operation.StateFailed,
|
|
EffectMayHaveApplied: true,
|
|
ExpectedState: &operation.ExpectedState{
|
|
Kind: operation.ExpectedServiceState,
|
|
},
|
|
}
|
|
crashWindow, handled := inspectActiveServiceSuspension(record, true)
|
|
if !handled || !crashWindow.Resolved || crashWindow.Succeeded ||
|
|
crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" {
|
|
t.Fatalf("dispatch crash did not require a new graceful suspension = %#v handled=%v", crashWindow, handled)
|
|
}
|
|
record.State = operation.StateDispatching
|
|
liveDispatch, handled := inspectActiveServiceSuspension(record, true)
|
|
if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" {
|
|
t.Fatalf("live graceful suspension was closed before the native call returned = %#v handled=%v", liveDispatch, handled)
|
|
}
|
|
record.State = operation.StateFailed
|
|
record.EffectMayHaveApplied = true
|
|
record.CauseCode = "graceful_shutdown_ambiguous"
|
|
ambiguousGraceful, handled := inspectActiveServiceSuspension(record, true)
|
|
if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded ||
|
|
ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" {
|
|
t.Fatalf("ambiguous graceful suspension did not close safely = %#v handled=%v",
|
|
ambiguousGraceful, handled)
|
|
}
|
|
record.CauseCode = ""
|
|
|
|
record.State = operation.StateFailed
|
|
record.Error = &operation.PublicError{Code: "force_required", Message: "explicit force required"}
|
|
authoritative, handled := inspectActiveServiceSuspension(record, true)
|
|
if !handled || !authoritative.Resolved || authoritative.Succeeded || authoritative.ErrorCode != "force_required" {
|
|
t.Fatalf("authoritative graceful timeout was lost = %#v handled=%v", authoritative, handled)
|
|
}
|
|
}
|
|
|
|
func containsAny(value string, candidates ...string) bool {
|
|
for _, candidate := range candidates {
|
|
if len(candidate) != 0 && len(value) >= len(candidate) && strings.Contains(value, candidate) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestAbsentCreateTrafficResidueRequiresExplicitCleanup(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-create-traffic-residue",
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "create-vm",
|
|
Fingerprint: "request-fingerprint",
|
|
State: operation.StateFailed,
|
|
EffectMayHaveApplied: true,
|
|
}
|
|
resolution, steps := inspectAbsentCreateVM(record.ResourceID, true, true)
|
|
inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, steps)
|
|
if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryCleanupRequired ||
|
|
!inspection.Cleanup.Available || len(inspection.Cleanup.Steps) != 1 ||
|
|
inspection.Cleanup.Steps[0] != CleanupStepRemoveTrafficState || !lowerHexDigest(inspection.Cleanup.PlanID) {
|
|
t.Fatalf("create traffic residue inspection = %#v", inspection)
|
|
}
|
|
|
|
resolution, steps = inspectAbsentCreateVM(record.ResourceID, true, false)
|
|
inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps)
|
|
if inspection.Outcome != InspectionOutcomeFailed || inspection.SummaryCode != InspectionSummaryFailed ||
|
|
inspection.ErrorCode != "creation_absent" || inspection.Cleanup.Available {
|
|
t.Fatalf("clean absent create inspection = %#v", inspection)
|
|
}
|
|
|
|
resolution, steps = inspectAbsentCreateVM(record.ResourceID, false, true)
|
|
inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps)
|
|
if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete ||
|
|
inspection.Cleanup.Available {
|
|
t.Fatalf("unproven create absence inspection = %#v", inspection)
|
|
}
|
|
}
|
|
|
|
func TestBackupPushMatchesOnlyExactOwnedTarget(t *testing.T) {
|
|
target := "/var/lib/virt-cloud/managed/snapshot.qcow2"
|
|
matching := `<domainbackup mode="push"><disks><disk name="vda" backup="yes" type="file"><driver type="qcow2"/><target file="` + target + `"/></disk></disks></domainbackup>`
|
|
if !backupPushMatchesTarget(matching, target) {
|
|
t.Fatal("exact push backup target was not recognized")
|
|
}
|
|
for name, document := range map[string]string{
|
|
"foreign target": `<domainbackup mode="push"><disks><disk name="vda" backup="yes" type="file"><target file="/tmp/foreign.qcow2"/></disk></disks></domainbackup>`,
|
|
"wrong disk": `<domainbackup mode="push"><disks><disk name="vdb" backup="yes" type="file"><target file="` + target + `"/></disk></disks></domainbackup>`,
|
|
"pull backup": `<domainbackup mode="pull"><server transport="unix" socket="/tmp/backup.sock"/><disks><disk name="vda" backup="yes" type="file"><scratch file="` + target + `"/></disk></disks></domainbackup>`,
|
|
"multiple disks": `<domainbackup mode="push"><disks><disk name="vda" backup="yes" type="file"><target file="` + target + `"/></disk><disk name="vdb" backup="yes" type="file"><target file="` + target + `"/></disk></disks></domainbackup>`,
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if backupPushMatchesTarget(document, target) {
|
|
t.Fatal("non-matching backup was accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|