You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1150 lines
50 KiB
Go
1150 lines
50 KiB
Go
//go:build libvirt
|
|
|
|
package hypervisor
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.luoli.live/root/virt/agent/operation"
|
|
"git.luoli.live/root/virt/agent/peermigration"
|
|
"libvirt.org/go/libvirt"
|
|
)
|
|
|
|
type cleanupJobDomainFake struct {
|
|
active bool
|
|
activeErr error
|
|
job *libvirt.DomainJobInfo
|
|
jobErr error
|
|
document string
|
|
inactiveXML string
|
|
xmlErr error
|
|
blockInfo map[string]*libvirt.DomainBlockJobInfo
|
|
blockErr map[string]error
|
|
blockQueries []string
|
|
}
|
|
|
|
func (f *cleanupJobDomainFake) IsActive() (bool, error) { return f.active, f.activeErr }
|
|
|
|
func (f *cleanupJobDomainFake) GetJobStats(libvirt.DomainGetJobStatsFlags) (*libvirt.DomainJobInfo, error) {
|
|
return f.job, f.jobErr
|
|
}
|
|
|
|
func (f *cleanupJobDomainFake) GetXMLDesc(flags libvirt.DomainXMLFlags) (string, error) {
|
|
if flags == libvirt.DOMAIN_XML_INACTIVE && f.inactiveXML != "" {
|
|
return f.inactiveXML, f.xmlErr
|
|
}
|
|
return f.document, f.xmlErr
|
|
}
|
|
|
|
func (f *cleanupJobDomainFake) GetBlockJobInfo(target string, _ libvirt.DomainBlockJobInfoFlags) (*libvirt.DomainBlockJobInfo, error) {
|
|
f.blockQueries = append(f.blockQueries, target)
|
|
if err := f.blockErr[target]; err != nil {
|
|
return nil, err
|
|
}
|
|
if info, ok := f.blockInfo[target]; ok {
|
|
return info, nil
|
|
}
|
|
return &libvirt.DomainBlockJobInfo{}, nil
|
|
}
|
|
|
|
func TestCleanupJobEvidenceRequiresIdleDomainAndBlockJobs(t *testing.T) {
|
|
document := `<domain><devices><disk device="disk"><target dev="vdb"/></disk><disk device="disk"><target dev="vda"/></disk></devices></domain>`
|
|
idle := &cleanupJobDomainFake{
|
|
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document,
|
|
}
|
|
evidence, safe := inspectCleanupJobEvidence(idle)
|
|
joinedEvidence := strings.Join(evidence, "\n")
|
|
if !safe || !reflect.DeepEqual(idle.blockQueries, []string{"vda", "vdb"}) ||
|
|
!strings.Contains(joinedEvidence, "domain-runtime:active:job-none") ||
|
|
!strings.Contains(joinedEvidence, "disk-target:vda") ||
|
|
!strings.Contains(joinedEvidence, "disk-target:vdb") ||
|
|
!strings.Contains(joinedEvidence, "persistent-domain-xml:") {
|
|
t.Fatalf("idle job evidence=%v safe=%v queries=%v", evidence, safe, idle.blockQueries)
|
|
}
|
|
|
|
inactive := &cleanupJobDomainFake{document: document}
|
|
evidence, safe = inspectCleanupJobEvidence(inactive)
|
|
if !safe || len(inactive.blockQueries) != 0 || evidence[0] != "domain-runtime:inactive" {
|
|
t.Fatalf("inactive job evidence=%v safe=%v queries=%v", evidence, safe, inactive.blockQueries)
|
|
}
|
|
|
|
for name, domain := range map[string]*cleanupJobDomainFake{
|
|
"active domain job": {
|
|
active: true, job: activeBackupJob(), document: document,
|
|
},
|
|
"active block job": {
|
|
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document,
|
|
blockInfo: map[string]*libvirt.DomainBlockJobInfo{"vda": {Type: libvirt.DOMAIN_BLOCK_JOB_TYPE_COPY, End: 10}},
|
|
},
|
|
"block state unreadable": {
|
|
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE}, document: document,
|
|
blockErr: map[string]error{"vda": errors.New("query failed")},
|
|
},
|
|
"disk evidence incomplete": {
|
|
active: true, job: &libvirt.DomainJobInfo{Type: libvirt.DOMAIN_JOB_NONE},
|
|
document: `<domain><devices><disk device="disk"/></devices></domain>`,
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if evidence, safe := inspectCleanupJobEvidence(domain); safe || len(evidence) != 0 {
|
|
t.Fatalf("unsafe job evidence=%v safe=%v", evidence, safe)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBackupRecoveryEvidenceRequiresRootAndSeedConvergence(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
replacement, disk, diskClaim, seed bool
|
|
want backupRecoveryEvidence
|
|
}{
|
|
{name: "root and seed committed", replacement: true, disk: true, diskClaim: true, seed: true, want: backupRecoveryEvidenceApplied},
|
|
{name: "nothing dispatched", want: backupRecoveryEvidenceNotApplied},
|
|
{name: "seed only", seed: true, want: backupRecoveryEvidenceUnknown},
|
|
{name: "root only", replacement: true, disk: true, diskClaim: true, want: backupRecoveryEvidenceUnknown},
|
|
{name: "replacement residue only", replacement: true, want: backupRecoveryEvidenceUnknown},
|
|
{name: "transition claim without volume", diskClaim: true, want: backupRecoveryEvidenceUnknown},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if got := classifyBackupRecoveryEvidence(
|
|
test.replacement, test.disk, test.diskClaim, test.seed,
|
|
); got != test.want {
|
|
t.Fatalf("evidence = %v, want %v", got, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCleanupPlanIdentifierBindsPrivateExactEvidence(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-exact-cleanup-plan", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "delete-vm",
|
|
Fingerprint: operation.Fingerprint(
|
|
"DELETE", "/v1/vms/123e4567-e89b-42d3-a456-426614174000", []byte(`{}`),
|
|
),
|
|
State: operation.StateFailed,
|
|
EffectMayHaveApplied: true,
|
|
}
|
|
first := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{},
|
|
newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:first"))
|
|
second := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{},
|
|
newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:second"))
|
|
if first.Cleanup.PlanID == second.Cleanup.PlanID {
|
|
t.Fatal("cleanup plan id did not change with the exact resource set")
|
|
}
|
|
encoded, err := json.Marshal(first)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(encoded) == "" || containsAny(string(encoded), "managed-volume:first") ||
|
|
!strings.Contains(string(encoded), record.Fingerprint) {
|
|
t.Fatalf("private cleanup evidence leaked: %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestForceRequiredOperationsAlwaysUseFreshReadOnlyInspection(t *testing.T) {
|
|
for _, action := range []string{"delete-vm", "service-suspend"} {
|
|
record := operation.Record{
|
|
OperationID: "operation-fresh-" + strings.ReplaceAll(action, "-", ""),
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: action,
|
|
State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: "force_required", Message: "explicit force required"},
|
|
}
|
|
if storedOperationInspectionAvailable(record) {
|
|
t.Fatalf("%s force-required record reused cached inspection", action)
|
|
}
|
|
}
|
|
ordinaryFailure := operation.Record{
|
|
Action: "power-start", State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: "operation_failed"},
|
|
}
|
|
if !storedOperationInspectionAvailable(ordinaryFailure) {
|
|
t.Fatal("ordinary terminal failure unexpectedly required host inspection")
|
|
}
|
|
ordinaryFailure.EffectMayHaveApplied = true
|
|
if storedOperationInspectionAvailable(ordinaryFailure) {
|
|
t.Fatal("effectful terminal failure reused its stored failure instead of live host inspection")
|
|
}
|
|
}
|
|
|
|
func TestOperationInspectionPublishesBoundedEffectEvidence(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-settings-effect-evidence",
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "update-settings",
|
|
Fingerprint: operation.Fingerprint("POST", "/v1/vms/123e4567-e89b-42d3-a456-426614174000/settings", []byte(`{}`)),
|
|
State: operation.StateFailed,
|
|
EffectMayHaveApplied: true,
|
|
}
|
|
resolution := operation.Resolution{
|
|
Resolved: true,
|
|
Succeeded: true,
|
|
Result: []byte(`{
|
|
"uuid":"123e4567-e89b-42d3-a456-426614174000",
|
|
"name":"vm-123e4567",
|
|
"status":"running",
|
|
"vcpu":2,
|
|
"memory_mib":2048,
|
|
"hostname":"safe.example",
|
|
"settings_changed":true,
|
|
"root_password":"must-not-leak",
|
|
"credential":"must-not-leak",
|
|
"private_key":"must-not-leak",
|
|
"public_keys":["must-not-leak"],
|
|
"seed":"must-not-leak",
|
|
"user-data":"must-not-leak"
|
|
}`),
|
|
}
|
|
inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{})
|
|
if inspection.Action != record.Action || inspection.ResourceID != record.ResourceID ||
|
|
inspection.RequestFingerprint != record.Fingerprint {
|
|
t.Fatalf("inspection omitted immutable operation identity: %#v", inspection)
|
|
}
|
|
if inspection.EffectResolution != InspectionEffectApplied {
|
|
t.Fatalf("effect resolution = %q, want %q", inspection.EffectResolution, InspectionEffectApplied)
|
|
}
|
|
if len(inspection.EffectProjection) == 0 {
|
|
t.Fatal("applied settings inspection omitted its safe effect projection")
|
|
}
|
|
encoded, err := json.Marshal(inspection)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, forbidden := range []string{
|
|
"must-not-leak", "root_password", "credential", "private_key", "public_keys", "seed", "user-data",
|
|
} {
|
|
if strings.Contains(string(encoded), forbidden) {
|
|
t.Fatalf("inspection leaked forbidden effect data %q: %s", forbidden, encoded)
|
|
}
|
|
}
|
|
if !strings.Contains(string(inspection.EffectProjection), `"settings_changed":true`) ||
|
|
!strings.Contains(string(inspection.EffectProjection), `"hostname":"safe.example"`) {
|
|
t.Fatalf("settings projection omitted safe convergence evidence: %s", inspection.EffectProjection)
|
|
}
|
|
|
|
failed := buildOperationInspection(record, operation.Resolution{
|
|
Resolved: true, Succeeded: false, ErrorCode: "settings_not_committed",
|
|
}, OperationAbortInspection{}, operationCleanupInspection{})
|
|
if failed.EffectResolution != InspectionEffectNotApplied || len(failed.EffectProjection) != 0 {
|
|
t.Fatalf("failed effect inspection = %#v", failed)
|
|
}
|
|
|
|
exactNoEffect := inspectionFromRecord(operation.Record{
|
|
OperationID: "operation-settings-exact-no-effect",
|
|
ResourceID: record.ResourceID,
|
|
Action: "update-settings",
|
|
State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: "cloudinit_validation_failed"},
|
|
})
|
|
if exactNoEffect.EffectResolution != InspectionEffectNotApplied ||
|
|
exactNoEffect.FenceResolutionRequired ||
|
|
exactNoEffect.ErrorCode != "cloudinit_validation_failed" {
|
|
t.Fatalf("exact no-effect Journal inspection = %#v", exactNoEffect)
|
|
}
|
|
|
|
partial := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{},
|
|
newOperationCleanupInspection([]string{CleanupStepPersistCleanupMetadata}, "private-host-evidence"))
|
|
if partial.EffectResolution != InspectionEffectPartial || len(partial.EffectProjection) != 0 {
|
|
t.Fatalf("partial effect inspection = %#v", partial)
|
|
}
|
|
|
|
inconclusive := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, operationCleanupInspection{})
|
|
if inconclusive.EffectResolution != InspectionEffectInconclusive || len(inconclusive.EffectProjection) != 0 {
|
|
t.Fatalf("inconclusive effect inspection = %#v", inconclusive)
|
|
}
|
|
}
|
|
|
|
func TestReceiptBackedTombstoneInspectionRequiresProjectableEvidence(t *testing.T) {
|
|
journal, err := operation.Open(filepath.Join(t.TempDir(), "operations.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = journal.Close() })
|
|
now := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC)
|
|
journal.SetClockForTesting(func() time.Time { return now })
|
|
resourceID := "123e4567-e89b-42d3-a456-426614174000"
|
|
spec := operation.Spec{
|
|
OperationID: "operation-pruned-settings-receipt",
|
|
ResourceID: resourceID,
|
|
Action: "update-settings",
|
|
Fingerprint: operation.Fingerprint("POST", "/v1/vms/"+resourceID+"/settings", []byte(`{}`)),
|
|
Destructive: true,
|
|
}
|
|
result := json.RawMessage(`{
|
|
"uuid":"123e4567-e89b-42d3-a456-426614174000",
|
|
"name":"vm-safe",
|
|
"status":"running",
|
|
"hostname":"safe.example",
|
|
"settings_changed":true,
|
|
"internal_note":"must-not-leak"
|
|
}`)
|
|
if _, _, err := journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
|
|
return result, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, err := journal.ClaimPendingCompletions(1)
|
|
if err != nil || len(claimed) != 1 {
|
|
t.Fatalf("completion claim = %#v err=%v", claimed, err)
|
|
}
|
|
if err := journal.AcknowledgeCompletion(claimed[0].EventID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now = now.Add(91 * 24 * time.Hour)
|
|
if accepted, err := journal.UseNonce("nonce-project-pruned-receipt", now, now.Add(time.Minute)); err != nil || !accepted {
|
|
t.Fatalf("trigger detail pruning accepted=%v err=%v", accepted, err)
|
|
}
|
|
record, err := journal.GetForInspection(spec.OperationID)
|
|
if err != nil || !record.Tombstone || !record.InspectionReceiptRecovered ||
|
|
record.InspectionEvidenceUnavailable {
|
|
t.Fatalf("receipt-backed record = %#v err=%v", record, err)
|
|
}
|
|
inspection, err := (&Libvirt{}).InspectOperation(context.Background(), record)
|
|
if err != nil || inspection.Outcome != InspectionOutcomeSucceeded ||
|
|
inspection.EffectResolution != InspectionEffectApplied || len(inspection.EffectProjection) == 0 {
|
|
t.Fatalf("receipt-backed tombstone inspection = %#v err=%v", inspection, err)
|
|
}
|
|
encoded, err := json.Marshal(inspection)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(encoded), "must-not-leak") || strings.Contains(string(encoded), "internal_note") ||
|
|
!strings.Contains(string(inspection.EffectProjection), `"settings_changed":true`) {
|
|
t.Fatalf("unsafe or incomplete receipt projection: %s", encoded)
|
|
}
|
|
|
|
t.Run("tampered receipt marker", func(t *testing.T) {
|
|
tampered := record
|
|
tampered.Result = nil
|
|
tampered.InspectionReceiptRecovered = false
|
|
tampered.InspectionEvidenceUnavailable = true
|
|
inspection, err := (&Libvirt{}).InspectOperation(context.Background(), tampered)
|
|
if err != nil || inspection.Outcome != InspectionOutcomeUncertain ||
|
|
inspection.SummaryCode != InspectionSummaryIncomplete ||
|
|
inspection.EffectResolution != InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 {
|
|
t.Fatalf("tampered receipt inspection = %#v err=%v", inspection, err)
|
|
}
|
|
})
|
|
|
|
t.Run("unprojectable recovered result", func(t *testing.T) {
|
|
unprojectable := record
|
|
unprojectable.Result = json.RawMessage(`{"uuid":"123e4567-e89b-42d3-a456-426614174999","settings_changed":true}`)
|
|
inspection, err := (&Libvirt{}).InspectOperation(context.Background(), unprojectable)
|
|
if err != nil || inspection.Outcome != InspectionOutcomeUncertain ||
|
|
inspection.SummaryCode != InspectionSummaryIncomplete ||
|
|
inspection.EffectResolution != InspectionEffectInconclusive || len(inspection.EffectProjection) != 0 {
|
|
t.Fatalf("unprojectable receipt inspection = %#v err=%v", inspection, err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestTemplateOperationProjectionPreservesFinalizerIdentity(t *testing.T) {
|
|
raw := []byte(`{
|
|
"id":"template-safe",
|
|
"name":"Debian 13",
|
|
"os_family":"linux",
|
|
"version":"13",
|
|
"architecture":"amd64",
|
|
"size_bytes":1048576,
|
|
"virtual_size_bytes":2147483648,
|
|
"sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
"revision":7,
|
|
"generation_token":"0123456789abcdef0123456789abcdef",
|
|
"created_at":"2026-10-20T01:00:00Z",
|
|
"source_token":"must-not-leak"
|
|
}`)
|
|
for _, action := range []string{"import-template", "update-template"} {
|
|
t.Run(action, func(t *testing.T) {
|
|
inspection := inspectionFromRecord(operation.Record{
|
|
OperationID: "operation-template-projection-" + strings.ReplaceAll(action, "-", ""),
|
|
ResourceID: "template:template-safe",
|
|
Action: action,
|
|
State: operation.StateSucceeded,
|
|
Result: raw,
|
|
})
|
|
var projection struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Revision uint64 `json:"revision"`
|
|
GenerationToken string `json:"generation_token"`
|
|
}
|
|
if err := json.Unmarshal(inspection.EffectProjection, &projection); err != nil {
|
|
t.Fatalf("decode template projection: %v (%s)", err, inspection.EffectProjection)
|
|
}
|
|
if projection.ID != "template-safe" || projection.Name != "Debian 13" || projection.Revision != 7 ||
|
|
projection.GenerationToken != "0123456789abcdef0123456789abcdef" {
|
|
t.Fatalf("template projection omitted finalizer identity: %#v", projection)
|
|
}
|
|
if strings.Contains(string(inspection.EffectProjection), "must-not-leak") ||
|
|
strings.Contains(string(inspection.EffectProjection), "source_token") {
|
|
t.Fatalf("template projection leaked source credential: %s", inspection.EffectProjection)
|
|
}
|
|
})
|
|
}
|
|
|
|
mismatched := operationEffectProjection(operation.Record{
|
|
ResourceID: "template:different", Action: "import-template",
|
|
}, raw)
|
|
if len(mismatched) != 0 {
|
|
t.Fatalf("template identity mismatch produced projection: %s", mismatched)
|
|
}
|
|
}
|
|
|
|
func TestRescueOperationProjectionPreservesMountedGeneration(t *testing.T) {
|
|
raw := []byte(`{
|
|
"active":true,
|
|
"staged":true,
|
|
"template_id":"0123456789abcdef0123456789abcdef",
|
|
"revision":7,
|
|
"generation_token":"abcdef0123456789abcdef0123456789",
|
|
"network_suspended":false,
|
|
"warning":"must-not-leak"
|
|
}`)
|
|
projection := operationEffectProjection(operation.Record{
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "enter-rescue",
|
|
}, raw)
|
|
var decoded RescueState
|
|
if err := json.Unmarshal(projection, &decoded); err != nil {
|
|
t.Fatalf("decode rescue projection: %v (%s)", err, projection)
|
|
}
|
|
if !decoded.Active || !decoded.Staged || decoded.Revision != 7 ||
|
|
decoded.TemplateID != "0123456789abcdef0123456789abcdef" ||
|
|
decoded.GenerationToken != "abcdef0123456789abcdef0123456789" {
|
|
t.Fatalf("rescue projection omitted mounted generation: %#v", decoded)
|
|
}
|
|
if strings.Contains(string(projection), "must-not-leak") ||
|
|
strings.Contains(string(projection), "warning") {
|
|
t.Fatalf("rescue projection leaked warning text: %s", projection)
|
|
}
|
|
}
|
|
|
|
func TestVPCPortOperationProjectionBindsCompleteIdentity(t *testing.T) {
|
|
resourceID := "123e4567-e89b-42d3-a456-426614174000"
|
|
port := validVPCPortMetadata()
|
|
expected := &operation.ExpectedState{
|
|
Kind: operation.ExpectedVPCPort, VPCPresent: true,
|
|
VPCPortKey: port.PortKey, VPCKey: port.VPCKey,
|
|
VPCPrivateIP: port.PrivateIP, VPCCIDR: port.CIDR,
|
|
VPCGateway: port.Gateway, VPCMAC: port.MAC,
|
|
VPCBridge: port.Bridge, VPCRevision: port.HostRevision,
|
|
VPCDigest: port.HostDigest,
|
|
}
|
|
result := VPCPortResult{
|
|
UUID: resourceID, PortKey: port.PortKey, VPCKey: port.VPCKey,
|
|
VPCPort: &port, Status: VMStateRunning, Attached: true,
|
|
}
|
|
raw, err := json.Marshal(result)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record := operation.Record{
|
|
ResourceID: resourceID, Action: "apply-vpc-port", ExpectedState: expected,
|
|
}
|
|
projection := operationEffectProjection(record, raw)
|
|
if len(projection) == 0 {
|
|
t.Fatal("exact VPC port result did not produce a finalizer projection")
|
|
}
|
|
|
|
for name, mutate := range map[string]func(*VPCPortResult){
|
|
"resource": func(value *VPCPortResult) { value.UUID = "123e4567-e89b-42d3-a456-426614174999" },
|
|
"port key": func(value *VPCPortResult) { value.PortKey = "ffffffffffffffffffffffffffffffff" },
|
|
"private ip": func(value *VPCPortResult) { value.VPCPort.PrivateIP = "10.60.0.10" },
|
|
"host revision": func(value *VPCPortResult) { value.VPCPort.HostRevision++ },
|
|
"effect flags": func(value *VPCPortResult) { value.Removed = true },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
candidate := result
|
|
candidatePort := port
|
|
candidate.VPCPort = &candidatePort
|
|
mutate(&candidate)
|
|
encoded, marshalErr := json.Marshal(candidate)
|
|
if marshalErr != nil {
|
|
t.Fatal(marshalErr)
|
|
}
|
|
if projected := operationEffectProjection(record, encoded); len(projected) != 0 {
|
|
t.Fatalf("mismatched VPC port produced projection: %s", projected)
|
|
}
|
|
})
|
|
}
|
|
|
|
removeExpected := *expected
|
|
removeExpected.VPCPresent = false
|
|
removed := result
|
|
removed.VPCPort = nil
|
|
removed.Attached = false
|
|
removed.Removed = true
|
|
removeRaw, err := json.Marshal(removed)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if projected := operationEffectProjection(operation.Record{
|
|
ResourceID: resourceID, Action: "remove-vpc-port", ExpectedState: &removeExpected,
|
|
}, removeRaw); len(projected) == 0 {
|
|
t.Fatal("exact VPC port removal did not produce a finalizer projection")
|
|
}
|
|
}
|
|
|
|
func TestPortableBackupOperationProjectionPreservesFinalizerIdentity(t *testing.T) {
|
|
backupKey := "0123456789abcdef0123456789abcdef"
|
|
digest := strings.Repeat("a", 64)
|
|
resourceID := "123e4567-e89b-42d3-a456-426614174000"
|
|
|
|
backup := operationEffectProjection(operation.Record{
|
|
ResourceID: resourceID,
|
|
Action: "create-backup:" + backupKey + ":filesystem",
|
|
}, []byte(`{
|
|
"backup_key":"`+backupKey+`",
|
|
"size_bytes":1048576,
|
|
"virtual_size_bytes":2147483648,
|
|
"sha256":"`+digest+`",
|
|
"object_etag":"\"backup-etag\"",
|
|
"object_version_id":"version/7+opaque",
|
|
"consistency":"filesystem",
|
|
"source_contract":{"contract":1,"architecture":"x86_64","machine":"q35","firmware":"uefi","secure_boot":false,"disk_format":"qcow2","system_disk_only":true,"public_network_contract":1,"vpc_inherited":false},
|
|
"source_url":"must-not-leak"
|
|
}`))
|
|
var backupProjection BackupResult
|
|
if err := json.Unmarshal(backup, &backupProjection); err != nil {
|
|
t.Fatalf("decode backup projection: %v (%s)", err, backup)
|
|
}
|
|
if backupProjection.BackupKey != backupKey || backupProjection.SizeBytes != 1048576 ||
|
|
backupProjection.VirtualSizeBytes != 2147483648 || backupProjection.SHA256 != digest ||
|
|
backupProjection.ObjectETag != `"backup-etag"` ||
|
|
backupProjection.ObjectVersionID != "version/7+opaque" ||
|
|
backupProjection.Consistency != "filesystem" ||
|
|
backupProjection.SourceContract == nil ||
|
|
*backupProjection.SourceContract != fixedPortableRestoreContract() {
|
|
t.Fatalf("backup projection omitted finalizer identity: %#v", backupProjection)
|
|
}
|
|
if strings.Contains(string(backup), "must-not-leak") || strings.Contains(string(backup), "source_url") {
|
|
t.Fatalf("backup projection leaked transfer credential: %s", backup)
|
|
}
|
|
downgraded := operationEffectProjection(operation.Record{
|
|
ResourceID: resourceID,
|
|
Action: "create-backup:" + backupKey + ":filesystem",
|
|
}, []byte(`{"backup_key":"`+backupKey+`","size_bytes":1,"virtual_size_bytes":1,"sha256":"`+digest+`","consistency":"crash","source_contract":{"contract":1,"architecture":"x86_64","machine":"q35","firmware":"uefi","secure_boot":false,"disk_format":"qcow2","system_disk_only":true,"public_network_contract":1,"vpc_inherited":false}}`))
|
|
if len(downgraded) == 0 {
|
|
t.Fatal("filesystem backup downgraded to crash consistency lost its finalizer projection")
|
|
}
|
|
missingContract := operationEffectProjection(operation.Record{
|
|
ResourceID: resourceID,
|
|
Action: "create-backup:" + backupKey + ":filesystem",
|
|
}, []byte(`{"backup_key":"`+backupKey+`","size_bytes":1,"virtual_size_bytes":1,"sha256":"`+digest+`","consistency":"filesystem"}`))
|
|
if len(missingContract) != 0 {
|
|
t.Fatalf("backup without source contract produced projection: %s", missingContract)
|
|
}
|
|
wrongContract := fixedPortableRestoreContract()
|
|
wrongContract.Machine = "pc-i440fx"
|
|
wrongResult := BackupResult{
|
|
BackupKey: backupKey, SizeBytes: 1, VirtualSizeBytes: 1,
|
|
SHA256: digest, Consistency: "filesystem", SourceContract: &wrongContract,
|
|
}
|
|
wrongRaw, err := json.Marshal(wrongResult)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if projected := operationEffectProjection(operation.Record{
|
|
ResourceID: resourceID,
|
|
Action: "create-backup:" + backupKey + ":filesystem",
|
|
}, wrongRaw); len(projected) != 0 {
|
|
t.Fatalf("backup with wrong source contract produced projection: %s", projected)
|
|
}
|
|
|
|
restore := operationEffectProjection(operation.Record{
|
|
ResourceID: resourceID,
|
|
Action: "restore-backup:" + backupKey,
|
|
}, []byte(`{
|
|
"uuid":"`+resourceID+`",
|
|
"name":"restored-vm",
|
|
"status":"running",
|
|
"backup_key":"`+backupKey+`",
|
|
"sha256":"`+digest+`",
|
|
"source_url":"must-not-leak"
|
|
}`))
|
|
var restoreProjection RestoreBackupResult
|
|
if err := json.Unmarshal(restore, &restoreProjection); err != nil {
|
|
t.Fatalf("decode restore projection: %v (%s)", err, restore)
|
|
}
|
|
if restoreProjection.UUID != resourceID || restoreProjection.Name != "restored-vm" ||
|
|
restoreProjection.BackupKey != backupKey || restoreProjection.SHA256 != digest {
|
|
t.Fatalf("restore projection omitted finalizer identity: %#v", restoreProjection)
|
|
}
|
|
if strings.Contains(string(restore), "must-not-leak") || strings.Contains(string(restore), "source_url") {
|
|
t.Fatalf("restore projection leaked transfer credential: %s", restore)
|
|
}
|
|
|
|
for name, test := range map[string]struct {
|
|
record operation.Record
|
|
raw []byte
|
|
}{
|
|
"backup key mismatch": {
|
|
record: operation.Record{ResourceID: resourceID, Action: "create-backup:" + backupKey + ":filesystem"},
|
|
raw: []byte(`{"backup_key":"ffffffffffffffffffffffffffffffff","size_bytes":1,"virtual_size_bytes":1,"sha256":"` + digest + `","consistency":"filesystem"}`),
|
|
},
|
|
"backup consistency upgrade": {
|
|
record: operation.Record{ResourceID: resourceID, Action: "create-backup:" + backupKey + ":crash"},
|
|
raw: []byte(`{"backup_key":"` + backupKey + `","size_bytes":1,"virtual_size_bytes":1,"sha256":"` + digest + `","consistency":"filesystem"}`),
|
|
},
|
|
"restore uuid mismatch": {
|
|
record: operation.Record{ResourceID: resourceID, Action: "restore-backup:" + backupKey},
|
|
raw: []byte(`{"uuid":"123e4567-e89b-42d3-a456-426614174999","backup_key":"` + backupKey + `","sha256":"` + digest + `"}`),
|
|
},
|
|
"restore digest malformed": {
|
|
record: operation.Record{ResourceID: resourceID, Action: "restore-backup:" + backupKey},
|
|
raw: []byte(`{"uuid":"` + resourceID + `","backup_key":"` + backupKey + `","sha256":"ABC"}`),
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if projected := operationEffectProjection(test.record, test.raw); len(projected) != 0 {
|
|
t.Fatalf("identity mismatch produced projection: %s", projected)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPeerMigrationStateProjectionProvesPowerAndAutostart(t *testing.T) {
|
|
resourceID := "123e4567-e89b-42d3-a456-426614174000"
|
|
migrationID := strings.Repeat("a", 32)
|
|
record := operation.Record{
|
|
ResourceID: resourceID,
|
|
Action: "activate-peer-migration:" + migrationID,
|
|
ExpectedState: &operation.ExpectedState{
|
|
Kind: operation.ExpectedPeerMigrationState,
|
|
Enabled: true, StartIfSuspended: false,
|
|
},
|
|
}
|
|
projected := operationEffectProjection(record, []byte(`{
|
|
"uuid":"`+resourceID+`",
|
|
"state":"shutoff",
|
|
"running":false,
|
|
"autostart_enabled":true,
|
|
"name":"must-not-leak"
|
|
}`))
|
|
var proof struct {
|
|
UUID string `json:"uuid"`
|
|
State VMStateName `json:"state"`
|
|
Running bool `json:"running"`
|
|
AutostartEnabled bool `json:"autostart_enabled"`
|
|
}
|
|
if err := json.Unmarshal(projected, &proof); err != nil {
|
|
t.Fatalf("decode peer state projection: %v (%s)", err, projected)
|
|
}
|
|
if proof.UUID != resourceID || proof.State != VMStateShutoff || proof.Running || !proof.AutostartEnabled {
|
|
t.Fatalf("peer state proof = %#v", proof)
|
|
}
|
|
if strings.Contains(string(projected), "must-not-leak") {
|
|
t.Fatalf("peer state projection leaked non-contract data: %s", projected)
|
|
}
|
|
|
|
for name, raw := range map[string][]byte{
|
|
"wrong runtime": []byte(`{"uuid":"` + resourceID + `","state":"running","running":true,"autostart_enabled":true}`),
|
|
"wrong autostart": []byte(`{"uuid":"` + resourceID + `","state":"shutoff","running":false,"autostart_enabled":false}`),
|
|
"missing proof": []byte(`{"uuid":"` + resourceID + `","state":"shutoff"}`),
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if value := operationEffectProjection(record, raw); len(value) != 0 {
|
|
t.Fatalf("mismatched peer state produced projection: %s", value)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPeerMigrationReceiveProjectionPreservesCanonicalNetworkAndVPC(t *testing.T) {
|
|
resourceID := "123e4567-e89b-42d3-a456-426614174000"
|
|
migrationID := strings.Repeat("a", 32)
|
|
artifact := peermigration.Artifact{
|
|
ID: strings.Repeat("b", 32), Role: "root", Consistency: "crash",
|
|
SizeBytes: 1, VirtualSizeBytes: 1, SHA256: strings.Repeat("c", 64),
|
|
CreatedAt: time.Now().Unix(),
|
|
}
|
|
definition := validDomainDefinition()
|
|
for _, withVPC := range []bool{false, true} {
|
|
var port *VPCPortMetadata
|
|
if withVPC {
|
|
value := validVPCPortMetadata()
|
|
port = &value
|
|
}
|
|
result := ReceivePeerMigrationResult{
|
|
VMMetadata: VMMetadata{
|
|
UUID: resourceID, Name: "migration-target", Status: VMStateShutoff,
|
|
VCPUs: 2, MemoryMiB: 2048, DiskGiB: 20, BandwidthMbps: 100,
|
|
PublicNetwork: definition.PublicNetwork, Machine: "q35",
|
|
Firmware: "uefi", VPCPort: port,
|
|
},
|
|
MigrationID: migrationID, ManifestDigest: strings.Repeat("d", 64),
|
|
Artifacts: []peermigration.Artifact{artifact}, StagedPoweredOff: true,
|
|
}
|
|
raw, err := json.Marshal(result)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
projected := operationEffectProjection(operation.Record{
|
|
ResourceID: resourceID,
|
|
Action: "receive-peer-migration:" + migrationID,
|
|
}, raw)
|
|
var proof struct {
|
|
operationVMEffectProjection
|
|
MigrationID string `json:"migration_id"`
|
|
ManifestDigest string `json:"manifest_digest"`
|
|
Artifacts []peermigration.Artifact `json:"artifacts"`
|
|
StagedPoweredOff bool `json:"staged_powered_off"`
|
|
}
|
|
if err := json.Unmarshal(projected, &proof); err != nil {
|
|
t.Fatalf("decode receive projection (VPC=%v): %v (%s)", withVPC, err, projected)
|
|
}
|
|
if proof.UUID != resourceID || proof.Status != VMStateShutoff ||
|
|
proof.MigrationID != migrationID || !proof.StagedPoweredOff ||
|
|
!reflect.DeepEqual(proof.PublicNetwork, definition.PublicNetwork) ||
|
|
!reflect.DeepEqual(proof.VPCPort, port) {
|
|
t.Fatalf("receive projection lost canonical identity (VPC=%v): %#v", withVPC, proof)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSuccessfulBackupCleanupPendingRetainsImmutableObjectProjection(t *testing.T) {
|
|
key := "0123456789abcdef0123456789abcdef"
|
|
record := operation.Record{
|
|
OperationID: "operation-backup-cleanup-pending",
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "create-backup:" + key + ":crash",
|
|
State: operation.StateSucceeded,
|
|
Result: json.RawMessage(`{"backup_key":"` + key +
|
|
`","size_bytes":1048576,"virtual_size_bytes":2147483648,"sha256":"` +
|
|
strings.Repeat("a", 64) + `","consistency":"crash","cleanup_pending":true,` +
|
|
`"source_contract":{"contract":1,"architecture":"x86_64","machine":"q35",` +
|
|
`"firmware":"uefi","secure_boot":false,"disk_format":"qcow2",` +
|
|
`"system_disk_only":true,"public_network_contract":1,"vpc_inherited":false}}`),
|
|
}
|
|
if !storedOperationInspectionAvailable(record) {
|
|
t.Fatal("successful backup with local cleanup residue lost its immutable result")
|
|
}
|
|
inspection := inspectionFromRecord(record)
|
|
if inspection.Outcome != InspectionOutcomeSucceeded ||
|
|
inspection.EffectResolution != InspectionEffectApplied ||
|
|
!strings.Contains(string(inspection.EffectProjection), `"cleanup_pending":true`) {
|
|
t.Fatalf("backup cleanup-pending projection = %#v", inspection)
|
|
}
|
|
if !successfulBackupCleanupPending(record) {
|
|
t.Fatal("valid successful backup cleanup marker was not recognized")
|
|
}
|
|
stored := &domainSnapshotMetadataXML{
|
|
Key: key, Consistency: "crash", SizeBytes: 2 << 30,
|
|
}
|
|
if !backupResidueCleanupAllowed(record, stored) {
|
|
t.Fatal("successful operation-owned backup residue was not eligible for cleanup inspection")
|
|
}
|
|
cleanup := (&Libvirt{}).backupResidueCleanupInspection(
|
|
&cleanupJobDomainFake{document: `<domain><devices></devices></domain>`}, nil,
|
|
snapshotVolumeName(record.ResourceID, key), key, stored,
|
|
)
|
|
inspection = appliedBackupInspectionWithCleanup(record, inspection, cleanup)
|
|
if inspection.Outcome != InspectionOutcomeSucceeded ||
|
|
inspection.EffectResolution != InspectionEffectApplied ||
|
|
!strings.Contains(string(inspection.EffectProjection), `"cleanup_pending":true`) ||
|
|
inspection.SummaryCode != InspectionSummaryCleanupRequired ||
|
|
!inspection.Cleanup.Available ||
|
|
!reflect.DeepEqual(inspection.Cleanup.Steps, []string{CleanupStepRemoveSnapshotMetadata}) {
|
|
t.Fatalf("backup applied cleanup inspection = %#v", inspection)
|
|
}
|
|
|
|
withoutMarker := record
|
|
withoutMarker.Result = json.RawMessage(strings.ReplaceAll(string(record.Result),
|
|
`,"cleanup_pending":true`, ""))
|
|
if successfulBackupCleanupPending(withoutMarker) || backupResidueCleanupAllowed(withoutMarker, stored) {
|
|
t.Fatal("successful backup without cleanup marker exposed a cleanup plan")
|
|
}
|
|
}
|
|
|
|
func TestUncertainTemplateUpdateDoesNotUseCurrentCatalogStateAsOperationEvidence(t *testing.T) {
|
|
resolution, handled, err := (&Libvirt{}).reconcileCatalogOperation(context.Background(), operation.Record{
|
|
OperationID: "operation-template-update-inspect",
|
|
ResourceID: "template:template-safe",
|
|
Action: "update-template",
|
|
State: operation.StateFailed,
|
|
EffectMayHaveApplied: true,
|
|
})
|
|
if err != nil || !handled || resolution.Resolved {
|
|
t.Fatalf("template update inspection = %#v handled=%v err=%v", resolution, handled, err)
|
|
}
|
|
}
|
|
|
|
func TestOperationInspectionDoesNotPromoteTargetStateToExactEffect(t *testing.T) {
|
|
for name, record := range map[string]operation.Record{
|
|
"power target state": {
|
|
OperationID: "operation-power-target-state", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "power-start", State: operation.StateFailed, EffectMayHaveApplied: true,
|
|
},
|
|
"network link target state": {
|
|
OperationID: "operation-network-target-state", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "network-suspend", State: operation.StateFailed, EffectMayHaveApplied: true,
|
|
},
|
|
"storage partially configured": {
|
|
OperationID: "operation-storage-target-state", ResourceID: "host-storage:7",
|
|
Action: "configure-storage", State: operation.StateFailed, EffectMayHaveApplied: true,
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
for _, resolution := range []operation.Resolution{
|
|
{Resolved: true, Succeeded: true, Result: []byte(`{"network_enabled":false}`)},
|
|
{Resolved: true, Succeeded: false, ErrorCode: "network_state_not_applied"},
|
|
} {
|
|
inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, operationCleanupInspection{})
|
|
if inspection.EffectResolution != InspectionEffectInconclusive ||
|
|
inspection.Outcome != InspectionOutcomeUncertain ||
|
|
inspection.SummaryCode != InspectionSummaryIncomplete {
|
|
t.Fatalf("target-state observation became exact effect verdict: %#v", inspection)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
stored := operation.Record{
|
|
OperationID: "operation-power-journal-success", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "power-start", State: operation.StateSucceeded,
|
|
}
|
|
if inspection := inspectionFromRecord(stored); inspection.EffectResolution != InspectionEffectApplied {
|
|
t.Fatalf("immutable Journal success lost its exact effect verdict: %#v", inspection)
|
|
} else if inspection.FenceResolutionRequired {
|
|
t.Fatalf("immutable Journal success incorrectly requires a fence resolution: %#v", inspection)
|
|
}
|
|
}
|
|
|
|
func TestOperationInspectionNeverProjectsUnknownOrMalformedResults(t *testing.T) {
|
|
for name, record := range map[string]operation.Record{
|
|
"unknown action": {
|
|
OperationID: "operation-unknown-projection", ResourceID: "resource-unknown",
|
|
Action: "future-secret-operation", State: operation.StateSucceeded,
|
|
Result: []byte(`{"password":"must-not-leak"}`),
|
|
},
|
|
"malformed known result": {
|
|
OperationID: "operation-malformed-projection", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "update-settings", State: operation.StateSucceeded,
|
|
Result: []byte(`{"hostname":"truncated"`),
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
inspection := inspectionFromRecord(record)
|
|
if inspection.EffectResolution != InspectionEffectApplied {
|
|
t.Fatalf("effect resolution = %q", inspection.EffectResolution)
|
|
}
|
|
if len(inspection.EffectProjection) != 0 {
|
|
t.Fatalf("unsafe projection escaped: %s", inspection.EffectProjection)
|
|
}
|
|
encoded, err := json.Marshal(inspection)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(encoded), "must-not-leak") || strings.Contains(string(encoded), "truncated") {
|
|
t.Fatalf("stored result leaked through inspection: %s", encoded)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDeleteInspectionClassifiesActiveInactiveAndAbsentWithoutForceReplay(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-delete-inspection-states",
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "delete-vm", Fingerprint: "request-fingerprint", State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: "force_required"},
|
|
}
|
|
activeGraceful, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || !activeGraceful.Resolved || activeGraceful.Succeeded || activeGraceful.ErrorCode != "force_required" {
|
|
t.Fatalf("active graceful inspection = %#v handled=%v", activeGraceful, handled)
|
|
}
|
|
record.State = operation.StateFailed
|
|
record.EffectMayHaveApplied = true
|
|
record.Error = nil
|
|
crashWindow, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || !crashWindow.Resolved || crashWindow.Succeeded ||
|
|
crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" {
|
|
t.Fatalf("active graceful crash-window inspection did not require a safe retry = %#v handled=%v", crashWindow, handled)
|
|
}
|
|
record.State = operation.StateDispatching
|
|
liveDispatch, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" {
|
|
t.Fatalf("live graceful dispatch was closed before the native call returned = %#v handled=%v", liveDispatch, handled)
|
|
}
|
|
record.CauseCode = "result_uncertain"
|
|
knownCause, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || knownCause.Resolved || knownCause.ErrorCode != "" {
|
|
t.Fatalf("known-cause graceful operation was closed as a restart crash = %#v handled=%v", knownCause, handled)
|
|
}
|
|
record.CauseCode = ""
|
|
if _, handled := inspectActiveDeleteResolution(record, false); handled {
|
|
t.Fatal("inactive graceful delete was treated as an active force boundary")
|
|
}
|
|
record.CauseCode = "graceful_shutdown_ambiguous"
|
|
ambiguousGraceful, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded ||
|
|
ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" {
|
|
t.Fatalf("ambiguous graceful delete did not close safely = %#v handled=%v",
|
|
ambiguousGraceful, handled)
|
|
}
|
|
record.CauseCode = ""
|
|
|
|
record.Action = "force-delete-vm"
|
|
record.State = operation.StateFailed
|
|
activeForce, handled := inspectActiveDeleteResolution(record, true)
|
|
activeForceInspection := buildOperationInspection(record, activeForce, OperationAbortInspection{}, operationCleanupInspection{})
|
|
if !handled || !activeForce.Resolved || activeForce.Succeeded ||
|
|
activeForce.ErrorCode != "force_delete_unconfirmed_active" ||
|
|
activeForceInspection.Cleanup.Available ||
|
|
activeForceInspection.Outcome != InspectionOutcomeUncertain ||
|
|
activeForceInspection.EffectResolution != InspectionEffectInconclusive ||
|
|
activeForceInspection.SummaryCode != InspectionSummaryIncomplete {
|
|
t.Fatalf("active force inspection = %#v resolution=%#v handled=%v", activeForceInspection, activeForce, handled)
|
|
}
|
|
record.State = operation.StateDispatching
|
|
liveForce, handled := inspectActiveDeleteResolution(record, true)
|
|
if !handled || liveForce.Resolved {
|
|
t.Fatalf("live force dispatch was closed before its native call returned = %#v handled=%v", liveForce, handled)
|
|
}
|
|
record.State = operation.StateFailed
|
|
record.EffectMayHaveApplied = true
|
|
|
|
inactivePlan := newOperationCleanupInspection([]string{CleanupStepUndefineDomain}, "domain:inactive")
|
|
inactiveInspection := buildOperationInspection(record, operation.Resolution{}, OperationAbortInspection{}, inactivePlan)
|
|
if !inactiveInspection.Cleanup.Available || inactiveInspection.Cleanup.Steps[0] != CleanupStepUndefineDomain ||
|
|
inactiveInspection.SummaryCode != InspectionSummaryCleanupRequired {
|
|
t.Fatalf("inactive delete inspection = %#v", inactiveInspection)
|
|
}
|
|
|
|
absent := inspectAbsentDeleteResolution(operationCleanupInspection{}, true)
|
|
absentInspection := buildOperationInspection(record, absent, OperationAbortInspection{}, operationCleanupInspection{})
|
|
var absentResult DeleteVMResult
|
|
if err := json.Unmarshal(absent.Result, &absentResult); err != nil {
|
|
t.Fatalf("decode force delete inspection result: %v", err)
|
|
}
|
|
if !absent.Resolved || !absent.Succeeded || absentInspection.Outcome != InspectionOutcomeUncertain ||
|
|
absentInspection.EffectResolution != InspectionEffectInconclusive ||
|
|
absentInspection.Cleanup.Available ||
|
|
!absentResult.Deleted || !absentResult.ManagedArtifactsAbsent || !absentResult.Forced {
|
|
t.Fatalf("absent delete inspection = %#v resolution=%#v", absentInspection, absent)
|
|
}
|
|
normalAbsent := inspectAbsentDeleteResolution(operationCleanupInspection{}, false)
|
|
var normalResult DeleteVMResult
|
|
if err := json.Unmarshal(normalAbsent.Result, &normalResult); err != nil ||
|
|
!normalResult.Deleted || !normalResult.ManagedArtifactsAbsent || normalResult.Forced {
|
|
t.Fatalf("normal delete inspection result=%#v err=%v", normalResult, err)
|
|
}
|
|
absentResidue := newOperationCleanupInspection([]string{CleanupStepDeleteManagedVolumes}, "managed-volume:one")
|
|
residueResolution := inspectAbsentDeleteResolution(absentResidue, true)
|
|
residueInspection := buildOperationInspection(record, residueResolution, OperationAbortInspection{}, absentResidue)
|
|
if residueResolution.Resolved || !residueInspection.Cleanup.Available ||
|
|
residueInspection.SummaryCode != InspectionSummaryCleanupRequired {
|
|
t.Fatalf("absent delete residue inspection = %#v resolution=%#v", residueInspection, residueResolution)
|
|
}
|
|
}
|
|
|
|
func TestServiceSuspendInspectionDoesNotInventForceAuthorizationAfterDispatchCrash(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-service-suspend-crash-window",
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "service-suspend",
|
|
State: operation.StateFailed,
|
|
EffectMayHaveApplied: true,
|
|
ExpectedState: &operation.ExpectedState{
|
|
Kind: operation.ExpectedServiceState,
|
|
},
|
|
}
|
|
crashWindow, handled := inspectActiveServiceSuspension(record, true)
|
|
if !handled || !crashWindow.Resolved || crashWindow.Succeeded ||
|
|
crashWindow.ErrorCode != "graceful_shutdown_unconfirmed" {
|
|
t.Fatalf("dispatch crash did not require a new graceful suspension = %#v handled=%v", crashWindow, handled)
|
|
}
|
|
record.State = operation.StateDispatching
|
|
liveDispatch, handled := inspectActiveServiceSuspension(record, true)
|
|
if !handled || liveDispatch.Resolved || liveDispatch.ErrorCode != "" {
|
|
t.Fatalf("live graceful suspension was closed before the native call returned = %#v handled=%v", liveDispatch, handled)
|
|
}
|
|
record.State = operation.StateFailed
|
|
record.EffectMayHaveApplied = true
|
|
record.CauseCode = "graceful_shutdown_ambiguous"
|
|
ambiguousGraceful, handled := inspectActiveServiceSuspension(record, true)
|
|
if !handled || !ambiguousGraceful.Resolved || ambiguousGraceful.Succeeded ||
|
|
ambiguousGraceful.ErrorCode != "graceful_shutdown_unconfirmed" {
|
|
t.Fatalf("ambiguous graceful suspension did not close safely = %#v handled=%v",
|
|
ambiguousGraceful, handled)
|
|
}
|
|
record.CauseCode = ""
|
|
|
|
record.State = operation.StateFailed
|
|
record.Error = &operation.PublicError{Code: "force_required", Message: "explicit force required"}
|
|
authoritative, handled := inspectActiveServiceSuspension(record, true)
|
|
if !handled || !authoritative.Resolved || authoritative.Succeeded || authoritative.ErrorCode != "force_required" {
|
|
t.Fatalf("authoritative graceful timeout was lost = %#v handled=%v", authoritative, handled)
|
|
}
|
|
}
|
|
|
|
func containsAny(value string, candidates ...string) bool {
|
|
for _, candidate := range candidates {
|
|
if len(candidate) != 0 && len(value) >= len(candidate) && strings.Contains(value, candidate) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestAbsentCreateTrafficResidueRequiresExplicitCleanup(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-create-traffic-residue",
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "create-vm",
|
|
Fingerprint: "request-fingerprint",
|
|
State: operation.StateFailed,
|
|
EffectMayHaveApplied: true,
|
|
}
|
|
resolution, steps := inspectAbsentCreateVM(record.ResourceID, true, true)
|
|
inspection := buildOperationInspection(record, resolution, OperationAbortInspection{}, steps)
|
|
if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryCleanupRequired ||
|
|
!inspection.Cleanup.Available || len(inspection.Cleanup.Steps) != 1 ||
|
|
inspection.Cleanup.Steps[0] != CleanupStepRemoveTrafficState || !lowerHexDigest(inspection.Cleanup.PlanID) {
|
|
t.Fatalf("create traffic residue inspection = %#v", inspection)
|
|
}
|
|
|
|
resolution, steps = inspectAbsentCreateVM(record.ResourceID, true, false)
|
|
inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps)
|
|
if inspection.Outcome != InspectionOutcomeFailed || inspection.SummaryCode != InspectionSummaryFailed ||
|
|
inspection.ErrorCode != "creation_absent" || inspection.EffectResolution != InspectionEffectNotApplied ||
|
|
inspection.Cleanup.Available {
|
|
t.Fatalf("clean absent create inspection = %#v", inspection)
|
|
}
|
|
var absentProjection struct {
|
|
CreationAbsent bool `json:"creation_absent"`
|
|
ManagedArtifactsAbsent bool `json:"managed_artifacts_absent"`
|
|
}
|
|
if len(inspection.EffectProjection) == 0 ||
|
|
json.Unmarshal(inspection.EffectProjection, &absentProjection) != nil ||
|
|
!absentProjection.CreationAbsent || !absentProjection.ManagedArtifactsAbsent {
|
|
t.Fatalf("clean absent create projection = %s", inspection.EffectProjection)
|
|
}
|
|
|
|
resolution, steps = inspectAbsentCreateVM(record.ResourceID, false, true)
|
|
inspection = buildOperationInspection(record, resolution, OperationAbortInspection{}, steps)
|
|
if inspection.Outcome != InspectionOutcomeUncertain || inspection.SummaryCode != InspectionSummaryIncomplete ||
|
|
inspection.Cleanup.Available {
|
|
t.Fatalf("unproven create absence inspection = %#v", inspection)
|
|
}
|
|
}
|
|
|
|
func TestFailedCreateAlwaysRequiresLiveInspection(t *testing.T) {
|
|
for name, record := range map[string]operation.Record{
|
|
"ordinary failure without effect flag": {
|
|
OperationID: "operation-create-ordinary-failure", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "create-vm", State: operation.StateFailed,
|
|
Error: &operation.PublicError{Code: "operation_failed", Message: "create failed"},
|
|
},
|
|
"unknown effect failure": {
|
|
OperationID: "operation-create-unknown-effect", ResourceID: "123e4567-e89b-42d3-a456-426614174001",
|
|
Action: "create-vm", State: operation.StateFailed, EffectMayHaveApplied: true,
|
|
Error: &operation.PublicError{Code: "result_unconfirmed", Message: "create result was not confirmed"},
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if storedOperationInspectionAvailable(record) {
|
|
t.Fatal("failed create reused its original Journal error instead of requiring live host inspection")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBackupPushMatchesOnlyExactOwnedTarget(t *testing.T) {
|
|
target := "/var/lib/virt-cloud/managed/snapshot.qcow2"
|
|
matching := `<domainbackup mode="push"><disks><disk name="vda" backup="yes" type="file"><driver type="qcow2"/><target file="` + target + `"/></disk></disks></domainbackup>`
|
|
if !backupPushMatchesTarget(matching, target) {
|
|
t.Fatal("exact push backup target was not recognized")
|
|
}
|
|
for name, document := range map[string]string{
|
|
"foreign target": `<domainbackup mode="push"><disks><disk name="vda" backup="yes" type="file"><target file="/tmp/foreign.qcow2"/></disk></disks></domainbackup>`,
|
|
"wrong disk": `<domainbackup mode="push"><disks><disk name="vdb" backup="yes" type="file"><target file="` + target + `"/></disk></disks></domainbackup>`,
|
|
"pull backup": `<domainbackup mode="pull"><server transport="unix" socket="/tmp/backup.sock"/><disks><disk name="vda" backup="yes" type="file"><scratch file="` + target + `"/></disk></disks></domainbackup>`,
|
|
"multiple disks": `<domainbackup mode="push"><disks><disk name="vda" backup="yes" type="file"><target file="` + target + `"/></disk><disk name="vdb" backup="yes" type="file"><target file="` + target + `"/></disk></disks></domainbackup>`,
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if backupPushMatchesTarget(document, target) {
|
|
t.Fatal("non-matching backup was accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBackupResidueCleanupRequiresTerminalOwnedSnapshot(t *testing.T) {
|
|
key := "0123456789abcdef0123456789abcdef"
|
|
domain := &cleanupJobDomainFake{
|
|
document: `<domain><devices><disk device="disk"><target dev="vda"/></disk></devices></domain>`,
|
|
}
|
|
for name, consistency := range map[string]struct {
|
|
requested string
|
|
actual string
|
|
}{
|
|
"crash": {requested: "crash", actual: "crash"},
|
|
"filesystem fallback": {requested: "filesystem", actual: "crash"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
record := operation.Record{
|
|
OperationID: "operation-backup-residue-" + strings.ReplaceAll(name, " ", "-"),
|
|
ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "create-backup:" + key + ":" + consistency.requested,
|
|
State: operation.StateFailed, EffectMayHaveApplied: true,
|
|
}
|
|
stored := &domainSnapshotMetadataXML{
|
|
Key: key, Consistency: consistency.actual, SizeBytes: 1 << 30,
|
|
}
|
|
if !backupResidueCleanupAllowed(record, stored) {
|
|
t.Fatal("terminal operation-owned residue was not eligible for cleanup")
|
|
}
|
|
plan := (&Libvirt{}).backupResidueCleanupInspection(
|
|
domain, nil, snapshotVolumeName(record.ResourceID, key), key, stored,
|
|
)
|
|
inspection := buildOperationInspection(record, operation.Resolution{},
|
|
OperationAbortInspection{}, plan)
|
|
if !inspection.Cleanup.Available ||
|
|
!reflect.DeepEqual(inspection.Cleanup.Steps, []string{CleanupStepRemoveSnapshotMetadata}) ||
|
|
inspection.EffectResolution != InspectionEffectPartial ||
|
|
inspection.Outcome != InspectionOutcomeUncertain {
|
|
t.Fatalf("backup residue inspection = %#v", inspection)
|
|
}
|
|
})
|
|
}
|
|
|
|
active := operation.Record{
|
|
OperationID: "operation-backup-residue-active", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
|
|
Action: "create-backup:" + key + ":filesystem", State: operation.StateDispatching,
|
|
}
|
|
stored := &domainSnapshotMetadataXML{Key: key, Consistency: "crash", SizeBytes: 1 << 30}
|
|
if backupResidueCleanupAllowed(active, stored) {
|
|
t.Fatal("active backup worker exposed a local residue cleanup plan")
|
|
}
|
|
fenced := active
|
|
fenced.State = operation.StateFailed
|
|
fenced.EffectMayHaveApplied = true
|
|
fenced.ExecutionFenced = true
|
|
if backupResidueCleanupAllowed(fenced, stored) {
|
|
t.Fatal("execution-fenced backup worker exposed a local residue cleanup plan")
|
|
}
|
|
|
|
failed := active
|
|
failed.State = operation.StateFailed
|
|
failed.EffectMayHaveApplied = true
|
|
busyDomain := &cleanupJobDomainFake{
|
|
active: true, job: activeBackupJob(), document: domain.document,
|
|
}
|
|
if plan := (&Libvirt{}).backupResidueCleanupInspection(
|
|
busyDomain, nil, snapshotVolumeName(failed.ResourceID, key), key, stored,
|
|
); len(plan.steps) != 0 {
|
|
t.Fatalf("active libvirt backup job exposed cleanup steps: %#v", plan.steps)
|
|
}
|
|
}
|