@ -50,6 +50,18 @@ func testSpec(id, resource string) Spec {
}
}
func storageConfigurationSpec ( id string ) Spec {
return Spec {
OperationID : id ,
ResourceID : "host-storage" ,
Action : "configure-storage" ,
Fingerprint : Fingerprint ( "POST" , "/v1/storage/configuration" , [ ] byte ( id ) ) ,
Destructive : true ,
LockKeys : [ ] string { "storage-config" } ,
HeavyIO : true ,
}
}
func TestDefaultResourceLockKeyUsesCurrentNamespace ( t * testing . T ) {
key := defaultResourceLockKey ( "operation-resource-key" , "opaque-resource" , "delete-vm" )
if len ( key ) != len ( "resource-hash:" ) + 64 || key [ : len ( "resource-hash:" ) ] != "resource-hash:" {
@ -63,6 +75,104 @@ func TestDefaultResourceLockKeyUsesCurrentNamespace(t *testing.T) {
}
}
func TestStorageConfigurationTerminalFailureDoesNotFenceNewOperation ( t * testing . T ) {
for _ , test := range [ ] struct {
name string
executionFenced bool
} {
{ name : "effect unknown" } ,
{ name : "execution fenced" , executionFenced : true } ,
} {
t . Run ( test . name , func ( t * testing . T ) {
j := openTestJournal ( t )
first := storageConfigurationSpec ( "operation-storage-failed" )
var calls atomic . Int32
record , replayed , err := j . Execute ( context . Background ( ) , first , func ( context . Context ) ( json . RawMessage , error ) {
calls . Add ( 1 )
return nil , testFailure {
code : "storage_configuration_failed" , message : "storage configuration failed" ,
effect : true , safe : false , fence : test . executionFenced ,
}
} )
if err == nil || replayed || record . State != StateFailed || ! record . EffectMayHaveApplied ||
record . SafeToResubmit || record . ExecutionFenced != test . executionFenced {
t . Fatalf ( "first storage result = %#v replayed=%v err=%v" , record , replayed , err )
}
replayedRecord , replayed , err := j . Execute ( context . Background ( ) , first , func ( context . Context ) ( json . RawMessage , error ) {
calls . Add ( 1 )
return nil , errors . New ( "same operation must not run again" )
} )
if err != nil || ! replayed || replayedRecord . State != StateFailed || calls . Load ( ) != 1 {
t . Fatalf ( "same operation replay = %#v replayed=%v calls=%d err=%v" , replayedRecord , replayed , calls . Load ( ) , err )
}
successor := storageConfigurationSpec ( "operation-storage-successor" )
succeeded , replayed , err := j . Execute ( context . Background ( ) , successor , func ( context . Context ) ( json . RawMessage , error ) {
calls . Add ( 1 )
return json . RawMessage ( ` { "configured":true} ` ) , nil
} )
if err != nil || replayed || succeeded . State != StateSucceeded || calls . Load ( ) != 2 {
t . Fatalf ( "independent successor = %#v replayed=%v calls=%d err=%v" , succeeded , replayed , calls . Load ( ) , err )
}
} )
}
}
func TestStorageConfigurationAdmissionRemainsSerializedWhileRunning ( t * testing . T ) {
j := openTestJournal ( t )
firstStarted := make ( chan struct { } )
releaseFirst := make ( chan struct { } )
secondStarted := make ( chan struct { } )
firstDone := make ( chan error , 1 )
secondDone := make ( chan error , 1 )
go func ( ) {
_ , _ , err := j . Execute ( context . Background ( ) , storageConfigurationSpec ( "operation-storage-running" ) , func ( context . Context ) ( json . RawMessage , error ) {
close ( firstStarted )
<- releaseFirst
return nil , testFailure {
code : "storage_configuration_failed" , message : "storage configuration failed" ,
effect : true , safe : false ,
}
} )
firstDone <- err
} ( )
<- firstStarted
go func ( ) {
_ , _ , err := j . Execute ( context . Background ( ) , storageConfigurationSpec ( "operation-storage-waiting" ) , func ( context . Context ) ( json . RawMessage , error ) {
close ( secondStarted )
return json . RawMessage ( ` { "configured":true} ` ) , nil
} )
secondDone <- err
} ( )
deadline := time . Now ( ) . Add ( time . Second )
for {
select {
case <- secondStarted :
t . Fatal ( "second storage mutation crossed admission while the first was running" )
default :
}
record , err := j . Get ( "operation-storage-waiting" )
if err == nil && record . State == StateReserved {
break
}
if time . Now ( ) . After ( deadline ) {
t . Fatalf ( "second storage operation did not reach serialized admission: %#v err=%v" , record , err )
}
time . Sleep ( time . Millisecond )
}
close ( releaseFirst )
if err := <- firstDone ; err == nil {
t . Fatal ( "first storage mutation unexpectedly succeeded" )
}
if err := <- secondDone ; err != nil {
t . Fatalf ( "second storage mutation remained blocked after terminal failure: %v" , err )
}
}
func TestOperationIDIsPermanentTombstoneAndMutationRunsOnce ( t * testing . T ) {
j := openTestJournal ( t )
spec := testSpec ( "operation-single-dispatch" , "vm-single" )
@ -537,6 +647,168 @@ func TestSuccessfulForceSuccessorOnlyRemovesPredecessorBlockerIndex(t *testing.T
}
}
func TestInitializeRemovesLegacyStorageBlockerAndPreservesVMFence ( t * testing . T ) {
path := filepath . Join ( t . TempDir ( ) , "operations.db" )
j , err := Open ( path )
if err != nil {
t . Fatal ( err )
}
storage := storageConfigurationSpec ( "operation-legacy-storage" )
if _ , _ , err := j . Execute ( context . Background ( ) , storage , func ( context . Context ) ( json . RawMessage , error ) {
return nil , testFailure {
code : "storage_configuration_failed" , message : "storage effect unknown" , effect : true , safe : false ,
}
} ) ; err == nil {
t . Fatal ( "legacy storage seed unexpectedly succeeded" )
}
vm := testSpec ( "operation-legacy-vm-fence" , "vm-legacy-fence" )
if _ , _ , err := j . Execute ( context . Background ( ) , vm , func ( context . Context ) ( json . RawMessage , error ) {
return nil , testFailure { code : "vm_effect_unknown" , message : "VM effect unknown" , effect : true , safe : false }
} ) ; err == nil {
t . Fatal ( "VM fence seed unexpectedly succeeded" )
}
storageBlockerKey := blockerEntryKey ( "storage-config" , storage . OperationID )
vmLockKeys := effectiveLockKeys ( vm . OperationID , vm . ResourceID , vm . Action , vm . LockKeys )
vmBlockerKey := blockerEntryKey ( vmLockKeys [ 0 ] , vm . OperationID )
storageReceiptKey := [ ] byte ( completionEventID ( storage . OperationID ) )
var operationBefore , receiptBefore [ ] byte
if err := j . db . Update ( func ( tx * bolt . Tx ) error {
operations := tx . Bucket ( operationsBucket )
blockers := tx . Bucket ( resourceBlockersBucket )
receipts := tx . Bucket ( completionReceiptsBucket )
if err := blockers . Put ( storageBlockerKey , nil ) ; err != nil {
return err
}
operationBefore = append ( [ ] byte ( nil ) , operations . Get ( [ ] byte ( storage . OperationID ) ) ... )
receiptBefore = append ( [ ] byte ( nil ) , receipts . Get ( storageReceiptKey ) ... )
if len ( operationBefore ) == 0 || len ( receiptBefore ) == 0 {
return errors . New ( "storage operation evidence was not persisted" )
}
return nil
} ) ; err != nil {
t . Fatal ( err )
}
if err := j . Close ( ) ; err != nil {
t . Fatal ( err )
}
j , err = Open ( path )
if err != nil {
t . Fatal ( err )
}
defer j . Close ( )
if err := j . db . View ( func ( tx * bolt . Tx ) error {
operations := tx . Bucket ( operationsBucket )
blockers := tx . Bucket ( resourceBlockersBucket )
receipts := tx . Bucket ( completionReceiptsBucket )
metadata := tx . Bucket ( metadataBucket )
if string ( metadata . Get ( schemaVersionKey ) ) != string ( schemaVersion ) {
return fmt . Errorf ( "schema version changed during blocker repair" )
}
cursor := blockers . Cursor ( )
if key , _ := cursor . Seek ( storageBlockerKey ) ; bytes . Equal ( key , storageBlockerKey ) {
return errors . New ( "legacy storage blocker survived restart repair" )
}
if key , _ := cursor . Seek ( vmBlockerKey ) ; ! bytes . Equal ( key , vmBlockerKey ) {
return errors . New ( "VM blocker was removed during storage repair" )
}
if ! bytes . Equal ( operationBefore , operations . Get ( [ ] byte ( storage . OperationID ) ) ) ||
! bytes . Equal ( receiptBefore , receipts . Get ( storageReceiptKey ) ) {
return errors . New ( "storage blocker repair rewrote immutable operation evidence" )
}
return nil
} ) ; err != nil {
t . Fatal ( err )
}
successor := storageConfigurationSpec ( "operation-after-storage-repair" )
if record , replayed , err := j . Execute ( context . Background ( ) , successor , func ( context . Context ) ( json . RawMessage , error ) {
return json . RawMessage ( ` { "configured":true} ` ) , nil
} ) ; err != nil || replayed || record . State != StateSucceeded {
t . Fatalf ( "storage successor after restart repair = %#v replayed=%v err=%v" , record , replayed , err )
}
vmCalled := false
if _ , _ , err := j . Execute ( context . Background ( ) , testSpec ( "operation-after-vm-fence" , vm . ResourceID ) , func ( context . Context ) ( json . RawMessage , error ) {
vmCalled = true
return nil , nil
} ) ; ! errors . Is ( err , ErrResourceBlocked ) || vmCalled {
t . Fatalf ( "VM fence changed during storage repair: called=%v err=%v" , vmCalled , err )
}
}
func TestInitializeRejectsBrokenResourceBlockerReferencesAtomically ( t * testing . T ) {
for _ , test := range [ ] struct {
name string
seed func ( * bolt . Tx ) error
} {
{
name : "missing operation" ,
seed : func ( tx * bolt . Tx ) error {
return tx . Bucket ( resourceBlockersBucket ) . Put (
blockerEntryKey ( "storage-config" , "operation-zz-missing-reference" ) , nil ,
)
} ,
} ,
{
name : "damaged operation" ,
seed : func ( tx * bolt . Tx ) error {
operationID := "operation-zz-damaged-reference"
if err := tx . Bucket ( operationsBucket ) . Put ( [ ] byte ( operationID ) , [ ] byte ( ` { "record": ` ) ) ; err != nil {
return err
}
return tx . Bucket ( resourceBlockersBucket ) . Put ( blockerEntryKey ( "storage-config" , operationID ) , nil )
} ,
} ,
} {
t . Run ( test . name , func ( t * testing . T ) {
path := filepath . Join ( t . TempDir ( ) , "operations.db" )
j , err := Open ( path )
if err != nil {
t . Fatal ( err )
}
storage := storageConfigurationSpec ( "operation-valid-storage-reference" )
if _ , _ , err := j . Execute ( context . Background ( ) , storage , func ( context . Context ) ( json . RawMessage , error ) {
return nil , testFailure { code : "storage_configuration_failed" , message : "unknown" , effect : true }
} ) ; err == nil {
t . Fatal ( "storage seed unexpectedly succeeded" )
}
storageBlockerKey := blockerEntryKey ( "storage-config" , storage . OperationID )
if err := j . db . Update ( func ( tx * bolt . Tx ) error {
if err := tx . Bucket ( resourceBlockersBucket ) . Put ( storageBlockerKey , nil ) ; err != nil {
return err
}
return test . seed ( tx )
} ) ; err != nil {
t . Fatal ( err )
}
if err := j . Close ( ) ; err != nil {
t . Fatal ( err )
}
if reopened , err := Open ( path ) ; err == nil {
_ = reopened . Close ( )
t . Fatal ( "broken blocker reference was accepted" )
}
db , err := bolt . Open ( path , 0 o600 , nil )
if err != nil {
t . Fatal ( err )
}
defer db . Close ( )
if err := db . View ( func ( tx * bolt . Tx ) error {
cursor := tx . Bucket ( resourceBlockersBucket ) . Cursor ( )
key , _ := cursor . Seek ( storageBlockerKey )
if ! bytes . Equal ( key , storageBlockerKey ) {
return errors . New ( "failed startup partially deleted the valid storage blocker" )
}
return nil
} ) ; err != nil {
t . Fatal ( err )
}
} )
}
}
func TestFreshSchemaRejectsOldJournalWithoutMigration ( t * testing . T ) {
path := filepath . Join ( t . TempDir ( ) , "operations.db" )
db , err := bolt . Open ( path , 0 o600 , nil )