github.com/djenriquez/nomad-1@v0.8.1/client/restarts_test.go (about) 1 package client 2 3 import ( 4 "fmt" 5 "testing" 6 "time" 7 8 cstructs "github.com/hashicorp/nomad/client/driver/structs" 9 "github.com/hashicorp/nomad/nomad/structs" 10 ) 11 12 func testPolicy(success bool, mode string) *structs.RestartPolicy { 13 return &structs.RestartPolicy{ 14 Interval: 2 * time.Minute, 15 Delay: 1 * time.Second, 16 Attempts: 3, 17 Mode: mode, 18 } 19 } 20 21 // withinJitter is a helper that returns whether the returned delay is within 22 // the jitter. 23 func withinJitter(expected, actual time.Duration) bool { 24 return float64((actual.Nanoseconds()-expected.Nanoseconds())/ 25 expected.Nanoseconds()) <= jitter 26 } 27 28 func testWaitResult(exit int) *cstructs.WaitResult { 29 return cstructs.NewWaitResult(exit, 0, nil) 30 } 31 32 func TestClient_RestartTracker_ModeDelay(t *testing.T) { 33 t.Parallel() 34 p := testPolicy(true, structs.RestartPolicyModeDelay) 35 rt := newRestartTracker(p, structs.JobTypeService) 36 for i := 0; i < p.Attempts; i++ { 37 state, when := rt.SetWaitResult(testWaitResult(127)).GetState() 38 if state != structs.TaskRestarting { 39 t.Fatalf("NextRestart() returned %v, want %v", state, structs.TaskRestarting) 40 } 41 if !withinJitter(p.Delay, when) { 42 t.Fatalf("NextRestart() returned %v; want %v+jitter", when, p.Delay) 43 } 44 } 45 46 // Follow up restarts should cause delay. 47 for i := 0; i < 3; i++ { 48 state, when := rt.SetWaitResult(testWaitResult(127)).GetState() 49 if state != structs.TaskRestarting { 50 t.Fail() 51 } 52 if !(when > p.Delay && when <= p.Interval) { 53 t.Fatalf("NextRestart() returned %v; want > %v and <= %v", when, p.Delay, p.Interval) 54 } 55 } 56 } 57 58 func TestClient_RestartTracker_ModeFail(t *testing.T) { 59 t.Parallel() 60 p := testPolicy(true, structs.RestartPolicyModeFail) 61 rt := newRestartTracker(p, structs.JobTypeSystem) 62 for i := 0; i < p.Attempts; i++ { 63 state, when := rt.SetWaitResult(testWaitResult(127)).GetState() 64 if state != structs.TaskRestarting { 65 t.Fatalf("NextRestart() returned %v, want %v", state, structs.TaskRestarting) 66 } 67 if !withinJitter(p.Delay, when) { 68 t.Fatalf("NextRestart() returned %v; want %v+jitter", when, p.Delay) 69 } 70 } 71 72 // Next restart should cause fail 73 if state, _ := rt.SetWaitResult(testWaitResult(127)).GetState(); state != structs.TaskNotRestarting { 74 t.Fatalf("NextRestart() returned %v; want %v", state, structs.TaskNotRestarting) 75 } 76 } 77 78 func TestClient_RestartTracker_NoRestartOnSuccess(t *testing.T) { 79 t.Parallel() 80 p := testPolicy(false, structs.RestartPolicyModeDelay) 81 rt := newRestartTracker(p, structs.JobTypeBatch) 82 if state, _ := rt.SetWaitResult(testWaitResult(0)).GetState(); state != structs.TaskTerminated { 83 t.Fatalf("NextRestart() returned %v, expected: %v", state, structs.TaskTerminated) 84 } 85 } 86 87 func TestClient_RestartTracker_ZeroAttempts(t *testing.T) { 88 t.Parallel() 89 p := testPolicy(true, structs.RestartPolicyModeFail) 90 p.Attempts = 0 91 92 // Test with a non-zero exit code 93 rt := newRestartTracker(p, structs.JobTypeService) 94 if state, when := rt.SetWaitResult(testWaitResult(1)).GetState(); state != structs.TaskNotRestarting { 95 t.Fatalf("expect no restart, got restart/delay: %v/%v", state, when) 96 } 97 98 // Even with a zero (successful) exit code non-batch jobs should exit 99 // with TaskNotRestarting 100 rt = newRestartTracker(p, structs.JobTypeService) 101 if state, when := rt.SetWaitResult(testWaitResult(0)).GetState(); state != structs.TaskNotRestarting { 102 t.Fatalf("expect no restart, got restart/delay: %v/%v", state, when) 103 } 104 105 // Batch jobs with a zero exit code and 0 attempts *do* exit cleanly 106 // with Terminated 107 rt = newRestartTracker(p, structs.JobTypeBatch) 108 if state, when := rt.SetWaitResult(testWaitResult(0)).GetState(); state != structs.TaskTerminated { 109 t.Fatalf("expect terminated, got restart/delay: %v/%v", state, when) 110 } 111 112 // Batch jobs with a non-zero exit code and 0 attempts exit with 113 // TaskNotRestarting 114 rt = newRestartTracker(p, structs.JobTypeBatch) 115 if state, when := rt.SetWaitResult(testWaitResult(1)).GetState(); state != structs.TaskNotRestarting { 116 t.Fatalf("expect no restart, got restart/delay: %v/%v", state, when) 117 } 118 } 119 120 func TestClient_RestartTracker_RestartTriggered(t *testing.T) { 121 t.Parallel() 122 p := testPolicy(true, structs.RestartPolicyModeFail) 123 p.Attempts = 0 124 rt := newRestartTracker(p, structs.JobTypeService) 125 if state, when := rt.SetRestartTriggered(false).GetState(); state != structs.TaskRestarting && when != 0 { 126 t.Fatalf("expect restart immediately, got %v %v", state, when) 127 } 128 } 129 130 func TestClient_RestartTracker_RestartTriggered_Failure(t *testing.T) { 131 t.Parallel() 132 p := testPolicy(true, structs.RestartPolicyModeFail) 133 p.Attempts = 1 134 rt := newRestartTracker(p, structs.JobTypeService) 135 if state, when := rt.SetRestartTriggered(true).GetState(); state != structs.TaskRestarting || when == 0 { 136 t.Fatalf("expect restart got %v %v", state, when) 137 } 138 if state, when := rt.SetRestartTriggered(true).GetState(); state != structs.TaskNotRestarting || when != 0 { 139 t.Fatalf("expect failed got %v %v", state, when) 140 } 141 } 142 143 func TestClient_RestartTracker_StartError_Recoverable_Fail(t *testing.T) { 144 t.Parallel() 145 p := testPolicy(true, structs.RestartPolicyModeFail) 146 rt := newRestartTracker(p, structs.JobTypeSystem) 147 recErr := structs.NewRecoverableError(fmt.Errorf("foo"), true) 148 for i := 0; i < p.Attempts; i++ { 149 state, when := rt.SetStartError(recErr).GetState() 150 if state != structs.TaskRestarting { 151 t.Fatalf("NextRestart() returned %v, want %v", state, structs.TaskRestarting) 152 } 153 if !withinJitter(p.Delay, when) { 154 t.Fatalf("NextRestart() returned %v; want %v+jitter", when, p.Delay) 155 } 156 } 157 158 // Next restart should cause fail 159 if state, _ := rt.SetStartError(recErr).GetState(); state != structs.TaskNotRestarting { 160 t.Fatalf("NextRestart() returned %v; want %v", state, structs.TaskNotRestarting) 161 } 162 } 163 164 func TestClient_RestartTracker_StartError_Recoverable_Delay(t *testing.T) { 165 t.Parallel() 166 p := testPolicy(true, structs.RestartPolicyModeDelay) 167 rt := newRestartTracker(p, structs.JobTypeSystem) 168 recErr := structs.NewRecoverableError(fmt.Errorf("foo"), true) 169 for i := 0; i < p.Attempts; i++ { 170 state, when := rt.SetStartError(recErr).GetState() 171 if state != structs.TaskRestarting { 172 t.Fatalf("NextRestart() returned %v, want %v", state, structs.TaskRestarting) 173 } 174 if !withinJitter(p.Delay, when) { 175 t.Fatalf("NextRestart() returned %v; want %v+jitter", when, p.Delay) 176 } 177 } 178 179 // Next restart should cause delay 180 state, when := rt.SetStartError(recErr).GetState() 181 if state != structs.TaskRestarting { 182 t.Fatalf("NextRestart() returned %v; want %v", state, structs.TaskRestarting) 183 } 184 if !(when > p.Delay && when <= p.Interval) { 185 t.Fatalf("NextRestart() returned %v; want > %v and <= %v", when, p.Delay, p.Interval) 186 } 187 }