github.com/kardianos/nomad@v0.1.3-0.20151022182107-b13df73ee850/nomad/leader_test.go (about) 1 package nomad 2 3 import ( 4 "errors" 5 "fmt" 6 "testing" 7 "time" 8 9 "github.com/hashicorp/nomad/nomad/mock" 10 "github.com/hashicorp/nomad/nomad/structs" 11 "github.com/hashicorp/nomad/testutil" 12 ) 13 14 func TestLeader_LeftServer(t *testing.T) { 15 s1 := testServer(t, nil) 16 defer s1.Shutdown() 17 18 s2 := testServer(t, func(c *Config) { 19 c.DevDisableBootstrap = true 20 }) 21 defer s2.Shutdown() 22 23 s3 := testServer(t, func(c *Config) { 24 c.DevDisableBootstrap = true 25 }) 26 defer s3.Shutdown() 27 servers := []*Server{s1, s2, s3} 28 testJoin(t, s1, s2, s3) 29 30 for _, s := range servers { 31 testutil.WaitForResult(func() (bool, error) { 32 peers, _ := s.raftPeers.Peers() 33 return len(peers) == 3, nil 34 }, func(err error) { 35 t.Fatalf("should have 3 peers") 36 }) 37 } 38 39 // Kill any server 40 servers[0].Shutdown() 41 42 testutil.WaitForResult(func() (bool, error) { 43 // Force remove the non-leader (transition to left state) 44 name := fmt.Sprintf("%s.%s", 45 servers[0].config.NodeName, servers[0].config.Region) 46 if err := servers[1].RemoveFailedNode(name); err != nil { 47 t.Fatalf("err: %v", err) 48 } 49 50 for _, s := range servers[1:] { 51 peers, _ := s.raftPeers.Peers() 52 return len(peers) == 2, errors.New(fmt.Sprintf("%v", peers)) 53 } 54 55 return true, nil 56 }, func(err error) { 57 t.Fatalf("err: %s", err) 58 }) 59 } 60 61 func TestLeader_LeftLeader(t *testing.T) { 62 s1 := testServer(t, nil) 63 defer s1.Shutdown() 64 65 s2 := testServer(t, func(c *Config) { 66 c.DevDisableBootstrap = true 67 }) 68 defer s2.Shutdown() 69 70 s3 := testServer(t, func(c *Config) { 71 c.DevDisableBootstrap = true 72 }) 73 defer s3.Shutdown() 74 servers := []*Server{s1, s2, s3} 75 testJoin(t, s1, s2, s3) 76 77 for _, s := range servers { 78 testutil.WaitForResult(func() (bool, error) { 79 peers, _ := s.raftPeers.Peers() 80 return len(peers) == 3, nil 81 }, func(err error) { 82 t.Fatalf("should have 3 peers") 83 }) 84 } 85 86 // Kill the leader! 87 var leader *Server 88 for _, s := range servers { 89 if s.IsLeader() { 90 leader = s 91 break 92 } 93 } 94 if leader == nil { 95 t.Fatalf("Should have a leader") 96 } 97 leader.Leave() 98 leader.Shutdown() 99 100 for _, s := range servers { 101 if s == leader { 102 continue 103 } 104 testutil.WaitForResult(func() (bool, error) { 105 peers, _ := s.raftPeers.Peers() 106 return len(peers) == 2, errors.New(fmt.Sprintf("%v", peers)) 107 }, func(err error) { 108 t.Fatalf("should have 2 peers: %v", err) 109 }) 110 } 111 } 112 113 func TestLeader_MultiBootstrap(t *testing.T) { 114 s1 := testServer(t, nil) 115 defer s1.Shutdown() 116 117 s2 := testServer(t, nil) 118 defer s2.Shutdown() 119 servers := []*Server{s1, s2} 120 testJoin(t, s1, s2) 121 122 for _, s := range servers { 123 testutil.WaitForResult(func() (bool, error) { 124 peers := s.Members() 125 return len(peers) == 2, nil 126 }, func(err error) { 127 t.Fatalf("should have 2 peers") 128 }) 129 } 130 131 // Ensure we don't have multiple raft peers 132 for _, s := range servers { 133 peers, _ := s.raftPeers.Peers() 134 if len(peers) != 1 { 135 t.Fatalf("should only have 1 raft peer!") 136 } 137 } 138 } 139 140 func TestLeader_PlanQueue_Reset(t *testing.T) { 141 s1 := testServer(t, nil) 142 defer s1.Shutdown() 143 144 s2 := testServer(t, func(c *Config) { 145 c.DevDisableBootstrap = true 146 }) 147 defer s2.Shutdown() 148 149 s3 := testServer(t, func(c *Config) { 150 c.DevDisableBootstrap = true 151 }) 152 defer s3.Shutdown() 153 servers := []*Server{s1, s2, s3} 154 testJoin(t, s1, s2, s3) 155 156 for _, s := range servers { 157 testutil.WaitForResult(func() (bool, error) { 158 peers, _ := s.raftPeers.Peers() 159 return len(peers) == 3, nil 160 }, func(err error) { 161 t.Fatalf("should have 3 peers") 162 }) 163 } 164 165 var leader *Server 166 for _, s := range servers { 167 if s.IsLeader() { 168 leader = s 169 break 170 } 171 } 172 if leader == nil { 173 t.Fatalf("Should have a leader") 174 } 175 176 if !leader.planQueue.Enabled() { 177 t.Fatalf("should enable plan queue") 178 } 179 180 for _, s := range servers { 181 if !s.IsLeader() && s.planQueue.Enabled() { 182 t.Fatalf("plan queue should not be enabled") 183 } 184 } 185 186 // Kill the leader 187 leader.Shutdown() 188 time.Sleep(100 * time.Millisecond) 189 190 // Wait for a new leader 191 leader = nil 192 testutil.WaitForResult(func() (bool, error) { 193 for _, s := range servers { 194 if s.IsLeader() { 195 leader = s 196 return true, nil 197 } 198 } 199 return false, nil 200 }, func(err error) { 201 t.Fatalf("should have leader") 202 }) 203 204 // Check that the new leader has a pending GC expiration 205 testutil.WaitForResult(func() (bool, error) { 206 return leader.planQueue.Enabled(), nil 207 }, func(err error) { 208 t.Fatalf("should enable plan queue") 209 }) 210 } 211 212 func TestLeader_EvalBroker_Reset(t *testing.T) { 213 s1 := testServer(t, func(c *Config) { 214 c.NumSchedulers = 0 215 }) 216 defer s1.Shutdown() 217 218 s2 := testServer(t, func(c *Config) { 219 c.NumSchedulers = 0 220 c.DevDisableBootstrap = true 221 }) 222 defer s2.Shutdown() 223 224 s3 := testServer(t, func(c *Config) { 225 c.NumSchedulers = 0 226 c.DevDisableBootstrap = true 227 }) 228 defer s3.Shutdown() 229 servers := []*Server{s1, s2, s3} 230 testJoin(t, s1, s2, s3) 231 testutil.WaitForLeader(t, s1.RPC) 232 233 for _, s := range servers { 234 testutil.WaitForResult(func() (bool, error) { 235 peers, _ := s.raftPeers.Peers() 236 return len(peers) == 3, nil 237 }, func(err error) { 238 t.Fatalf("should have 3 peers") 239 }) 240 } 241 242 var leader *Server 243 for _, s := range servers { 244 if s.IsLeader() { 245 leader = s 246 break 247 } 248 } 249 if leader == nil { 250 t.Fatalf("Should have a leader") 251 } 252 253 // Inject a pending eval 254 req := structs.EvalUpdateRequest{ 255 Evals: []*structs.Evaluation{mock.Eval()}, 256 } 257 _, _, err := leader.raftApply(structs.EvalUpdateRequestType, req) 258 if err != nil { 259 t.Fatalf("err: %v", err) 260 } 261 262 // Kill the leader 263 leader.Shutdown() 264 time.Sleep(100 * time.Millisecond) 265 266 // Wait for a new leader 267 leader = nil 268 testutil.WaitForResult(func() (bool, error) { 269 for _, s := range servers { 270 if s.IsLeader() { 271 leader = s 272 return true, nil 273 } 274 } 275 return false, nil 276 }, func(err error) { 277 t.Fatalf("should have leader") 278 }) 279 280 // Check that the new leader has a pending evaluation 281 testutil.WaitForResult(func() (bool, error) { 282 stats := leader.evalBroker.Stats() 283 return stats.TotalReady == 1, nil 284 }, func(err error) { 285 t.Fatalf("should have pending evaluation") 286 }) 287 } 288 289 func TestLeader_PeriodicDispatch(t *testing.T) { 290 s1 := testServer(t, func(c *Config) { 291 c.NumSchedulers = 0 292 c.EvalGCInterval = 5 * time.Millisecond 293 }) 294 defer s1.Shutdown() 295 296 // Wait for a periodic dispatch 297 testutil.WaitForResult(func() (bool, error) { 298 stats := s1.evalBroker.Stats() 299 bySched, ok := stats.ByScheduler[structs.JobTypeCore] 300 if !ok { 301 return false, nil 302 } 303 return bySched.Ready > 0, nil 304 }, func(err error) { 305 t.Fatalf("should pending job") 306 }) 307 } 308 309 func TestLeader_ReapFailedEval(t *testing.T) { 310 s1 := testServer(t, func(c *Config) { 311 c.NumSchedulers = 0 312 c.EvalDeliveryLimit = 1 313 }) 314 defer s1.Shutdown() 315 testutil.WaitForLeader(t, s1.RPC) 316 317 // Wait for a periodic dispatch 318 eval := mock.Eval() 319 testutil.WaitForResult(func() (bool, error) { 320 err := s1.evalBroker.Enqueue(eval) 321 return err == nil, err 322 }, func(err error) { 323 t.Fatalf("err: %v", err) 324 }) 325 326 // Dequeue and Nack 327 out, token, err := s1.evalBroker.Dequeue(defaultSched, time.Second) 328 if err != nil { 329 t.Fatalf("err: %v", err) 330 } 331 s1.evalBroker.Nack(out.ID, token) 332 333 // Wait updated evaluation 334 state := s1.fsm.State() 335 testutil.WaitForResult(func() (bool, error) { 336 out, err := state.EvalByID(eval.ID) 337 if err != nil { 338 return false, err 339 } 340 return out != nil && out.Status == structs.EvalStatusFailed, nil 341 }, func(err error) { 342 t.Fatalf("err: %v", err) 343 }) 344 }