github.com/kardianos/nomad@v0.1.3-0.20151022182107-b13df73ee850/nomad/worker_test.go (about) 1 package nomad 2 3 import ( 4 "log" 5 "reflect" 6 "strings" 7 "sync" 8 "testing" 9 "time" 10 11 "github.com/hashicorp/nomad/nomad/mock" 12 "github.com/hashicorp/nomad/nomad/structs" 13 "github.com/hashicorp/nomad/scheduler" 14 "github.com/hashicorp/nomad/testutil" 15 ) 16 17 type NoopScheduler struct { 18 state scheduler.State 19 planner scheduler.Planner 20 eval *structs.Evaluation 21 err error 22 } 23 24 func (n *NoopScheduler) Process(eval *structs.Evaluation) error { 25 if n.state == nil { 26 panic("missing state") 27 } 28 if n.planner == nil { 29 panic("missing planner") 30 } 31 n.eval = eval 32 return n.err 33 } 34 35 func init() { 36 scheduler.BuiltinSchedulers["noop"] = func(logger *log.Logger, s scheduler.State, p scheduler.Planner) scheduler.Scheduler { 37 n := &NoopScheduler{ 38 state: s, 39 planner: p, 40 } 41 return n 42 } 43 } 44 45 func TestWorker_dequeueEvaluation(t *testing.T) { 46 s1 := testServer(t, func(c *Config) { 47 c.NumSchedulers = 0 48 c.EnabledSchedulers = []string{structs.JobTypeService} 49 }) 50 defer s1.Shutdown() 51 testutil.WaitForLeader(t, s1.RPC) 52 53 // Create the evaluation 54 eval1 := mock.Eval() 55 testutil.WaitForResult(func() (bool, error) { 56 err := s1.evalBroker.Enqueue(eval1) 57 return err == nil, err 58 }, func(err error) { 59 t.Fatalf("err: %v", err) 60 }) 61 62 // Create a worker 63 w := &Worker{srv: s1, logger: s1.logger} 64 65 // Attempt dequeue 66 eval, token, shutdown := w.dequeueEvaluation(10 * time.Millisecond) 67 if shutdown { 68 t.Fatalf("should not shutdown") 69 } 70 if token == "" { 71 t.Fatalf("should get token") 72 } 73 74 // Ensure we get a sane eval 75 if !reflect.DeepEqual(eval, eval1) { 76 t.Fatalf("bad: %#v %#v", eval, eval1) 77 } 78 } 79 80 func TestWorker_dequeueEvaluation_paused(t *testing.T) { 81 s1 := testServer(t, func(c *Config) { 82 c.NumSchedulers = 0 83 c.EnabledSchedulers = []string{structs.JobTypeService} 84 }) 85 defer s1.Shutdown() 86 testutil.WaitForLeader(t, s1.RPC) 87 88 // Create the evaluation 89 eval1 := mock.Eval() 90 testutil.WaitForResult(func() (bool, error) { 91 err := s1.evalBroker.Enqueue(eval1) 92 return err == nil, err 93 }, func(err error) { 94 t.Fatalf("err: %v", err) 95 }) 96 97 // Create a worker 98 w := &Worker{srv: s1, logger: s1.logger} 99 w.pauseCond = sync.NewCond(&w.pauseLock) 100 101 // PAUSE the worker 102 w.SetPause(true) 103 104 go func() { 105 time.Sleep(100 * time.Millisecond) 106 w.SetPause(false) 107 }() 108 109 // Attempt dequeue 110 start := time.Now() 111 eval, token, shutdown := w.dequeueEvaluation(10 * time.Millisecond) 112 if diff := time.Since(start); diff < 100*time.Millisecond { 113 t.Fatalf("should have paused: %v", diff) 114 } 115 if shutdown { 116 t.Fatalf("should not shutdown") 117 } 118 if token == "" { 119 t.Fatalf("should get token") 120 } 121 122 // Ensure we get a sane eval 123 if !reflect.DeepEqual(eval, eval1) { 124 t.Fatalf("bad: %#v %#v", eval, eval1) 125 } 126 } 127 128 func TestWorker_dequeueEvaluation_shutdown(t *testing.T) { 129 s1 := testServer(t, func(c *Config) { 130 c.NumSchedulers = 0 131 c.EnabledSchedulers = []string{structs.JobTypeService} 132 }) 133 defer s1.Shutdown() 134 testutil.WaitForLeader(t, s1.RPC) 135 136 // Create a worker 137 w := &Worker{srv: s1, logger: s1.logger} 138 139 go func() { 140 time.Sleep(10 * time.Millisecond) 141 s1.Shutdown() 142 }() 143 144 // Attempt dequeue 145 eval, _, shutdown := w.dequeueEvaluation(10 * time.Millisecond) 146 if !shutdown { 147 t.Fatalf("should not shutdown") 148 } 149 150 // Ensure we get a sane eval 151 if eval != nil { 152 t.Fatalf("bad: %#v", eval) 153 } 154 } 155 156 func TestWorker_sendAck(t *testing.T) { 157 s1 := testServer(t, func(c *Config) { 158 c.NumSchedulers = 0 159 c.EnabledSchedulers = []string{structs.JobTypeService} 160 }) 161 defer s1.Shutdown() 162 testutil.WaitForLeader(t, s1.RPC) 163 164 // Create the evaluation 165 eval1 := mock.Eval() 166 testutil.WaitForResult(func() (bool, error) { 167 err := s1.evalBroker.Enqueue(eval1) 168 return err == nil, err 169 }, func(err error) { 170 t.Fatalf("err: %v", err) 171 }) 172 173 // Create a worker 174 w := &Worker{srv: s1, logger: s1.logger} 175 176 // Attempt dequeue 177 eval, token, _ := w.dequeueEvaluation(10 * time.Millisecond) 178 179 // Check the depth is 0, 1 unacked 180 stats := s1.evalBroker.Stats() 181 if stats.TotalReady != 0 && stats.TotalUnacked != 1 { 182 t.Fatalf("bad: %#v", stats) 183 } 184 185 // Send the Nack 186 w.sendAck(eval.ID, token, false) 187 188 // Check the depth is 1, nothing unacked 189 stats = s1.evalBroker.Stats() 190 if stats.TotalReady != 1 && stats.TotalUnacked != 0 { 191 t.Fatalf("bad: %#v", stats) 192 } 193 194 // Attempt dequeue 195 eval, token, _ = w.dequeueEvaluation(10 * time.Millisecond) 196 197 // Send the Ack 198 w.sendAck(eval.ID, token, true) 199 200 // Check the depth is 0 201 stats = s1.evalBroker.Stats() 202 if stats.TotalReady != 0 && stats.TotalUnacked != 0 { 203 t.Fatalf("bad: %#v", stats) 204 } 205 } 206 207 func TestWorker_waitForIndex(t *testing.T) { 208 s1 := testServer(t, func(c *Config) { 209 c.NumSchedulers = 0 210 c.EnabledSchedulers = []string{structs.JobTypeService} 211 }) 212 defer s1.Shutdown() 213 testutil.WaitForLeader(t, s1.RPC) 214 215 // Get the current index 216 index := s1.raft.AppliedIndex() 217 218 // Cause an increment 219 go func() { 220 time.Sleep(10 * time.Millisecond) 221 s1.raft.Barrier(0) 222 }() 223 224 // Wait for a future index 225 w := &Worker{srv: s1, logger: s1.logger} 226 err := w.waitForIndex(index+1, time.Second) 227 if err != nil { 228 t.Fatalf("err: %v", err) 229 } 230 231 // Cause a timeout 232 err = w.waitForIndex(index+100, 10*time.Millisecond) 233 if err == nil || !strings.Contains(err.Error(), "timeout") { 234 t.Fatalf("err: %v", err) 235 } 236 } 237 238 func TestWorker_invokeScheduler(t *testing.T) { 239 s1 := testServer(t, func(c *Config) { 240 c.NumSchedulers = 0 241 c.EnabledSchedulers = []string{structs.JobTypeService} 242 }) 243 defer s1.Shutdown() 244 245 w := &Worker{srv: s1, logger: s1.logger} 246 eval := mock.Eval() 247 eval.Type = "noop" 248 249 err := w.invokeScheduler(eval, structs.GenerateUUID()) 250 if err != nil { 251 t.Fatalf("err: %v", err) 252 } 253 } 254 255 func TestWorker_SubmitPlan(t *testing.T) { 256 s1 := testServer(t, func(c *Config) { 257 c.NumSchedulers = 0 258 c.EnabledSchedulers = []string{structs.JobTypeService} 259 }) 260 defer s1.Shutdown() 261 testutil.WaitForLeader(t, s1.RPC) 262 263 // Register node 264 node := mock.Node() 265 testRegisterNode(t, s1, node) 266 267 // Create the register request 268 eval1 := mock.Eval() 269 testutil.WaitForResult(func() (bool, error) { 270 err := s1.evalBroker.Enqueue(eval1) 271 return err == nil, err 272 }, func(err error) { 273 t.Fatalf("err: %v", err) 274 }) 275 evalOut, token, err := s1.evalBroker.Dequeue([]string{eval1.Type}, time.Second) 276 if err != nil { 277 t.Fatalf("err: %v", err) 278 } 279 if evalOut != eval1 { 280 t.Fatalf("Bad eval") 281 } 282 283 // Create an allocation plan 284 alloc := mock.Alloc() 285 plan := &structs.Plan{ 286 EvalID: eval1.ID, 287 NodeAllocation: map[string][]*structs.Allocation{ 288 node.ID: []*structs.Allocation{alloc}, 289 }, 290 } 291 292 // Attempt to submit a plan 293 w := &Worker{srv: s1, logger: s1.logger, evalToken: token} 294 result, state, err := w.SubmitPlan(plan) 295 if err != nil { 296 t.Fatalf("err: %v", err) 297 } 298 299 // Should have no update 300 if state != nil { 301 t.Fatalf("unexpected state update") 302 } 303 304 // Result should have allocated 305 if result == nil { 306 t.Fatalf("missing result") 307 } 308 309 if result.AllocIndex == 0 { 310 t.Fatalf("Bad: %#v", result) 311 } 312 if len(result.NodeAllocation) != 1 { 313 t.Fatalf("Bad: %#v", result) 314 } 315 } 316 317 func TestWorker_SubmitPlan_MissingNodeRefresh(t *testing.T) { 318 s1 := testServer(t, func(c *Config) { 319 c.NumSchedulers = 0 320 c.EnabledSchedulers = []string{structs.JobTypeService} 321 }) 322 defer s1.Shutdown() 323 testutil.WaitForLeader(t, s1.RPC) 324 325 // Register node 326 node := mock.Node() 327 testRegisterNode(t, s1, node) 328 329 // Create the register request 330 eval1 := mock.Eval() 331 testutil.WaitForResult(func() (bool, error) { 332 err := s1.evalBroker.Enqueue(eval1) 333 return err == nil, err 334 }, func(err error) { 335 t.Fatalf("err: %v", err) 336 }) 337 evalOut, token, err := s1.evalBroker.Dequeue([]string{eval1.Type}, time.Second) 338 if err != nil { 339 t.Fatalf("err: %v", err) 340 } 341 if evalOut != eval1 { 342 t.Fatalf("Bad eval") 343 } 344 345 // Create an allocation plan, with unregistered node 346 node2 := mock.Node() 347 alloc := mock.Alloc() 348 plan := &structs.Plan{ 349 EvalID: eval1.ID, 350 NodeAllocation: map[string][]*structs.Allocation{ 351 node2.ID: []*structs.Allocation{alloc}, 352 }, 353 } 354 355 // Attempt to submit a plan 356 w := &Worker{srv: s1, logger: s1.logger, evalToken: token} 357 result, state, err := w.SubmitPlan(plan) 358 if err != nil { 359 t.Fatalf("err: %v", err) 360 } 361 362 // Result should have allocated 363 if result == nil { 364 t.Fatalf("missing result") 365 } 366 367 // Expect no allocation and forced refresh 368 if result.AllocIndex != 0 { 369 t.Fatalf("Bad: %#v", result) 370 } 371 if result.RefreshIndex == 0 { 372 t.Fatalf("Bad: %#v", result) 373 } 374 if len(result.NodeAllocation) != 0 { 375 t.Fatalf("Bad: %#v", result) 376 } 377 378 // Should have an update 379 if state == nil { 380 t.Fatalf("expected state update") 381 } 382 } 383 384 func TestWorker_UpdateEval(t *testing.T) { 385 s1 := testServer(t, func(c *Config) { 386 c.NumSchedulers = 0 387 c.EnabledSchedulers = []string{structs.JobTypeService} 388 }) 389 defer s1.Shutdown() 390 testutil.WaitForLeader(t, s1.RPC) 391 392 // Register node 393 node := mock.Node() 394 testRegisterNode(t, s1, node) 395 396 // Create the register request 397 eval1 := mock.Eval() 398 testutil.WaitForResult(func() (bool, error) { 399 err := s1.evalBroker.Enqueue(eval1) 400 return err == nil, err 401 }, func(err error) { 402 t.Fatalf("err: %v", err) 403 }) 404 evalOut, token, err := s1.evalBroker.Dequeue([]string{eval1.Type}, time.Second) 405 if err != nil { 406 t.Fatalf("err: %v", err) 407 } 408 if evalOut != eval1 { 409 t.Fatalf("Bad eval") 410 } 411 412 eval2 := evalOut.Copy() 413 eval2.Status = structs.EvalStatusComplete 414 415 // Attempt to update eval 416 w := &Worker{srv: s1, logger: s1.logger, evalToken: token} 417 err = w.UpdateEval(eval2) 418 if err != nil { 419 t.Fatalf("err: %v", err) 420 } 421 422 out, err := s1.fsm.State().EvalByID(eval2.ID) 423 if err != nil { 424 t.Fatalf("err: %v", err) 425 } 426 if out.Status != structs.EvalStatusComplete { 427 t.Fatalf("bad: %v", out) 428 } 429 } 430 431 func TestWorker_CreateEval(t *testing.T) { 432 s1 := testServer(t, func(c *Config) { 433 c.NumSchedulers = 0 434 c.EnabledSchedulers = []string{structs.JobTypeService} 435 }) 436 defer s1.Shutdown() 437 testutil.WaitForLeader(t, s1.RPC) 438 439 // Register node 440 node := mock.Node() 441 testRegisterNode(t, s1, node) 442 443 // Create the register request 444 eval1 := mock.Eval() 445 testutil.WaitForResult(func() (bool, error) { 446 err := s1.evalBroker.Enqueue(eval1) 447 return err == nil, err 448 }, func(err error) { 449 t.Fatalf("err: %v", err) 450 }) 451 evalOut, token, err := s1.evalBroker.Dequeue([]string{eval1.Type}, time.Second) 452 if err != nil { 453 t.Fatalf("err: %v", err) 454 } 455 if evalOut != eval1 { 456 t.Fatalf("Bad eval") 457 } 458 459 eval2 := mock.Eval() 460 eval2.PreviousEval = eval1.ID 461 462 // Attempt to create eval 463 w := &Worker{srv: s1, logger: s1.logger, evalToken: token} 464 err = w.CreateEval(eval2) 465 if err != nil { 466 t.Fatalf("err: %v", err) 467 } 468 469 out, err := s1.fsm.State().EvalByID(eval2.ID) 470 if err != nil { 471 t.Fatalf("err: %v", err) 472 } 473 if out.PreviousEval != eval1.ID { 474 t.Fatalf("bad: %v", out) 475 } 476 }