github.com/manicqin/nomad@v0.9.5/nomad/state/state_store_test.go (about) 1 package state 2 3 import ( 4 "context" 5 "fmt" 6 "reflect" 7 "sort" 8 "strings" 9 "testing" 10 "time" 11 12 "github.com/hashicorp/go-memdb" 13 "github.com/hashicorp/nomad/helper" 14 "github.com/hashicorp/nomad/helper/uuid" 15 "github.com/hashicorp/nomad/nomad/mock" 16 "github.com/hashicorp/nomad/nomad/structs" 17 "github.com/stretchr/testify/assert" 18 "github.com/stretchr/testify/require" 19 ) 20 21 func testStateStore(t *testing.T) *StateStore { 22 return TestStateStore(t) 23 } 24 25 func TestStateStore_Blocking_Error(t *testing.T) { 26 t.Parallel() 27 28 expected := fmt.Errorf("test error") 29 errFn := func(memdb.WatchSet, *StateStore) (interface{}, uint64, error) { 30 return nil, 0, expected 31 } 32 33 state := testStateStore(t) 34 _, idx, err := state.BlockingQuery(errFn, 10, context.Background()) 35 assert.EqualError(t, err, expected.Error()) 36 assert.Zero(t, idx) 37 } 38 39 func TestStateStore_Blocking_Timeout(t *testing.T) { 40 t.Parallel() 41 42 noopFn := func(memdb.WatchSet, *StateStore) (interface{}, uint64, error) { 43 return nil, 5, nil 44 } 45 46 state := testStateStore(t) 47 timeout := time.Now().Add(250 * time.Millisecond) 48 deadlineCtx, cancel := context.WithDeadline(context.Background(), timeout) 49 defer cancel() 50 51 _, idx, err := state.BlockingQuery(noopFn, 10, deadlineCtx) 52 assert.EqualError(t, err, context.DeadlineExceeded.Error()) 53 assert.EqualValues(t, 5, idx) 54 assert.WithinDuration(t, timeout, time.Now(), 100*time.Millisecond) 55 } 56 57 func TestStateStore_Blocking_MinQuery(t *testing.T) { 58 t.Parallel() 59 60 node := mock.Node() 61 count := 0 62 queryFn := func(ws memdb.WatchSet, s *StateStore) (interface{}, uint64, error) { 63 _, err := s.NodeByID(ws, node.ID) 64 if err != nil { 65 return nil, 0, err 66 } 67 68 count++ 69 if count == 1 { 70 return false, 5, nil 71 } else if count > 2 { 72 return false, 20, fmt.Errorf("called too many times") 73 } 74 75 return true, 11, nil 76 } 77 78 state := testStateStore(t) 79 timeout := time.Now().Add(100 * time.Millisecond) 80 deadlineCtx, cancel := context.WithDeadline(context.Background(), timeout) 81 defer cancel() 82 83 time.AfterFunc(5*time.Millisecond, func() { 84 state.UpsertNode(11, node) 85 }) 86 87 resp, idx, err := state.BlockingQuery(queryFn, 10, deadlineCtx) 88 if assert.Nil(t, err) { 89 assert.Equal(t, 2, count) 90 assert.EqualValues(t, 11, idx) 91 assert.True(t, resp.(bool)) 92 } 93 } 94 95 // COMPAT 0.11: Uses AllocUpdateRequest.Alloc 96 // This test checks that: 97 // 1) The job is denormalized 98 // 2) Allocations are created 99 func TestStateStore_UpsertPlanResults_AllocationsCreated_Denormalized(t *testing.T) { 100 t.Parallel() 101 102 state := testStateStore(t) 103 alloc := mock.Alloc() 104 job := alloc.Job 105 alloc.Job = nil 106 107 if err := state.UpsertJob(999, job); err != nil { 108 t.Fatalf("err: %v", err) 109 } 110 111 eval := mock.Eval() 112 eval.JobID = job.ID 113 114 // Create an eval 115 if err := state.UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { 116 t.Fatalf("err: %v", err) 117 } 118 119 // Create a plan result 120 res := structs.ApplyPlanResultsRequest{ 121 AllocUpdateRequest: structs.AllocUpdateRequest{ 122 Alloc: []*structs.Allocation{alloc}, 123 Job: job, 124 }, 125 EvalID: eval.ID, 126 } 127 assert := assert.New(t) 128 err := state.UpsertPlanResults(1000, &res) 129 assert.Nil(err) 130 131 ws := memdb.NewWatchSet() 132 out, err := state.AllocByID(ws, alloc.ID) 133 assert.Nil(err) 134 assert.Equal(alloc, out) 135 136 index, err := state.Index("allocs") 137 assert.Nil(err) 138 assert.EqualValues(1000, index) 139 140 if watchFired(ws) { 141 t.Fatalf("bad") 142 } 143 144 evalOut, err := state.EvalByID(ws, eval.ID) 145 assert.Nil(err) 146 assert.NotNil(evalOut) 147 assert.EqualValues(1000, evalOut.ModifyIndex) 148 } 149 150 // This test checks that: 151 // 1) The job is denormalized 152 // 2) Allocations are denormalized and updated with the diff 153 // That stopped allocs Job is unmodified 154 func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { 155 t.Parallel() 156 157 state := testStateStore(t) 158 alloc := mock.Alloc() 159 job := alloc.Job 160 alloc.Job = nil 161 162 stoppedAlloc := mock.Alloc() 163 stoppedAlloc.Job = job 164 stoppedAllocDiff := &structs.AllocationDiff{ 165 ID: stoppedAlloc.ID, 166 DesiredDescription: "desired desc", 167 ClientStatus: structs.AllocClientStatusLost, 168 } 169 preemptedAlloc := mock.Alloc() 170 preemptedAlloc.Job = job 171 preemptedAllocDiff := &structs.AllocationDiff{ 172 ID: preemptedAlloc.ID, 173 PreemptedByAllocation: alloc.ID, 174 } 175 176 require := require.New(t) 177 require.NoError(state.UpsertAllocs(900, []*structs.Allocation{stoppedAlloc, preemptedAlloc})) 178 require.NoError(state.UpsertJob(999, job)) 179 180 // modify job and ensure that stopped and preempted alloc point to original Job 181 mJob := job.Copy() 182 mJob.TaskGroups[0].Name = "other" 183 184 require.NoError(state.UpsertJob(1001, mJob)) 185 186 eval := mock.Eval() 187 eval.JobID = job.ID 188 189 // Create an eval 190 require.NoError(state.UpsertEvals(1, []*structs.Evaluation{eval})) 191 192 // Create a plan result 193 res := structs.ApplyPlanResultsRequest{ 194 AllocUpdateRequest: structs.AllocUpdateRequest{ 195 AllocsUpdated: []*structs.Allocation{alloc}, 196 AllocsStopped: []*structs.AllocationDiff{stoppedAllocDiff}, 197 Job: mJob, 198 }, 199 EvalID: eval.ID, 200 AllocsPreempted: []*structs.AllocationDiff{preemptedAllocDiff}, 201 } 202 assert := assert.New(t) 203 planModifyIndex := uint64(1000) 204 err := state.UpsertPlanResults(planModifyIndex, &res) 205 require.NoError(err) 206 207 ws := memdb.NewWatchSet() 208 out, err := state.AllocByID(ws, alloc.ID) 209 require.NoError(err) 210 assert.Equal(alloc, out) 211 212 outJob, err := state.JobByID(ws, job.Namespace, job.ID) 213 require.NoError(err) 214 require.Equal(mJob.TaskGroups, outJob.TaskGroups) 215 require.NotEmpty(job.TaskGroups, outJob.TaskGroups) 216 217 updatedStoppedAlloc, err := state.AllocByID(ws, stoppedAlloc.ID) 218 require.NoError(err) 219 assert.Equal(stoppedAllocDiff.DesiredDescription, updatedStoppedAlloc.DesiredDescription) 220 assert.Equal(structs.AllocDesiredStatusStop, updatedStoppedAlloc.DesiredStatus) 221 assert.Equal(stoppedAllocDiff.ClientStatus, updatedStoppedAlloc.ClientStatus) 222 assert.Equal(planModifyIndex, updatedStoppedAlloc.AllocModifyIndex) 223 assert.Equal(planModifyIndex, updatedStoppedAlloc.AllocModifyIndex) 224 assert.Equal(job.TaskGroups, updatedStoppedAlloc.Job.TaskGroups) 225 226 updatedPreemptedAlloc, err := state.AllocByID(ws, preemptedAlloc.ID) 227 require.NoError(err) 228 assert.Equal(structs.AllocDesiredStatusEvict, updatedPreemptedAlloc.DesiredStatus) 229 assert.Equal(preemptedAllocDiff.PreemptedByAllocation, updatedPreemptedAlloc.PreemptedByAllocation) 230 assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) 231 assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) 232 assert.Equal(job.TaskGroups, updatedPreemptedAlloc.Job.TaskGroups) 233 234 index, err := state.Index("allocs") 235 require.NoError(err) 236 assert.EqualValues(planModifyIndex, index) 237 238 require.False(watchFired(ws)) 239 240 evalOut, err := state.EvalByID(ws, eval.ID) 241 require.NoError(err) 242 require.NotNil(evalOut) 243 assert.EqualValues(planModifyIndex, evalOut.ModifyIndex) 244 245 } 246 247 // This test checks that the deployment is created and allocations count towards 248 // the deployment 249 func TestStateStore_UpsertPlanResults_Deployment(t *testing.T) { 250 t.Parallel() 251 252 state := testStateStore(t) 253 alloc := mock.Alloc() 254 alloc2 := mock.Alloc() 255 job := alloc.Job 256 alloc.Job = nil 257 alloc2.Job = nil 258 259 d := mock.Deployment() 260 alloc.DeploymentID = d.ID 261 alloc2.DeploymentID = d.ID 262 263 if err := state.UpsertJob(999, job); err != nil { 264 t.Fatalf("err: %v", err) 265 } 266 267 eval := mock.Eval() 268 eval.JobID = job.ID 269 270 // Create an eval 271 if err := state.UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { 272 t.Fatalf("err: %v", err) 273 } 274 275 // Create a plan result 276 res := structs.ApplyPlanResultsRequest{ 277 AllocUpdateRequest: structs.AllocUpdateRequest{ 278 Alloc: []*structs.Allocation{alloc, alloc2}, 279 Job: job, 280 }, 281 Deployment: d, 282 EvalID: eval.ID, 283 } 284 285 err := state.UpsertPlanResults(1000, &res) 286 if err != nil { 287 t.Fatalf("err: %v", err) 288 } 289 290 ws := memdb.NewWatchSet() 291 assert := assert.New(t) 292 out, err := state.AllocByID(ws, alloc.ID) 293 assert.Nil(err) 294 assert.Equal(alloc, out) 295 296 dout, err := state.DeploymentByID(ws, d.ID) 297 assert.Nil(err) 298 assert.NotNil(dout) 299 300 tg, ok := dout.TaskGroups[alloc.TaskGroup] 301 assert.True(ok) 302 assert.NotNil(tg) 303 assert.Equal(2, tg.PlacedAllocs) 304 305 evalOut, err := state.EvalByID(ws, eval.ID) 306 assert.Nil(err) 307 assert.NotNil(evalOut) 308 assert.EqualValues(1000, evalOut.ModifyIndex) 309 310 if watchFired(ws) { 311 t.Fatalf("bad") 312 } 313 314 // Update the allocs to be part of a new deployment 315 d2 := d.Copy() 316 d2.ID = uuid.Generate() 317 318 allocNew := alloc.Copy() 319 allocNew.DeploymentID = d2.ID 320 allocNew2 := alloc2.Copy() 321 allocNew2.DeploymentID = d2.ID 322 323 // Create another plan 324 res = structs.ApplyPlanResultsRequest{ 325 AllocUpdateRequest: structs.AllocUpdateRequest{ 326 Alloc: []*structs.Allocation{allocNew, allocNew2}, 327 Job: job, 328 }, 329 Deployment: d2, 330 EvalID: eval.ID, 331 } 332 333 err = state.UpsertPlanResults(1001, &res) 334 if err != nil { 335 t.Fatalf("err: %v", err) 336 } 337 338 dout, err = state.DeploymentByID(ws, d2.ID) 339 assert.Nil(err) 340 assert.NotNil(dout) 341 342 tg, ok = dout.TaskGroups[alloc.TaskGroup] 343 assert.True(ok) 344 assert.NotNil(tg) 345 assert.Equal(2, tg.PlacedAllocs) 346 347 evalOut, err = state.EvalByID(ws, eval.ID) 348 assert.Nil(err) 349 assert.NotNil(evalOut) 350 assert.EqualValues(1001, evalOut.ModifyIndex) 351 } 352 353 // This test checks that: 354 // 1) Preempted allocations in plan results are updated 355 // 2) Evals are inserted for preempted jobs 356 func TestStateStore_UpsertPlanResults_PreemptedAllocs(t *testing.T) { 357 t.Parallel() 358 require := require.New(t) 359 360 state := testStateStore(t) 361 alloc := mock.Alloc() 362 job := alloc.Job 363 alloc.Job = nil 364 365 // Insert job 366 err := state.UpsertJob(999, job) 367 require.NoError(err) 368 369 // Create an eval 370 eval := mock.Eval() 371 eval.JobID = job.ID 372 err = state.UpsertEvals(1, []*structs.Evaluation{eval}) 373 require.NoError(err) 374 375 // Insert alloc that will be preempted in the plan 376 preemptedAlloc := mock.Alloc() 377 err = state.UpsertAllocs(2, []*structs.Allocation{preemptedAlloc}) 378 require.NoError(err) 379 380 minimalPreemptedAlloc := &structs.Allocation{ 381 ID: preemptedAlloc.ID, 382 PreemptedByAllocation: alloc.ID, 383 ModifyTime: time.Now().Unix(), 384 } 385 386 // Create eval for preempted job 387 eval2 := mock.Eval() 388 eval2.JobID = preemptedAlloc.JobID 389 390 // Create a plan result 391 res := structs.ApplyPlanResultsRequest{ 392 AllocUpdateRequest: structs.AllocUpdateRequest{ 393 Alloc: []*structs.Allocation{alloc}, 394 Job: job, 395 }, 396 EvalID: eval.ID, 397 NodePreemptions: []*structs.Allocation{minimalPreemptedAlloc}, 398 PreemptionEvals: []*structs.Evaluation{eval2}, 399 } 400 401 err = state.UpsertPlanResults(1000, &res) 402 require.NoError(err) 403 404 ws := memdb.NewWatchSet() 405 406 // Verify alloc and eval created by plan 407 out, err := state.AllocByID(ws, alloc.ID) 408 require.NoError(err) 409 require.Equal(alloc, out) 410 411 index, err := state.Index("allocs") 412 require.NoError(err) 413 require.EqualValues(1000, index) 414 415 evalOut, err := state.EvalByID(ws, eval.ID) 416 require.NoError(err) 417 require.NotNil(evalOut) 418 require.EqualValues(1000, evalOut.ModifyIndex) 419 420 // Verify preempted alloc 421 preempted, err := state.AllocByID(ws, preemptedAlloc.ID) 422 require.NoError(err) 423 require.Equal(preempted.DesiredStatus, structs.AllocDesiredStatusEvict) 424 require.Equal(preempted.DesiredDescription, fmt.Sprintf("Preempted by alloc ID %v", alloc.ID)) 425 require.Equal(preempted.Job.ID, preemptedAlloc.Job.ID) 426 require.Equal(preempted.Job, preemptedAlloc.Job) 427 428 // Verify eval for preempted job 429 preemptedJobEval, err := state.EvalByID(ws, eval2.ID) 430 require.NoError(err) 431 require.NotNil(preemptedJobEval) 432 require.EqualValues(1000, preemptedJobEval.ModifyIndex) 433 434 } 435 436 // This test checks that deployment updates are applied correctly 437 func TestStateStore_UpsertPlanResults_DeploymentUpdates(t *testing.T) { 438 t.Parallel() 439 state := testStateStore(t) 440 441 // Create a job that applies to all 442 job := mock.Job() 443 if err := state.UpsertJob(998, job); err != nil { 444 t.Fatalf("err: %v", err) 445 } 446 447 // Create a deployment that we will update its status 448 doutstanding := mock.Deployment() 449 doutstanding.JobID = job.ID 450 451 if err := state.UpsertDeployment(1000, doutstanding); err != nil { 452 t.Fatalf("err: %v", err) 453 } 454 455 eval := mock.Eval() 456 eval.JobID = job.ID 457 458 // Create an eval 459 if err := state.UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { 460 t.Fatalf("err: %v", err) 461 } 462 alloc := mock.Alloc() 463 alloc.Job = nil 464 465 dnew := mock.Deployment() 466 dnew.JobID = job.ID 467 alloc.DeploymentID = dnew.ID 468 469 // Update the old deployment 470 update := &structs.DeploymentStatusUpdate{ 471 DeploymentID: doutstanding.ID, 472 Status: "foo", 473 StatusDescription: "bar", 474 } 475 476 // Create a plan result 477 res := structs.ApplyPlanResultsRequest{ 478 AllocUpdateRequest: structs.AllocUpdateRequest{ 479 Alloc: []*structs.Allocation{alloc}, 480 Job: job, 481 }, 482 Deployment: dnew, 483 DeploymentUpdates: []*structs.DeploymentStatusUpdate{update}, 484 EvalID: eval.ID, 485 } 486 487 err := state.UpsertPlanResults(1000, &res) 488 if err != nil { 489 t.Fatalf("err: %v", err) 490 } 491 assert := assert.New(t) 492 ws := memdb.NewWatchSet() 493 494 // Check the deployments are correctly updated. 495 dout, err := state.DeploymentByID(ws, dnew.ID) 496 assert.Nil(err) 497 assert.NotNil(dout) 498 499 tg, ok := dout.TaskGroups[alloc.TaskGroup] 500 assert.True(ok) 501 assert.NotNil(tg) 502 assert.Equal(1, tg.PlacedAllocs) 503 504 doutstandingout, err := state.DeploymentByID(ws, doutstanding.ID) 505 assert.Nil(err) 506 assert.NotNil(doutstandingout) 507 assert.Equal(update.Status, doutstandingout.Status) 508 assert.Equal(update.StatusDescription, doutstandingout.StatusDescription) 509 assert.EqualValues(1000, doutstandingout.ModifyIndex) 510 511 evalOut, err := state.EvalByID(ws, eval.ID) 512 assert.Nil(err) 513 assert.NotNil(evalOut) 514 assert.EqualValues(1000, evalOut.ModifyIndex) 515 if watchFired(ws) { 516 t.Fatalf("bad") 517 } 518 } 519 520 func TestStateStore_UpsertDeployment(t *testing.T) { 521 t.Parallel() 522 523 state := testStateStore(t) 524 deployment := mock.Deployment() 525 526 // Create a watchset so we can test that upsert fires the watch 527 ws := memdb.NewWatchSet() 528 _, err := state.DeploymentsByJobID(ws, deployment.Namespace, deployment.ID, true) 529 if err != nil { 530 t.Fatalf("bad: %v", err) 531 } 532 533 err = state.UpsertDeployment(1000, deployment) 534 if err != nil { 535 t.Fatalf("err: %v", err) 536 } 537 if !watchFired(ws) { 538 t.Fatalf("bad") 539 } 540 541 ws = memdb.NewWatchSet() 542 out, err := state.DeploymentByID(ws, deployment.ID) 543 if err != nil { 544 t.Fatalf("err: %v", err) 545 } 546 547 if !reflect.DeepEqual(deployment, out) { 548 t.Fatalf("bad: %#v %#v", deployment, out) 549 } 550 551 index, err := state.Index("deployment") 552 if err != nil { 553 t.Fatalf("err: %v", err) 554 } 555 if index != 1000 { 556 t.Fatalf("bad: %d", index) 557 } 558 559 if watchFired(ws) { 560 t.Fatalf("bad") 561 } 562 } 563 564 // Tests that deployments of older create index and same job id are not returned 565 func TestStateStore_OldDeployment(t *testing.T) { 566 t.Parallel() 567 568 state := testStateStore(t) 569 job := mock.Job() 570 job.ID = "job1" 571 state.UpsertJob(1000, job) 572 573 deploy1 := mock.Deployment() 574 deploy1.JobID = job.ID 575 deploy1.JobCreateIndex = job.CreateIndex 576 577 deploy2 := mock.Deployment() 578 deploy2.JobID = job.ID 579 deploy2.JobCreateIndex = 11 580 581 require := require.New(t) 582 583 // Insert both deployments 584 err := state.UpsertDeployment(1001, deploy1) 585 require.Nil(err) 586 587 err = state.UpsertDeployment(1002, deploy2) 588 require.Nil(err) 589 590 ws := memdb.NewWatchSet() 591 // Should return both deployments 592 deploys, err := state.DeploymentsByJobID(ws, deploy1.Namespace, job.ID, true) 593 require.Nil(err) 594 require.Len(deploys, 2) 595 596 // Should only return deploy1 597 deploys, err = state.DeploymentsByJobID(ws, deploy1.Namespace, job.ID, false) 598 require.Nil(err) 599 require.Len(deploys, 1) 600 require.Equal(deploy1.ID, deploys[0].ID) 601 } 602 603 func TestStateStore_DeleteDeployment(t *testing.T) { 604 t.Parallel() 605 606 state := testStateStore(t) 607 d1 := mock.Deployment() 608 d2 := mock.Deployment() 609 610 err := state.UpsertDeployment(1000, d1) 611 if err != nil { 612 t.Fatalf("err: %v", err) 613 } 614 if err := state.UpsertDeployment(1001, d2); err != nil { 615 t.Fatalf("err: %v", err) 616 } 617 618 // Create a watchset so we can test that delete fires the watch 619 ws := memdb.NewWatchSet() 620 if _, err := state.DeploymentByID(ws, d1.ID); err != nil { 621 t.Fatalf("bad: %v", err) 622 } 623 624 err = state.DeleteDeployment(1002, []string{d1.ID, d2.ID}) 625 if err != nil { 626 t.Fatalf("err: %v", err) 627 } 628 629 if !watchFired(ws) { 630 t.Fatalf("bad") 631 } 632 633 ws = memdb.NewWatchSet() 634 out, err := state.DeploymentByID(ws, d1.ID) 635 if err != nil { 636 t.Fatalf("err: %v", err) 637 } 638 639 if out != nil { 640 t.Fatalf("bad: %#v %#v", d1, out) 641 } 642 643 index, err := state.Index("deployment") 644 if err != nil { 645 t.Fatalf("err: %v", err) 646 } 647 if index != 1002 { 648 t.Fatalf("bad: %d", index) 649 } 650 651 if watchFired(ws) { 652 t.Fatalf("bad") 653 } 654 } 655 656 func TestStateStore_Deployments(t *testing.T) { 657 t.Parallel() 658 659 state := testStateStore(t) 660 var deployments []*structs.Deployment 661 662 for i := 0; i < 10; i++ { 663 deployment := mock.Deployment() 664 deployments = append(deployments, deployment) 665 666 err := state.UpsertDeployment(1000+uint64(i), deployment) 667 if err != nil { 668 t.Fatalf("err: %v", err) 669 } 670 } 671 672 ws := memdb.NewWatchSet() 673 iter, err := state.Deployments(ws) 674 if err != nil { 675 t.Fatalf("err: %v", err) 676 } 677 678 var out []*structs.Deployment 679 for { 680 raw := iter.Next() 681 if raw == nil { 682 break 683 } 684 out = append(out, raw.(*structs.Deployment)) 685 } 686 687 lessThan := func(i, j int) bool { 688 return deployments[i].ID < deployments[j].ID 689 } 690 sort.Slice(deployments, lessThan) 691 sort.Slice(out, lessThan) 692 693 if !reflect.DeepEqual(deployments, out) { 694 t.Fatalf("bad: %#v %#v", deployments, out) 695 } 696 697 if watchFired(ws) { 698 t.Fatalf("bad") 699 } 700 } 701 702 func TestStateStore_DeploymentsByIDPrefix(t *testing.T) { 703 t.Parallel() 704 705 state := testStateStore(t) 706 deploy := mock.Deployment() 707 708 deploy.ID = "11111111-662e-d0ab-d1c9-3e434af7bdb4" 709 err := state.UpsertDeployment(1000, deploy) 710 if err != nil { 711 t.Fatalf("err: %v", err) 712 } 713 714 // Create a watchset so we can test that getters don't cause it to fire 715 ws := memdb.NewWatchSet() 716 iter, err := state.DeploymentsByIDPrefix(ws, deploy.Namespace, deploy.ID) 717 if err != nil { 718 t.Fatalf("err: %v", err) 719 } 720 721 gatherDeploys := func(iter memdb.ResultIterator) []*structs.Deployment { 722 var deploys []*structs.Deployment 723 for { 724 raw := iter.Next() 725 if raw == nil { 726 break 727 } 728 deploy := raw.(*structs.Deployment) 729 deploys = append(deploys, deploy) 730 } 731 return deploys 732 } 733 734 deploys := gatherDeploys(iter) 735 if len(deploys) != 1 { 736 t.Fatalf("err: %v", err) 737 } 738 739 if watchFired(ws) { 740 t.Fatalf("bad") 741 } 742 743 iter, err = state.DeploymentsByIDPrefix(ws, deploy.Namespace, "11") 744 if err != nil { 745 t.Fatalf("err: %v", err) 746 } 747 748 deploys = gatherDeploys(iter) 749 if len(deploys) != 1 { 750 t.Fatalf("err: %v", err) 751 } 752 753 deploy = mock.Deployment() 754 deploy.ID = "11222222-662e-d0ab-d1c9-3e434af7bdb4" 755 err = state.UpsertDeployment(1001, deploy) 756 if err != nil { 757 t.Fatalf("err: %v", err) 758 } 759 760 if !watchFired(ws) { 761 t.Fatalf("bad") 762 } 763 764 ws = memdb.NewWatchSet() 765 iter, err = state.DeploymentsByIDPrefix(ws, deploy.Namespace, "11") 766 if err != nil { 767 t.Fatalf("err: %v", err) 768 } 769 770 deploys = gatherDeploys(iter) 771 if len(deploys) != 2 { 772 t.Fatalf("err: %v", err) 773 } 774 775 iter, err = state.DeploymentsByIDPrefix(ws, deploy.Namespace, "1111") 776 if err != nil { 777 t.Fatalf("err: %v", err) 778 } 779 780 deploys = gatherDeploys(iter) 781 if len(deploys) != 1 { 782 t.Fatalf("err: %v", err) 783 } 784 785 if watchFired(ws) { 786 t.Fatalf("bad") 787 } 788 } 789 790 func TestStateStore_UpsertNode_Node(t *testing.T) { 791 t.Parallel() 792 793 require := require.New(t) 794 state := testStateStore(t) 795 node := mock.Node() 796 797 // Create a watchset so we can test that upsert fires the watch 798 ws := memdb.NewWatchSet() 799 _, err := state.NodeByID(ws, node.ID) 800 require.NoError(err) 801 802 require.NoError(state.UpsertNode(1000, node)) 803 require.True(watchFired(ws)) 804 805 ws = memdb.NewWatchSet() 806 out, err := state.NodeByID(ws, node.ID) 807 require.NoError(err) 808 809 out2, err := state.NodeBySecretID(ws, node.SecretID) 810 require.NoError(err) 811 require.EqualValues(node, out) 812 require.EqualValues(node, out2) 813 require.Len(out.Events, 1) 814 require.Equal(NodeRegisterEventRegistered, out.Events[0].Message) 815 816 index, err := state.Index("nodes") 817 require.NoError(err) 818 require.EqualValues(1000, index) 819 require.False(watchFired(ws)) 820 821 // Transition the node to down and then up and ensure we get a re-register 822 // event 823 down := out.Copy() 824 down.Status = structs.NodeStatusDown 825 require.NoError(state.UpsertNode(1001, down)) 826 require.NoError(state.UpsertNode(1002, out)) 827 828 out, err = state.NodeByID(ws, node.ID) 829 require.NoError(err) 830 require.Len(out.Events, 2) 831 require.Equal(NodeRegisterEventReregistered, out.Events[1].Message) 832 } 833 834 func TestStateStore_DeleteNode_Node(t *testing.T) { 835 t.Parallel() 836 837 state := testStateStore(t) 838 839 // Create and insert two nodes, which we'll delete 840 node0 := mock.Node() 841 node1 := mock.Node() 842 err := state.UpsertNode(1000, node0) 843 require.NoError(t, err) 844 err = state.UpsertNode(1001, node1) 845 require.NoError(t, err) 846 847 // Create a watchset so we can test that delete fires the watch 848 ws := memdb.NewWatchSet() 849 850 // Check that both nodes are not nil 851 out, err := state.NodeByID(ws, node0.ID) 852 require.NoError(t, err) 853 require.NotNil(t, out) 854 out, err = state.NodeByID(ws, node1.ID) 855 require.NoError(t, err) 856 require.NotNil(t, out) 857 858 // Delete both nodes in a batch, fires the watch 859 err = state.DeleteNode(1002, []string{node0.ID, node1.ID}) 860 require.NoError(t, err) 861 require.True(t, watchFired(ws)) 862 863 // Check that both nodes are nil 864 ws = memdb.NewWatchSet() 865 out, err = state.NodeByID(ws, node0.ID) 866 require.NoError(t, err) 867 require.Nil(t, out) 868 out, err = state.NodeByID(ws, node1.ID) 869 require.NoError(t, err) 870 require.Nil(t, out) 871 872 // Ensure that the index is still at 1002, from DeleteNode 873 index, err := state.Index("nodes") 874 require.NoError(t, err) 875 require.Equal(t, uint64(1002), index) 876 require.False(t, watchFired(ws)) 877 } 878 879 func TestStateStore_UpdateNodeStatus_Node(t *testing.T) { 880 t.Parallel() 881 require := require.New(t) 882 883 state := testStateStore(t) 884 node := mock.Node() 885 886 require.NoError(state.UpsertNode(800, node)) 887 888 // Create a watchset so we can test that update node status fires the watch 889 ws := memdb.NewWatchSet() 890 _, err := state.NodeByID(ws, node.ID) 891 require.NoError(err) 892 893 event := &structs.NodeEvent{ 894 Message: "Node ready foo", 895 Subsystem: structs.NodeEventSubsystemCluster, 896 Timestamp: time.Now(), 897 } 898 899 require.NoError(state.UpdateNodeStatus(801, node.ID, structs.NodeStatusReady, 70, event)) 900 require.True(watchFired(ws)) 901 902 ws = memdb.NewWatchSet() 903 out, err := state.NodeByID(ws, node.ID) 904 require.NoError(err) 905 require.Equal(structs.NodeStatusReady, out.Status) 906 require.EqualValues(801, out.ModifyIndex) 907 require.EqualValues(70, out.StatusUpdatedAt) 908 require.Len(out.Events, 2) 909 require.Equal(event.Message, out.Events[1].Message) 910 911 index, err := state.Index("nodes") 912 require.NoError(err) 913 require.EqualValues(801, index) 914 require.False(watchFired(ws)) 915 } 916 917 func TestStateStore_BatchUpdateNodeDrain(t *testing.T) { 918 t.Parallel() 919 require := require.New(t) 920 921 state := testStateStore(t) 922 923 n1, n2 := mock.Node(), mock.Node() 924 require.Nil(state.UpsertNode(1000, n1)) 925 require.Nil(state.UpsertNode(1001, n2)) 926 927 // Create a watchset so we can test that update node drain fires the watch 928 ws := memdb.NewWatchSet() 929 _, err := state.NodeByID(ws, n1.ID) 930 require.Nil(err) 931 932 expectedDrain := &structs.DrainStrategy{ 933 DrainSpec: structs.DrainSpec{ 934 Deadline: -1 * time.Second, 935 }, 936 } 937 938 update := map[string]*structs.DrainUpdate{ 939 n1.ID: { 940 DrainStrategy: expectedDrain, 941 }, 942 n2.ID: { 943 DrainStrategy: expectedDrain, 944 }, 945 } 946 947 event := &structs.NodeEvent{ 948 Message: "Drain strategy enabled", 949 Subsystem: structs.NodeEventSubsystemDrain, 950 Timestamp: time.Now(), 951 } 952 events := map[string]*structs.NodeEvent{ 953 n1.ID: event, 954 n2.ID: event, 955 } 956 957 require.Nil(state.BatchUpdateNodeDrain(1002, 7, update, events)) 958 require.True(watchFired(ws)) 959 960 ws = memdb.NewWatchSet() 961 for _, id := range []string{n1.ID, n2.ID} { 962 out, err := state.NodeByID(ws, id) 963 require.Nil(err) 964 require.True(out.Drain) 965 require.NotNil(out.DrainStrategy) 966 require.Equal(out.DrainStrategy, expectedDrain) 967 require.Len(out.Events, 2) 968 require.EqualValues(1002, out.ModifyIndex) 969 require.EqualValues(7, out.StatusUpdatedAt) 970 } 971 972 index, err := state.Index("nodes") 973 require.Nil(err) 974 require.EqualValues(1002, index) 975 require.False(watchFired(ws)) 976 } 977 978 func TestStateStore_UpdateNodeDrain_Node(t *testing.T) { 979 t.Parallel() 980 require := require.New(t) 981 982 state := testStateStore(t) 983 node := mock.Node() 984 985 require.Nil(state.UpsertNode(1000, node)) 986 987 // Create a watchset so we can test that update node drain fires the watch 988 ws := memdb.NewWatchSet() 989 _, err := state.NodeByID(ws, node.ID) 990 require.Nil(err) 991 992 expectedDrain := &structs.DrainStrategy{ 993 DrainSpec: structs.DrainSpec{ 994 Deadline: -1 * time.Second, 995 }, 996 } 997 998 event := &structs.NodeEvent{ 999 Message: "Drain strategy enabled", 1000 Subsystem: structs.NodeEventSubsystemDrain, 1001 Timestamp: time.Now(), 1002 } 1003 require.Nil(state.UpdateNodeDrain(1001, node.ID, expectedDrain, false, 7, event)) 1004 require.True(watchFired(ws)) 1005 1006 ws = memdb.NewWatchSet() 1007 out, err := state.NodeByID(ws, node.ID) 1008 require.Nil(err) 1009 require.True(out.Drain) 1010 require.NotNil(out.DrainStrategy) 1011 require.Equal(out.DrainStrategy, expectedDrain) 1012 require.Len(out.Events, 2) 1013 require.EqualValues(1001, out.ModifyIndex) 1014 require.EqualValues(7, out.StatusUpdatedAt) 1015 1016 index, err := state.Index("nodes") 1017 require.Nil(err) 1018 require.EqualValues(1001, index) 1019 require.False(watchFired(ws)) 1020 } 1021 1022 func TestStateStore_AddSingleNodeEvent(t *testing.T) { 1023 t.Parallel() 1024 require := require.New(t) 1025 1026 state := testStateStore(t) 1027 1028 node := mock.Node() 1029 1030 // We create a new node event every time we register a node 1031 err := state.UpsertNode(1000, node) 1032 require.Nil(err) 1033 1034 require.Equal(1, len(node.Events)) 1035 require.Equal(structs.NodeEventSubsystemCluster, node.Events[0].Subsystem) 1036 require.Equal(NodeRegisterEventRegistered, node.Events[0].Message) 1037 1038 // Create a watchset so we can test that AddNodeEvent fires the watch 1039 ws := memdb.NewWatchSet() 1040 _, err = state.NodeByID(ws, node.ID) 1041 require.Nil(err) 1042 1043 nodeEvent := &structs.NodeEvent{ 1044 Message: "failed", 1045 Subsystem: "Driver", 1046 Timestamp: time.Now(), 1047 } 1048 nodeEvents := map[string][]*structs.NodeEvent{ 1049 node.ID: {nodeEvent}, 1050 } 1051 err = state.UpsertNodeEvents(uint64(1001), nodeEvents) 1052 require.Nil(err) 1053 1054 require.True(watchFired(ws)) 1055 1056 ws = memdb.NewWatchSet() 1057 out, err := state.NodeByID(ws, node.ID) 1058 require.Nil(err) 1059 1060 require.Equal(2, len(out.Events)) 1061 require.Equal(nodeEvent, out.Events[1]) 1062 } 1063 1064 // To prevent stale node events from accumulating, we limit the number of 1065 // stored node events to 10. 1066 func TestStateStore_NodeEvents_RetentionWindow(t *testing.T) { 1067 t.Parallel() 1068 require := require.New(t) 1069 1070 state := testStateStore(t) 1071 1072 node := mock.Node() 1073 1074 err := state.UpsertNode(1000, node) 1075 if err != nil { 1076 t.Fatalf("err: %v", err) 1077 } 1078 require.Equal(1, len(node.Events)) 1079 require.Equal(structs.NodeEventSubsystemCluster, node.Events[0].Subsystem) 1080 require.Equal(NodeRegisterEventRegistered, node.Events[0].Message) 1081 1082 var out *structs.Node 1083 for i := 1; i <= 20; i++ { 1084 ws := memdb.NewWatchSet() 1085 out, err = state.NodeByID(ws, node.ID) 1086 require.Nil(err) 1087 1088 nodeEvent := &structs.NodeEvent{ 1089 Message: fmt.Sprintf("%dith failed", i), 1090 Subsystem: "Driver", 1091 Timestamp: time.Now(), 1092 } 1093 1094 nodeEvents := map[string][]*structs.NodeEvent{ 1095 out.ID: {nodeEvent}, 1096 } 1097 err := state.UpsertNodeEvents(uint64(i), nodeEvents) 1098 require.Nil(err) 1099 1100 require.True(watchFired(ws)) 1101 ws = memdb.NewWatchSet() 1102 out, err = state.NodeByID(ws, node.ID) 1103 require.Nil(err) 1104 } 1105 1106 ws := memdb.NewWatchSet() 1107 out, err = state.NodeByID(ws, node.ID) 1108 require.Nil(err) 1109 1110 require.Equal(10, len(out.Events)) 1111 require.Equal(uint64(11), out.Events[0].CreateIndex) 1112 require.Equal(uint64(20), out.Events[len(out.Events)-1].CreateIndex) 1113 } 1114 1115 func TestStateStore_UpdateNodeDrain_ResetEligiblity(t *testing.T) { 1116 t.Parallel() 1117 require := require.New(t) 1118 1119 state := testStateStore(t) 1120 node := mock.Node() 1121 require.Nil(state.UpsertNode(1000, node)) 1122 1123 // Create a watchset so we can test that update node drain fires the watch 1124 ws := memdb.NewWatchSet() 1125 _, err := state.NodeByID(ws, node.ID) 1126 require.Nil(err) 1127 1128 drain := &structs.DrainStrategy{ 1129 DrainSpec: structs.DrainSpec{ 1130 Deadline: -1 * time.Second, 1131 }, 1132 } 1133 1134 event1 := &structs.NodeEvent{ 1135 Message: "Drain strategy enabled", 1136 Subsystem: structs.NodeEventSubsystemDrain, 1137 Timestamp: time.Now(), 1138 } 1139 require.Nil(state.UpdateNodeDrain(1001, node.ID, drain, false, 7, event1)) 1140 require.True(watchFired(ws)) 1141 1142 // Remove the drain 1143 event2 := &structs.NodeEvent{ 1144 Message: "Drain strategy disabled", 1145 Subsystem: structs.NodeEventSubsystemDrain, 1146 Timestamp: time.Now(), 1147 } 1148 require.Nil(state.UpdateNodeDrain(1002, node.ID, nil, true, 9, event2)) 1149 1150 ws = memdb.NewWatchSet() 1151 out, err := state.NodeByID(ws, node.ID) 1152 require.Nil(err) 1153 require.False(out.Drain) 1154 require.Nil(out.DrainStrategy) 1155 require.Equal(out.SchedulingEligibility, structs.NodeSchedulingEligible) 1156 require.Len(out.Events, 3) 1157 require.EqualValues(1002, out.ModifyIndex) 1158 require.EqualValues(9, out.StatusUpdatedAt) 1159 1160 index, err := state.Index("nodes") 1161 require.Nil(err) 1162 require.EqualValues(1002, index) 1163 require.False(watchFired(ws)) 1164 } 1165 1166 func TestStateStore_UpdateNodeEligibility(t *testing.T) { 1167 t.Parallel() 1168 require := require.New(t) 1169 1170 state := testStateStore(t) 1171 node := mock.Node() 1172 1173 err := state.UpsertNode(1000, node) 1174 if err != nil { 1175 t.Fatalf("err: %v", err) 1176 } 1177 1178 expectedEligibility := structs.NodeSchedulingIneligible 1179 1180 // Create a watchset so we can test that update node drain fires the watch 1181 ws := memdb.NewWatchSet() 1182 if _, err := state.NodeByID(ws, node.ID); err != nil { 1183 t.Fatalf("bad: %v", err) 1184 } 1185 1186 event := &structs.NodeEvent{ 1187 Message: "Node marked as ineligible", 1188 Subsystem: structs.NodeEventSubsystemCluster, 1189 Timestamp: time.Now(), 1190 } 1191 require.Nil(state.UpdateNodeEligibility(1001, node.ID, expectedEligibility, 7, event)) 1192 require.True(watchFired(ws)) 1193 1194 ws = memdb.NewWatchSet() 1195 out, err := state.NodeByID(ws, node.ID) 1196 require.Nil(err) 1197 require.Equal(out.SchedulingEligibility, expectedEligibility) 1198 require.Len(out.Events, 2) 1199 require.Equal(out.Events[1], event) 1200 require.EqualValues(1001, out.ModifyIndex) 1201 require.EqualValues(7, out.StatusUpdatedAt) 1202 1203 index, err := state.Index("nodes") 1204 require.Nil(err) 1205 require.EqualValues(1001, index) 1206 require.False(watchFired(ws)) 1207 1208 // Set a drain strategy 1209 expectedDrain := &structs.DrainStrategy{ 1210 DrainSpec: structs.DrainSpec{ 1211 Deadline: -1 * time.Second, 1212 }, 1213 } 1214 require.Nil(state.UpdateNodeDrain(1002, node.ID, expectedDrain, false, 7, nil)) 1215 1216 // Try to set the node to eligible 1217 err = state.UpdateNodeEligibility(1003, node.ID, structs.NodeSchedulingEligible, 9, nil) 1218 require.NotNil(err) 1219 require.Contains(err.Error(), "while it is draining") 1220 } 1221 1222 func TestStateStore_Nodes(t *testing.T) { 1223 t.Parallel() 1224 1225 state := testStateStore(t) 1226 var nodes []*structs.Node 1227 1228 for i := 0; i < 10; i++ { 1229 node := mock.Node() 1230 nodes = append(nodes, node) 1231 1232 err := state.UpsertNode(1000+uint64(i), node) 1233 if err != nil { 1234 t.Fatalf("err: %v", err) 1235 } 1236 } 1237 1238 // Create a watchset so we can test that getters don't cause it to fire 1239 ws := memdb.NewWatchSet() 1240 iter, err := state.Nodes(ws) 1241 if err != nil { 1242 t.Fatalf("bad: %v", err) 1243 } 1244 1245 var out []*structs.Node 1246 for { 1247 raw := iter.Next() 1248 if raw == nil { 1249 break 1250 } 1251 out = append(out, raw.(*structs.Node)) 1252 } 1253 1254 sort.Sort(NodeIDSort(nodes)) 1255 sort.Sort(NodeIDSort(out)) 1256 1257 if !reflect.DeepEqual(nodes, out) { 1258 t.Fatalf("bad: %#v %#v", nodes, out) 1259 } 1260 1261 if watchFired(ws) { 1262 t.Fatalf("bad") 1263 } 1264 } 1265 1266 func TestStateStore_NodesByIDPrefix(t *testing.T) { 1267 t.Parallel() 1268 1269 state := testStateStore(t) 1270 node := mock.Node() 1271 1272 node.ID = "11111111-662e-d0ab-d1c9-3e434af7bdb4" 1273 err := state.UpsertNode(1000, node) 1274 if err != nil { 1275 t.Fatalf("err: %v", err) 1276 } 1277 1278 // Create a watchset so we can test that getters don't cause it to fire 1279 ws := memdb.NewWatchSet() 1280 iter, err := state.NodesByIDPrefix(ws, node.ID) 1281 if err != nil { 1282 t.Fatalf("err: %v", err) 1283 } 1284 1285 gatherNodes := func(iter memdb.ResultIterator) []*structs.Node { 1286 var nodes []*structs.Node 1287 for { 1288 raw := iter.Next() 1289 if raw == nil { 1290 break 1291 } 1292 node := raw.(*structs.Node) 1293 nodes = append(nodes, node) 1294 } 1295 return nodes 1296 } 1297 1298 nodes := gatherNodes(iter) 1299 if len(nodes) != 1 { 1300 t.Fatalf("err: %v", err) 1301 } 1302 1303 if watchFired(ws) { 1304 t.Fatalf("bad") 1305 } 1306 1307 iter, err = state.NodesByIDPrefix(ws, "11") 1308 if err != nil { 1309 t.Fatalf("err: %v", err) 1310 } 1311 1312 nodes = gatherNodes(iter) 1313 if len(nodes) != 1 { 1314 t.Fatalf("err: %v", err) 1315 } 1316 1317 node = mock.Node() 1318 node.ID = "11222222-662e-d0ab-d1c9-3e434af7bdb4" 1319 err = state.UpsertNode(1001, node) 1320 if err != nil { 1321 t.Fatalf("err: %v", err) 1322 } 1323 1324 if !watchFired(ws) { 1325 t.Fatalf("bad") 1326 } 1327 1328 ws = memdb.NewWatchSet() 1329 iter, err = state.NodesByIDPrefix(ws, "11") 1330 if err != nil { 1331 t.Fatalf("err: %v", err) 1332 } 1333 1334 nodes = gatherNodes(iter) 1335 if len(nodes) != 2 { 1336 t.Fatalf("err: %v", err) 1337 } 1338 1339 iter, err = state.NodesByIDPrefix(ws, "1111") 1340 if err != nil { 1341 t.Fatalf("err: %v", err) 1342 } 1343 1344 nodes = gatherNodes(iter) 1345 if len(nodes) != 1 { 1346 t.Fatalf("err: %v", err) 1347 } 1348 1349 if watchFired(ws) { 1350 t.Fatalf("bad") 1351 } 1352 } 1353 1354 func TestStateStore_RestoreNode(t *testing.T) { 1355 t.Parallel() 1356 1357 state := testStateStore(t) 1358 node := mock.Node() 1359 1360 restore, err := state.Restore() 1361 if err != nil { 1362 t.Fatalf("err: %v", err) 1363 } 1364 1365 err = restore.NodeRestore(node) 1366 if err != nil { 1367 t.Fatalf("err: %v", err) 1368 } 1369 restore.Commit() 1370 1371 ws := memdb.NewWatchSet() 1372 out, err := state.NodeByID(ws, node.ID) 1373 if err != nil { 1374 t.Fatalf("err: %v", err) 1375 } 1376 1377 if !reflect.DeepEqual(out, node) { 1378 t.Fatalf("Bad: %#v %#v", out, node) 1379 } 1380 } 1381 1382 func TestStateStore_UpsertJob_Job(t *testing.T) { 1383 t.Parallel() 1384 1385 state := testStateStore(t) 1386 job := mock.Job() 1387 1388 // Create a watchset so we can test that upsert fires the watch 1389 ws := memdb.NewWatchSet() 1390 _, err := state.JobByID(ws, job.Namespace, job.ID) 1391 if err != nil { 1392 t.Fatalf("bad: %v", err) 1393 } 1394 1395 if err := state.UpsertJob(1000, job); err != nil { 1396 t.Fatalf("err: %v", err) 1397 } 1398 if !watchFired(ws) { 1399 t.Fatalf("bad") 1400 } 1401 1402 ws = memdb.NewWatchSet() 1403 out, err := state.JobByID(ws, job.Namespace, job.ID) 1404 if err != nil { 1405 t.Fatalf("err: %v", err) 1406 } 1407 1408 if !reflect.DeepEqual(job, out) { 1409 t.Fatalf("bad: %#v %#v", job, out) 1410 } 1411 1412 index, err := state.Index("jobs") 1413 if err != nil { 1414 t.Fatalf("err: %v", err) 1415 } 1416 if index != 1000 { 1417 t.Fatalf("bad: %d", index) 1418 } 1419 1420 summary, err := state.JobSummaryByID(ws, job.Namespace, job.ID) 1421 if err != nil { 1422 t.Fatalf("err: %v", err) 1423 } 1424 if summary == nil { 1425 t.Fatalf("nil summary") 1426 } 1427 if summary.JobID != job.ID { 1428 t.Fatalf("bad summary id: %v", summary.JobID) 1429 } 1430 _, ok := summary.Summary["web"] 1431 if !ok { 1432 t.Fatalf("nil summary for task group") 1433 } 1434 if watchFired(ws) { 1435 t.Fatalf("bad") 1436 } 1437 1438 // Check the job versions 1439 allVersions, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1440 if err != nil { 1441 t.Fatalf("err: %v", err) 1442 } 1443 if len(allVersions) != 1 { 1444 t.Fatalf("got %d; want 1", len(allVersions)) 1445 } 1446 1447 if a := allVersions[0]; a.ID != job.ID || a.Version != 0 { 1448 t.Fatalf("bad: %v", a) 1449 } 1450 1451 // Test the looking up the job by version returns the same results 1452 vout, err := state.JobByIDAndVersion(ws, job.Namespace, job.ID, 0) 1453 if err != nil { 1454 t.Fatalf("err: %v", err) 1455 } 1456 1457 if !reflect.DeepEqual(out, vout) { 1458 t.Fatalf("bad: %#v %#v", out, vout) 1459 } 1460 } 1461 1462 func TestStateStore_UpdateUpsertJob_Job(t *testing.T) { 1463 t.Parallel() 1464 1465 state := testStateStore(t) 1466 job := mock.Job() 1467 1468 // Create a watchset so we can test that upsert fires the watch 1469 ws := memdb.NewWatchSet() 1470 _, err := state.JobByID(ws, job.Namespace, job.ID) 1471 if err != nil { 1472 t.Fatalf("bad: %v", err) 1473 } 1474 1475 if err := state.UpsertJob(1000, job); err != nil { 1476 t.Fatalf("err: %v", err) 1477 } 1478 1479 job2 := mock.Job() 1480 job2.ID = job.ID 1481 job2.AllAtOnce = true 1482 err = state.UpsertJob(1001, job2) 1483 if err != nil { 1484 t.Fatalf("err: %v", err) 1485 } 1486 1487 if !watchFired(ws) { 1488 t.Fatalf("bad") 1489 } 1490 1491 ws = memdb.NewWatchSet() 1492 out, err := state.JobByID(ws, job.Namespace, job.ID) 1493 if err != nil { 1494 t.Fatalf("err: %v", err) 1495 } 1496 1497 if !reflect.DeepEqual(job2, out) { 1498 t.Fatalf("bad: %#v %#v", job2, out) 1499 } 1500 1501 if out.CreateIndex != 1000 { 1502 t.Fatalf("bad: %#v", out) 1503 } 1504 if out.ModifyIndex != 1001 { 1505 t.Fatalf("bad: %#v", out) 1506 } 1507 if out.Version != 1 { 1508 t.Fatalf("bad: %#v", out) 1509 } 1510 1511 index, err := state.Index("jobs") 1512 if err != nil { 1513 t.Fatalf("err: %v", err) 1514 } 1515 if index != 1001 { 1516 t.Fatalf("bad: %d", index) 1517 } 1518 1519 // Test the looking up the job by version returns the same results 1520 vout, err := state.JobByIDAndVersion(ws, job.Namespace, job.ID, 1) 1521 if err != nil { 1522 t.Fatalf("err: %v", err) 1523 } 1524 1525 if !reflect.DeepEqual(out, vout) { 1526 t.Fatalf("bad: %#v %#v", out, vout) 1527 } 1528 1529 // Test that the job summary remains the same if the job is updated but 1530 // count remains same 1531 summary, err := state.JobSummaryByID(ws, job.Namespace, job.ID) 1532 if err != nil { 1533 t.Fatalf("err: %v", err) 1534 } 1535 if summary == nil { 1536 t.Fatalf("nil summary") 1537 } 1538 if summary.JobID != job.ID { 1539 t.Fatalf("bad summary id: %v", summary.JobID) 1540 } 1541 _, ok := summary.Summary["web"] 1542 if !ok { 1543 t.Fatalf("nil summary for task group") 1544 } 1545 1546 // Check the job versions 1547 allVersions, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1548 if err != nil { 1549 t.Fatalf("err: %v", err) 1550 } 1551 if len(allVersions) != 2 { 1552 t.Fatalf("got %d; want 1", len(allVersions)) 1553 } 1554 1555 if a := allVersions[0]; a.ID != job.ID || a.Version != 1 || !a.AllAtOnce { 1556 t.Fatalf("bad: %+v", a) 1557 } 1558 if a := allVersions[1]; a.ID != job.ID || a.Version != 0 || a.AllAtOnce { 1559 t.Fatalf("bad: %+v", a) 1560 } 1561 1562 if watchFired(ws) { 1563 t.Fatalf("bad") 1564 } 1565 } 1566 1567 func TestStateStore_UpdateUpsertJob_PeriodicJob(t *testing.T) { 1568 t.Parallel() 1569 1570 state := testStateStore(t) 1571 job := mock.PeriodicJob() 1572 1573 // Create a watchset so we can test that upsert fires the watch 1574 ws := memdb.NewWatchSet() 1575 _, err := state.JobByID(ws, job.Namespace, job.ID) 1576 if err != nil { 1577 t.Fatalf("bad: %v", err) 1578 } 1579 1580 if err := state.UpsertJob(1000, job); err != nil { 1581 t.Fatalf("err: %v", err) 1582 } 1583 1584 // Create a child and an evaluation 1585 job2 := job.Copy() 1586 job2.Periodic = nil 1587 job2.ID = fmt.Sprintf("%v/%s-1490635020", job.ID, structs.PeriodicLaunchSuffix) 1588 err = state.UpsertJob(1001, job2) 1589 if err != nil { 1590 t.Fatalf("err: %v", err) 1591 } 1592 1593 eval := mock.Eval() 1594 eval.JobID = job2.ID 1595 err = state.UpsertEvals(1002, []*structs.Evaluation{eval}) 1596 if err != nil { 1597 t.Fatalf("err: %v", err) 1598 } 1599 1600 job3 := job.Copy() 1601 job3.TaskGroups[0].Tasks[0].Name = "new name" 1602 err = state.UpsertJob(1003, job3) 1603 if err != nil { 1604 t.Fatalf("err: %v", err) 1605 } 1606 1607 if !watchFired(ws) { 1608 t.Fatalf("bad") 1609 } 1610 1611 ws = memdb.NewWatchSet() 1612 out, err := state.JobByID(ws, job.Namespace, job.ID) 1613 if err != nil { 1614 t.Fatalf("err: %v", err) 1615 } 1616 1617 if s, e := out.Status, structs.JobStatusRunning; s != e { 1618 t.Fatalf("got status %v; want %v", s, e) 1619 } 1620 1621 } 1622 1623 func TestStateStore_UpsertJob_BadNamespace(t *testing.T) { 1624 t.Parallel() 1625 1626 assert := assert.New(t) 1627 state := testStateStore(t) 1628 job := mock.Job() 1629 job.Namespace = "foo" 1630 1631 err := state.UpsertJob(1000, job) 1632 assert.Contains(err.Error(), "nonexistent namespace") 1633 1634 ws := memdb.NewWatchSet() 1635 out, err := state.JobByID(ws, job.Namespace, job.ID) 1636 assert.Nil(err) 1637 assert.Nil(out) 1638 } 1639 1640 // Upsert a job that is the child of a parent job and ensures its summary gets 1641 // updated. 1642 func TestStateStore_UpsertJob_ChildJob(t *testing.T) { 1643 t.Parallel() 1644 1645 state := testStateStore(t) 1646 1647 // Create a watchset so we can test that upsert fires the watch 1648 parent := mock.Job() 1649 ws := memdb.NewWatchSet() 1650 _, err := state.JobByID(ws, parent.Namespace, parent.ID) 1651 if err != nil { 1652 t.Fatalf("bad: %v", err) 1653 } 1654 1655 if err := state.UpsertJob(1000, parent); err != nil { 1656 t.Fatalf("err: %v", err) 1657 } 1658 1659 child := mock.Job() 1660 child.ParentID = parent.ID 1661 if err := state.UpsertJob(1001, child); err != nil { 1662 t.Fatalf("err: %v", err) 1663 } 1664 1665 summary, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID) 1666 if err != nil { 1667 t.Fatalf("err: %v", err) 1668 } 1669 if summary == nil { 1670 t.Fatalf("nil summary") 1671 } 1672 if summary.JobID != parent.ID { 1673 t.Fatalf("bad summary id: %v", parent.ID) 1674 } 1675 if summary.Children == nil { 1676 t.Fatalf("nil children summary") 1677 } 1678 if summary.Children.Pending != 1 || summary.Children.Running != 0 || summary.Children.Dead != 0 { 1679 t.Fatalf("bad children summary: %v", summary.Children) 1680 } 1681 if !watchFired(ws) { 1682 t.Fatalf("bad") 1683 } 1684 } 1685 1686 func TestStateStore_UpdateUpsertJob_JobVersion(t *testing.T) { 1687 t.Parallel() 1688 1689 state := testStateStore(t) 1690 1691 // Create a job and mark it as stable 1692 job := mock.Job() 1693 job.Stable = true 1694 job.Name = "0" 1695 1696 // Create a watchset so we can test that upsert fires the watch 1697 ws := memdb.NewWatchSet() 1698 _, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1699 if err != nil { 1700 t.Fatalf("bad: %v", err) 1701 } 1702 1703 if err := state.UpsertJob(1000, job); err != nil { 1704 t.Fatalf("err: %v", err) 1705 } 1706 1707 if !watchFired(ws) { 1708 t.Fatalf("bad") 1709 } 1710 1711 var finalJob *structs.Job 1712 for i := 1; i < 300; i++ { 1713 finalJob = mock.Job() 1714 finalJob.ID = job.ID 1715 finalJob.Name = fmt.Sprintf("%d", i) 1716 err = state.UpsertJob(uint64(1000+i), finalJob) 1717 if err != nil { 1718 t.Fatalf("err: %v", err) 1719 } 1720 } 1721 1722 ws = memdb.NewWatchSet() 1723 out, err := state.JobByID(ws, job.Namespace, job.ID) 1724 if err != nil { 1725 t.Fatalf("err: %v", err) 1726 } 1727 1728 if !reflect.DeepEqual(finalJob, out) { 1729 t.Fatalf("bad: %#v %#v", finalJob, out) 1730 } 1731 1732 if out.CreateIndex != 1000 { 1733 t.Fatalf("bad: %#v", out) 1734 } 1735 if out.ModifyIndex != 1299 { 1736 t.Fatalf("bad: %#v", out) 1737 } 1738 if out.Version != 299 { 1739 t.Fatalf("bad: %#v", out) 1740 } 1741 1742 index, err := state.Index("job_version") 1743 if err != nil { 1744 t.Fatalf("err: %v", err) 1745 } 1746 if index != 1299 { 1747 t.Fatalf("bad: %d", index) 1748 } 1749 1750 // Check the job versions 1751 allVersions, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1752 if err != nil { 1753 t.Fatalf("err: %v", err) 1754 } 1755 if len(allVersions) != structs.JobTrackedVersions { 1756 t.Fatalf("got %d; want %d", len(allVersions), structs.JobTrackedVersions) 1757 } 1758 1759 if a := allVersions[0]; a.ID != job.ID || a.Version != 299 || a.Name != "299" { 1760 t.Fatalf("bad: %+v", a) 1761 } 1762 if a := allVersions[1]; a.ID != job.ID || a.Version != 298 || a.Name != "298" { 1763 t.Fatalf("bad: %+v", a) 1764 } 1765 1766 // Ensure we didn't delete the stable job 1767 if a := allVersions[structs.JobTrackedVersions-1]; a.ID != job.ID || 1768 a.Version != 0 || a.Name != "0" || !a.Stable { 1769 t.Fatalf("bad: %+v", a) 1770 } 1771 1772 if watchFired(ws) { 1773 t.Fatalf("bad") 1774 } 1775 } 1776 1777 func TestStateStore_DeleteJob_Job(t *testing.T) { 1778 t.Parallel() 1779 1780 state := testStateStore(t) 1781 job := mock.Job() 1782 1783 err := state.UpsertJob(1000, job) 1784 if err != nil { 1785 t.Fatalf("err: %v", err) 1786 } 1787 1788 // Create a watchset so we can test that delete fires the watch 1789 ws := memdb.NewWatchSet() 1790 if _, err := state.JobByID(ws, job.Namespace, job.ID); err != nil { 1791 t.Fatalf("bad: %v", err) 1792 } 1793 1794 err = state.DeleteJob(1001, job.Namespace, job.ID) 1795 if err != nil { 1796 t.Fatalf("err: %v", err) 1797 } 1798 1799 if !watchFired(ws) { 1800 t.Fatalf("bad") 1801 } 1802 1803 ws = memdb.NewWatchSet() 1804 out, err := state.JobByID(ws, job.Namespace, job.ID) 1805 if err != nil { 1806 t.Fatalf("err: %v", err) 1807 } 1808 1809 if out != nil { 1810 t.Fatalf("bad: %#v %#v", job, out) 1811 } 1812 1813 index, err := state.Index("jobs") 1814 if err != nil { 1815 t.Fatalf("err: %v", err) 1816 } 1817 if index != 1001 { 1818 t.Fatalf("bad: %d", index) 1819 } 1820 1821 summary, err := state.JobSummaryByID(ws, job.Namespace, job.ID) 1822 if err != nil { 1823 t.Fatalf("err: %v", err) 1824 } 1825 if summary != nil { 1826 t.Fatalf("expected summary to be nil, but got: %v", summary) 1827 } 1828 1829 index, err = state.Index("job_summary") 1830 if err != nil { 1831 t.Fatalf("err: %v", err) 1832 } 1833 if index != 1001 { 1834 t.Fatalf("bad: %d", index) 1835 } 1836 1837 versions, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1838 if err != nil { 1839 t.Fatalf("err: %v", err) 1840 } 1841 if len(versions) != 0 { 1842 t.Fatalf("expected no job versions") 1843 } 1844 1845 index, err = state.Index("job_summary") 1846 if err != nil { 1847 t.Fatalf("err: %v", err) 1848 } 1849 if index != 1001 { 1850 t.Fatalf("bad: %d", index) 1851 } 1852 1853 if watchFired(ws) { 1854 t.Fatalf("bad") 1855 } 1856 } 1857 1858 func TestStateStore_DeleteJobTxn_BatchDeletes(t *testing.T) { 1859 t.Parallel() 1860 1861 state := testStateStore(t) 1862 1863 const testJobCount = 10 1864 const jobVersionCount = 4 1865 1866 stateIndex := uint64(1000) 1867 1868 jobs := make([]*structs.Job, testJobCount) 1869 for i := 0; i < testJobCount; i++ { 1870 stateIndex++ 1871 job := mock.BatchJob() 1872 1873 err := state.UpsertJob(stateIndex, job) 1874 require.NoError(t, err) 1875 1876 jobs[i] = job 1877 1878 // Create some versions 1879 for vi := 1; vi < jobVersionCount; vi++ { 1880 stateIndex++ 1881 1882 job := job.Copy() 1883 job.TaskGroups[0].Tasks[0].Env = map[string]string{ 1884 "Version": fmt.Sprintf("%d", vi), 1885 } 1886 1887 require.NoError(t, state.UpsertJob(stateIndex, job)) 1888 } 1889 } 1890 1891 ws := memdb.NewWatchSet() 1892 1893 // Sanity check that jobs are present in DB 1894 job, err := state.JobByID(ws, jobs[0].Namespace, jobs[0].ID) 1895 require.NoError(t, err) 1896 require.Equal(t, jobs[0].ID, job.ID) 1897 1898 jobVersions, err := state.JobVersionsByID(ws, jobs[0].Namespace, jobs[0].ID) 1899 require.NoError(t, err) 1900 require.Equal(t, jobVersionCount, len(jobVersions)) 1901 1902 // Actually delete 1903 const deletionIndex = uint64(10001) 1904 err = state.WithWriteTransaction(func(txn Txn) error { 1905 for i, job := range jobs { 1906 err := state.DeleteJobTxn(deletionIndex, job.Namespace, job.ID, txn) 1907 require.NoError(t, err, "failed at %d %e", i, err) 1908 } 1909 return nil 1910 }) 1911 assert.NoError(t, err) 1912 1913 assert.True(t, watchFired(ws)) 1914 1915 ws = memdb.NewWatchSet() 1916 out, err := state.JobByID(ws, jobs[0].Namespace, jobs[0].ID) 1917 require.NoError(t, err) 1918 require.Nil(t, out) 1919 1920 jobVersions, err = state.JobVersionsByID(ws, jobs[0].Namespace, jobs[0].ID) 1921 require.NoError(t, err) 1922 require.Empty(t, jobVersions) 1923 1924 index, err := state.Index("jobs") 1925 require.NoError(t, err) 1926 require.Equal(t, deletionIndex, index) 1927 } 1928 1929 func TestStateStore_DeleteJob_MultipleVersions(t *testing.T) { 1930 t.Parallel() 1931 1932 state := testStateStore(t) 1933 assert := assert.New(t) 1934 1935 // Create a job and mark it as stable 1936 job := mock.Job() 1937 job.Stable = true 1938 job.Priority = 0 1939 1940 // Create a watchset so we can test that upsert fires the watch 1941 ws := memdb.NewWatchSet() 1942 _, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1943 assert.Nil(err) 1944 assert.Nil(state.UpsertJob(1000, job)) 1945 assert.True(watchFired(ws)) 1946 1947 var finalJob *structs.Job 1948 for i := 1; i < 20; i++ { 1949 finalJob = mock.Job() 1950 finalJob.ID = job.ID 1951 finalJob.Priority = i 1952 assert.Nil(state.UpsertJob(uint64(1000+i), finalJob)) 1953 } 1954 1955 assert.Nil(state.DeleteJob(1020, job.Namespace, job.ID)) 1956 assert.True(watchFired(ws)) 1957 1958 ws = memdb.NewWatchSet() 1959 out, err := state.JobByID(ws, job.Namespace, job.ID) 1960 assert.Nil(err) 1961 assert.Nil(out) 1962 1963 index, err := state.Index("jobs") 1964 assert.Nil(err) 1965 assert.EqualValues(1020, index) 1966 1967 summary, err := state.JobSummaryByID(ws, job.Namespace, job.ID) 1968 assert.Nil(err) 1969 assert.Nil(summary) 1970 1971 index, err = state.Index("job_version") 1972 assert.Nil(err) 1973 assert.EqualValues(1020, index) 1974 1975 versions, err := state.JobVersionsByID(ws, job.Namespace, job.ID) 1976 assert.Nil(err) 1977 assert.Len(versions, 0) 1978 1979 index, err = state.Index("job_summary") 1980 assert.Nil(err) 1981 assert.EqualValues(1020, index) 1982 1983 assert.False(watchFired(ws)) 1984 } 1985 1986 func TestStateStore_DeleteJob_ChildJob(t *testing.T) { 1987 t.Parallel() 1988 1989 state := testStateStore(t) 1990 1991 parent := mock.Job() 1992 if err := state.UpsertJob(998, parent); err != nil { 1993 t.Fatalf("err: %v", err) 1994 } 1995 1996 child := mock.Job() 1997 child.ParentID = parent.ID 1998 1999 if err := state.UpsertJob(999, child); err != nil { 2000 t.Fatalf("err: %v", err) 2001 } 2002 2003 // Create a watchset so we can test that delete fires the watch 2004 ws := memdb.NewWatchSet() 2005 if _, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID); err != nil { 2006 t.Fatalf("bad: %v", err) 2007 } 2008 2009 err := state.DeleteJob(1001, child.Namespace, child.ID) 2010 if err != nil { 2011 t.Fatalf("err: %v", err) 2012 } 2013 if !watchFired(ws) { 2014 t.Fatalf("bad") 2015 } 2016 2017 ws = memdb.NewWatchSet() 2018 summary, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID) 2019 if err != nil { 2020 t.Fatalf("err: %v", err) 2021 } 2022 if summary == nil { 2023 t.Fatalf("nil summary") 2024 } 2025 if summary.JobID != parent.ID { 2026 t.Fatalf("bad summary id: %v", parent.ID) 2027 } 2028 if summary.Children == nil { 2029 t.Fatalf("nil children summary") 2030 } 2031 if summary.Children.Pending != 0 || summary.Children.Running != 0 || summary.Children.Dead != 1 { 2032 t.Fatalf("bad children summary: %v", summary.Children) 2033 } 2034 if watchFired(ws) { 2035 t.Fatalf("bad") 2036 } 2037 } 2038 2039 func TestStateStore_Jobs(t *testing.T) { 2040 t.Parallel() 2041 2042 state := testStateStore(t) 2043 var jobs []*structs.Job 2044 2045 for i := 0; i < 10; i++ { 2046 job := mock.Job() 2047 jobs = append(jobs, job) 2048 2049 err := state.UpsertJob(1000+uint64(i), job) 2050 if err != nil { 2051 t.Fatalf("err: %v", err) 2052 } 2053 } 2054 2055 ws := memdb.NewWatchSet() 2056 iter, err := state.Jobs(ws) 2057 if err != nil { 2058 t.Fatalf("err: %v", err) 2059 } 2060 2061 var out []*structs.Job 2062 for { 2063 raw := iter.Next() 2064 if raw == nil { 2065 break 2066 } 2067 out = append(out, raw.(*structs.Job)) 2068 } 2069 2070 sort.Sort(JobIDSort(jobs)) 2071 sort.Sort(JobIDSort(out)) 2072 2073 if !reflect.DeepEqual(jobs, out) { 2074 t.Fatalf("bad: %#v %#v", jobs, out) 2075 } 2076 if watchFired(ws) { 2077 t.Fatalf("bad") 2078 } 2079 } 2080 2081 func TestStateStore_JobVersions(t *testing.T) { 2082 t.Parallel() 2083 2084 state := testStateStore(t) 2085 var jobs []*structs.Job 2086 2087 for i := 0; i < 10; i++ { 2088 job := mock.Job() 2089 jobs = append(jobs, job) 2090 2091 err := state.UpsertJob(1000+uint64(i), job) 2092 if err != nil { 2093 t.Fatalf("err: %v", err) 2094 } 2095 } 2096 2097 ws := memdb.NewWatchSet() 2098 iter, err := state.JobVersions(ws) 2099 if err != nil { 2100 t.Fatalf("err: %v", err) 2101 } 2102 2103 var out []*structs.Job 2104 for { 2105 raw := iter.Next() 2106 if raw == nil { 2107 break 2108 } 2109 out = append(out, raw.(*structs.Job)) 2110 } 2111 2112 sort.Sort(JobIDSort(jobs)) 2113 sort.Sort(JobIDSort(out)) 2114 2115 if !reflect.DeepEqual(jobs, out) { 2116 t.Fatalf("bad: %#v %#v", jobs, out) 2117 } 2118 if watchFired(ws) { 2119 t.Fatalf("bad") 2120 } 2121 } 2122 2123 func TestStateStore_JobsByIDPrefix(t *testing.T) { 2124 t.Parallel() 2125 2126 state := testStateStore(t) 2127 job := mock.Job() 2128 2129 job.ID = "redis" 2130 err := state.UpsertJob(1000, job) 2131 if err != nil { 2132 t.Fatalf("err: %v", err) 2133 } 2134 2135 ws := memdb.NewWatchSet() 2136 iter, err := state.JobsByIDPrefix(ws, job.Namespace, job.ID) 2137 if err != nil { 2138 t.Fatalf("err: %v", err) 2139 } 2140 2141 gatherJobs := func(iter memdb.ResultIterator) []*structs.Job { 2142 var jobs []*structs.Job 2143 for { 2144 raw := iter.Next() 2145 if raw == nil { 2146 break 2147 } 2148 jobs = append(jobs, raw.(*structs.Job)) 2149 } 2150 return jobs 2151 } 2152 2153 jobs := gatherJobs(iter) 2154 if len(jobs) != 1 { 2155 t.Fatalf("err: %v", err) 2156 } 2157 2158 iter, err = state.JobsByIDPrefix(ws, job.Namespace, "re") 2159 if err != nil { 2160 t.Fatalf("err: %v", err) 2161 } 2162 2163 jobs = gatherJobs(iter) 2164 if len(jobs) != 1 { 2165 t.Fatalf("err: %v", err) 2166 } 2167 if watchFired(ws) { 2168 t.Fatalf("bad") 2169 } 2170 2171 job = mock.Job() 2172 job.ID = "riak" 2173 err = state.UpsertJob(1001, job) 2174 if err != nil { 2175 t.Fatalf("err: %v", err) 2176 } 2177 2178 if !watchFired(ws) { 2179 t.Fatalf("bad") 2180 } 2181 2182 ws = memdb.NewWatchSet() 2183 iter, err = state.JobsByIDPrefix(ws, job.Namespace, "r") 2184 if err != nil { 2185 t.Fatalf("err: %v", err) 2186 } 2187 2188 jobs = gatherJobs(iter) 2189 if len(jobs) != 2 { 2190 t.Fatalf("err: %v", err) 2191 } 2192 2193 iter, err = state.JobsByIDPrefix(ws, job.Namespace, "ri") 2194 if err != nil { 2195 t.Fatalf("err: %v", err) 2196 } 2197 2198 jobs = gatherJobs(iter) 2199 if len(jobs) != 1 { 2200 t.Fatalf("err: %v", err) 2201 } 2202 if watchFired(ws) { 2203 t.Fatalf("bad") 2204 } 2205 } 2206 2207 func TestStateStore_JobsByPeriodic(t *testing.T) { 2208 t.Parallel() 2209 2210 state := testStateStore(t) 2211 var periodic, nonPeriodic []*structs.Job 2212 2213 for i := 0; i < 10; i++ { 2214 job := mock.Job() 2215 nonPeriodic = append(nonPeriodic, job) 2216 2217 err := state.UpsertJob(1000+uint64(i), job) 2218 if err != nil { 2219 t.Fatalf("err: %v", err) 2220 } 2221 } 2222 2223 for i := 0; i < 10; i++ { 2224 job := mock.PeriodicJob() 2225 periodic = append(periodic, job) 2226 2227 err := state.UpsertJob(2000+uint64(i), job) 2228 if err != nil { 2229 t.Fatalf("err: %v", err) 2230 } 2231 } 2232 2233 ws := memdb.NewWatchSet() 2234 iter, err := state.JobsByPeriodic(ws, true) 2235 if err != nil { 2236 t.Fatalf("err: %v", err) 2237 } 2238 2239 var outPeriodic []*structs.Job 2240 for { 2241 raw := iter.Next() 2242 if raw == nil { 2243 break 2244 } 2245 outPeriodic = append(outPeriodic, raw.(*structs.Job)) 2246 } 2247 2248 iter, err = state.JobsByPeriodic(ws, false) 2249 if err != nil { 2250 t.Fatalf("err: %v", err) 2251 } 2252 2253 var outNonPeriodic []*structs.Job 2254 for { 2255 raw := iter.Next() 2256 if raw == nil { 2257 break 2258 } 2259 outNonPeriodic = append(outNonPeriodic, raw.(*structs.Job)) 2260 } 2261 2262 sort.Sort(JobIDSort(periodic)) 2263 sort.Sort(JobIDSort(nonPeriodic)) 2264 sort.Sort(JobIDSort(outPeriodic)) 2265 sort.Sort(JobIDSort(outNonPeriodic)) 2266 2267 if !reflect.DeepEqual(periodic, outPeriodic) { 2268 t.Fatalf("bad: %#v %#v", periodic, outPeriodic) 2269 } 2270 2271 if !reflect.DeepEqual(nonPeriodic, outNonPeriodic) { 2272 t.Fatalf("bad: %#v %#v", nonPeriodic, outNonPeriodic) 2273 } 2274 if watchFired(ws) { 2275 t.Fatalf("bad") 2276 } 2277 } 2278 2279 func TestStateStore_JobsByScheduler(t *testing.T) { 2280 t.Parallel() 2281 2282 state := testStateStore(t) 2283 var serviceJobs []*structs.Job 2284 var sysJobs []*structs.Job 2285 2286 for i := 0; i < 10; i++ { 2287 job := mock.Job() 2288 serviceJobs = append(serviceJobs, job) 2289 2290 err := state.UpsertJob(1000+uint64(i), job) 2291 if err != nil { 2292 t.Fatalf("err: %v", err) 2293 } 2294 } 2295 2296 for i := 0; i < 10; i++ { 2297 job := mock.SystemJob() 2298 job.Status = structs.JobStatusRunning 2299 sysJobs = append(sysJobs, job) 2300 2301 err := state.UpsertJob(2000+uint64(i), job) 2302 if err != nil { 2303 t.Fatalf("err: %v", err) 2304 } 2305 } 2306 2307 ws := memdb.NewWatchSet() 2308 iter, err := state.JobsByScheduler(ws, "service") 2309 if err != nil { 2310 t.Fatalf("err: %v", err) 2311 } 2312 2313 var outService []*structs.Job 2314 for { 2315 raw := iter.Next() 2316 if raw == nil { 2317 break 2318 } 2319 outService = append(outService, raw.(*structs.Job)) 2320 } 2321 2322 iter, err = state.JobsByScheduler(ws, "system") 2323 if err != nil { 2324 t.Fatalf("err: %v", err) 2325 } 2326 2327 var outSystem []*structs.Job 2328 for { 2329 raw := iter.Next() 2330 if raw == nil { 2331 break 2332 } 2333 outSystem = append(outSystem, raw.(*structs.Job)) 2334 } 2335 2336 sort.Sort(JobIDSort(serviceJobs)) 2337 sort.Sort(JobIDSort(sysJobs)) 2338 sort.Sort(JobIDSort(outService)) 2339 sort.Sort(JobIDSort(outSystem)) 2340 2341 if !reflect.DeepEqual(serviceJobs, outService) { 2342 t.Fatalf("bad: %#v %#v", serviceJobs, outService) 2343 } 2344 2345 if !reflect.DeepEqual(sysJobs, outSystem) { 2346 t.Fatalf("bad: %#v %#v", sysJobs, outSystem) 2347 } 2348 if watchFired(ws) { 2349 t.Fatalf("bad") 2350 } 2351 } 2352 2353 func TestStateStore_JobsByGC(t *testing.T) { 2354 t.Parallel() 2355 2356 state := testStateStore(t) 2357 gc, nonGc := make(map[string]struct{}), make(map[string]struct{}) 2358 2359 for i := 0; i < 20; i++ { 2360 var job *structs.Job 2361 if i%2 == 0 { 2362 job = mock.Job() 2363 } else { 2364 job = mock.PeriodicJob() 2365 } 2366 nonGc[job.ID] = struct{}{} 2367 2368 if err := state.UpsertJob(1000+uint64(i), job); err != nil { 2369 t.Fatalf("err: %v", err) 2370 } 2371 } 2372 2373 for i := 0; i < 20; i += 2 { 2374 job := mock.Job() 2375 job.Type = structs.JobTypeBatch 2376 gc[job.ID] = struct{}{} 2377 2378 if err := state.UpsertJob(2000+uint64(i), job); err != nil { 2379 t.Fatalf("err: %v", err) 2380 } 2381 2382 // Create an eval for it 2383 eval := mock.Eval() 2384 eval.JobID = job.ID 2385 eval.Status = structs.EvalStatusComplete 2386 if err := state.UpsertEvals(2000+uint64(i+1), []*structs.Evaluation{eval}); err != nil { 2387 t.Fatalf("err: %v", err) 2388 } 2389 2390 } 2391 2392 ws := memdb.NewWatchSet() 2393 iter, err := state.JobsByGC(ws, true) 2394 if err != nil { 2395 t.Fatalf("err: %v", err) 2396 } 2397 2398 outGc := make(map[string]struct{}) 2399 for i := iter.Next(); i != nil; i = iter.Next() { 2400 j := i.(*structs.Job) 2401 outGc[j.ID] = struct{}{} 2402 } 2403 2404 iter, err = state.JobsByGC(ws, false) 2405 if err != nil { 2406 t.Fatalf("err: %v", err) 2407 } 2408 2409 outNonGc := make(map[string]struct{}) 2410 for i := iter.Next(); i != nil; i = iter.Next() { 2411 j := i.(*structs.Job) 2412 outNonGc[j.ID] = struct{}{} 2413 } 2414 2415 if !reflect.DeepEqual(gc, outGc) { 2416 t.Fatalf("bad: %#v %#v", gc, outGc) 2417 } 2418 2419 if !reflect.DeepEqual(nonGc, outNonGc) { 2420 t.Fatalf("bad: %#v %#v", nonGc, outNonGc) 2421 } 2422 if watchFired(ws) { 2423 t.Fatalf("bad") 2424 } 2425 } 2426 2427 func TestStateStore_RestoreJob(t *testing.T) { 2428 t.Parallel() 2429 2430 state := testStateStore(t) 2431 job := mock.Job() 2432 2433 restore, err := state.Restore() 2434 if err != nil { 2435 t.Fatalf("err: %v", err) 2436 } 2437 2438 err = restore.JobRestore(job) 2439 if err != nil { 2440 t.Fatalf("err: %v", err) 2441 } 2442 restore.Commit() 2443 2444 ws := memdb.NewWatchSet() 2445 out, err := state.JobByID(ws, job.Namespace, job.ID) 2446 if err != nil { 2447 t.Fatalf("err: %v", err) 2448 } 2449 2450 if !reflect.DeepEqual(out, job) { 2451 t.Fatalf("Bad: %#v %#v", out, job) 2452 } 2453 } 2454 2455 func TestStateStore_UpsertPeriodicLaunch(t *testing.T) { 2456 t.Parallel() 2457 2458 state := testStateStore(t) 2459 job := mock.Job() 2460 launch := &structs.PeriodicLaunch{ 2461 ID: job.ID, 2462 Namespace: job.Namespace, 2463 Launch: time.Now(), 2464 } 2465 2466 // Create a watchset so we can test that upsert fires the watch 2467 ws := memdb.NewWatchSet() 2468 if _, err := state.PeriodicLaunchByID(ws, job.Namespace, launch.ID); err != nil { 2469 t.Fatalf("bad: %v", err) 2470 } 2471 2472 err := state.UpsertPeriodicLaunch(1000, launch) 2473 if err != nil { 2474 t.Fatalf("err: %v", err) 2475 } 2476 2477 if !watchFired(ws) { 2478 t.Fatalf("bad") 2479 } 2480 2481 ws = memdb.NewWatchSet() 2482 out, err := state.PeriodicLaunchByID(ws, job.Namespace, job.ID) 2483 if err != nil { 2484 t.Fatalf("err: %v", err) 2485 } 2486 if out.CreateIndex != 1000 { 2487 t.Fatalf("bad: %#v", out) 2488 } 2489 if out.ModifyIndex != 1000 { 2490 t.Fatalf("bad: %#v", out) 2491 } 2492 2493 if !reflect.DeepEqual(launch, out) { 2494 t.Fatalf("bad: %#v %#v", job, out) 2495 } 2496 2497 index, err := state.Index("periodic_launch") 2498 if err != nil { 2499 t.Fatalf("err: %v", err) 2500 } 2501 if index != 1000 { 2502 t.Fatalf("bad: %d", index) 2503 } 2504 2505 if watchFired(ws) { 2506 t.Fatalf("bad") 2507 } 2508 } 2509 2510 func TestStateStore_UpdateUpsertPeriodicLaunch(t *testing.T) { 2511 t.Parallel() 2512 2513 state := testStateStore(t) 2514 job := mock.Job() 2515 launch := &structs.PeriodicLaunch{ 2516 ID: job.ID, 2517 Namespace: job.Namespace, 2518 Launch: time.Now(), 2519 } 2520 2521 err := state.UpsertPeriodicLaunch(1000, launch) 2522 if err != nil { 2523 t.Fatalf("err: %v", err) 2524 } 2525 2526 // Create a watchset so we can test that upsert fires the watch 2527 ws := memdb.NewWatchSet() 2528 if _, err := state.PeriodicLaunchByID(ws, job.Namespace, launch.ID); err != nil { 2529 t.Fatalf("bad: %v", err) 2530 } 2531 2532 launch2 := &structs.PeriodicLaunch{ 2533 ID: job.ID, 2534 Namespace: job.Namespace, 2535 Launch: launch.Launch.Add(1 * time.Second), 2536 } 2537 err = state.UpsertPeriodicLaunch(1001, launch2) 2538 if err != nil { 2539 t.Fatalf("err: %v", err) 2540 } 2541 2542 if !watchFired(ws) { 2543 t.Fatalf("bad") 2544 } 2545 2546 ws = memdb.NewWatchSet() 2547 out, err := state.PeriodicLaunchByID(ws, job.Namespace, job.ID) 2548 if err != nil { 2549 t.Fatalf("err: %v", err) 2550 } 2551 if out.CreateIndex != 1000 { 2552 t.Fatalf("bad: %#v", out) 2553 } 2554 if out.ModifyIndex != 1001 { 2555 t.Fatalf("bad: %#v", out) 2556 } 2557 2558 if !reflect.DeepEqual(launch2, out) { 2559 t.Fatalf("bad: %#v %#v", launch2, out) 2560 } 2561 2562 index, err := state.Index("periodic_launch") 2563 if err != nil { 2564 t.Fatalf("err: %v", err) 2565 } 2566 if index != 1001 { 2567 t.Fatalf("bad: %d", index) 2568 } 2569 2570 if watchFired(ws) { 2571 t.Fatalf("bad") 2572 } 2573 } 2574 2575 func TestStateStore_DeletePeriodicLaunch(t *testing.T) { 2576 t.Parallel() 2577 2578 state := testStateStore(t) 2579 job := mock.Job() 2580 launch := &structs.PeriodicLaunch{ 2581 ID: job.ID, 2582 Namespace: job.Namespace, 2583 Launch: time.Now(), 2584 } 2585 2586 err := state.UpsertPeriodicLaunch(1000, launch) 2587 if err != nil { 2588 t.Fatalf("err: %v", err) 2589 } 2590 2591 // Create a watchset so we can test that delete fires the watch 2592 ws := memdb.NewWatchSet() 2593 if _, err := state.PeriodicLaunchByID(ws, job.Namespace, launch.ID); err != nil { 2594 t.Fatalf("bad: %v", err) 2595 } 2596 2597 err = state.DeletePeriodicLaunch(1001, launch.Namespace, launch.ID) 2598 if err != nil { 2599 t.Fatalf("err: %v", err) 2600 } 2601 2602 if !watchFired(ws) { 2603 t.Fatalf("bad") 2604 } 2605 2606 ws = memdb.NewWatchSet() 2607 out, err := state.PeriodicLaunchByID(ws, job.Namespace, job.ID) 2608 if err != nil { 2609 t.Fatalf("err: %v", err) 2610 } 2611 2612 if out != nil { 2613 t.Fatalf("bad: %#v %#v", job, out) 2614 } 2615 2616 index, err := state.Index("periodic_launch") 2617 if err != nil { 2618 t.Fatalf("err: %v", err) 2619 } 2620 if index != 1001 { 2621 t.Fatalf("bad: %d", index) 2622 } 2623 2624 if watchFired(ws) { 2625 t.Fatalf("bad") 2626 } 2627 } 2628 2629 func TestStateStore_PeriodicLaunches(t *testing.T) { 2630 t.Parallel() 2631 2632 state := testStateStore(t) 2633 var launches []*structs.PeriodicLaunch 2634 2635 for i := 0; i < 10; i++ { 2636 job := mock.Job() 2637 launch := &structs.PeriodicLaunch{ 2638 ID: job.ID, 2639 Namespace: job.Namespace, 2640 Launch: time.Now(), 2641 } 2642 launches = append(launches, launch) 2643 2644 err := state.UpsertPeriodicLaunch(1000+uint64(i), launch) 2645 if err != nil { 2646 t.Fatalf("err: %v", err) 2647 } 2648 } 2649 2650 ws := memdb.NewWatchSet() 2651 iter, err := state.PeriodicLaunches(ws) 2652 if err != nil { 2653 t.Fatalf("err: %v", err) 2654 } 2655 2656 out := make(map[string]*structs.PeriodicLaunch, 10) 2657 for { 2658 raw := iter.Next() 2659 if raw == nil { 2660 break 2661 } 2662 launch := raw.(*structs.PeriodicLaunch) 2663 if _, ok := out[launch.ID]; ok { 2664 t.Fatalf("duplicate: %v", launch.ID) 2665 } 2666 2667 out[launch.ID] = launch 2668 } 2669 2670 for _, launch := range launches { 2671 l, ok := out[launch.ID] 2672 if !ok { 2673 t.Fatalf("bad %v", launch.ID) 2674 } 2675 2676 if !reflect.DeepEqual(launch, l) { 2677 t.Fatalf("bad: %#v %#v", launch, l) 2678 } 2679 2680 delete(out, launch.ID) 2681 } 2682 2683 if len(out) != 0 { 2684 t.Fatalf("leftover: %#v", out) 2685 } 2686 2687 if watchFired(ws) { 2688 t.Fatalf("bad") 2689 } 2690 } 2691 2692 func TestStateStore_RestorePeriodicLaunch(t *testing.T) { 2693 t.Parallel() 2694 2695 state := testStateStore(t) 2696 job := mock.Job() 2697 launch := &structs.PeriodicLaunch{ 2698 ID: job.ID, 2699 Namespace: job.Namespace, 2700 Launch: time.Now(), 2701 } 2702 2703 restore, err := state.Restore() 2704 if err != nil { 2705 t.Fatalf("err: %v", err) 2706 } 2707 2708 err = restore.PeriodicLaunchRestore(launch) 2709 if err != nil { 2710 t.Fatalf("err: %v", err) 2711 } 2712 restore.Commit() 2713 2714 ws := memdb.NewWatchSet() 2715 out, err := state.PeriodicLaunchByID(ws, job.Namespace, job.ID) 2716 if err != nil { 2717 t.Fatalf("err: %v", err) 2718 } 2719 2720 if !reflect.DeepEqual(out, launch) { 2721 t.Fatalf("Bad: %#v %#v", out, job) 2722 } 2723 2724 if watchFired(ws) { 2725 t.Fatalf("bad") 2726 } 2727 } 2728 2729 func TestStateStore_RestoreJobVersion(t *testing.T) { 2730 t.Parallel() 2731 2732 state := testStateStore(t) 2733 job := mock.Job() 2734 2735 restore, err := state.Restore() 2736 if err != nil { 2737 t.Fatalf("err: %v", err) 2738 } 2739 2740 err = restore.JobVersionRestore(job) 2741 if err != nil { 2742 t.Fatalf("err: %v", err) 2743 } 2744 restore.Commit() 2745 2746 ws := memdb.NewWatchSet() 2747 out, err := state.JobByIDAndVersion(ws, job.Namespace, job.ID, job.Version) 2748 if err != nil { 2749 t.Fatalf("err: %v", err) 2750 } 2751 2752 if !reflect.DeepEqual(out, job) { 2753 t.Fatalf("Bad: %#v %#v", out, job) 2754 } 2755 2756 if watchFired(ws) { 2757 t.Fatalf("bad") 2758 } 2759 } 2760 2761 func TestStateStore_RestoreDeployment(t *testing.T) { 2762 t.Parallel() 2763 2764 state := testStateStore(t) 2765 d := mock.Deployment() 2766 2767 restore, err := state.Restore() 2768 if err != nil { 2769 t.Fatalf("err: %v", err) 2770 } 2771 2772 err = restore.DeploymentRestore(d) 2773 if err != nil { 2774 t.Fatalf("err: %v", err) 2775 } 2776 restore.Commit() 2777 2778 ws := memdb.NewWatchSet() 2779 out, err := state.DeploymentByID(ws, d.ID) 2780 if err != nil { 2781 t.Fatalf("err: %v", err) 2782 } 2783 2784 if !reflect.DeepEqual(out, d) { 2785 t.Fatalf("Bad: %#v %#v", out, d) 2786 } 2787 2788 if watchFired(ws) { 2789 t.Fatalf("bad") 2790 } 2791 } 2792 2793 func TestStateStore_RestoreJobSummary(t *testing.T) { 2794 t.Parallel() 2795 2796 state := testStateStore(t) 2797 job := mock.Job() 2798 jobSummary := &structs.JobSummary{ 2799 JobID: job.ID, 2800 Namespace: job.Namespace, 2801 Summary: map[string]structs.TaskGroupSummary{ 2802 "web": { 2803 Starting: 10, 2804 }, 2805 }, 2806 } 2807 restore, err := state.Restore() 2808 if err != nil { 2809 t.Fatalf("err: %v", err) 2810 } 2811 2812 err = restore.JobSummaryRestore(jobSummary) 2813 if err != nil { 2814 t.Fatalf("err: %v", err) 2815 } 2816 restore.Commit() 2817 2818 ws := memdb.NewWatchSet() 2819 out, err := state.JobSummaryByID(ws, job.Namespace, job.ID) 2820 if err != nil { 2821 t.Fatalf("err: %v", err) 2822 } 2823 2824 if !reflect.DeepEqual(out, jobSummary) { 2825 t.Fatalf("Bad: %#v %#v", out, jobSummary) 2826 } 2827 } 2828 2829 func TestStateStore_Indexes(t *testing.T) { 2830 t.Parallel() 2831 2832 state := testStateStore(t) 2833 node := mock.Node() 2834 2835 err := state.UpsertNode(1000, node) 2836 if err != nil { 2837 t.Fatalf("err: %v", err) 2838 } 2839 2840 iter, err := state.Indexes() 2841 if err != nil { 2842 t.Fatalf("err: %v", err) 2843 } 2844 2845 var out []*IndexEntry 2846 for { 2847 raw := iter.Next() 2848 if raw == nil { 2849 break 2850 } 2851 out = append(out, raw.(*IndexEntry)) 2852 } 2853 2854 expect := &IndexEntry{"nodes", 1000} 2855 if l := len(out); l != 1 && l != 2 { 2856 t.Fatalf("unexpected number of index entries: %v", out) 2857 } 2858 2859 for _, index := range out { 2860 if index.Key != expect.Key { 2861 continue 2862 } 2863 if index.Value != expect.Value { 2864 t.Fatalf("bad index; got %d; want %d", index.Value, expect.Value) 2865 } 2866 2867 // We matched 2868 return 2869 } 2870 2871 t.Fatal("did not find expected index entry") 2872 } 2873 2874 func TestStateStore_LatestIndex(t *testing.T) { 2875 t.Parallel() 2876 2877 state := testStateStore(t) 2878 2879 if err := state.UpsertNode(1000, mock.Node()); err != nil { 2880 t.Fatalf("err: %v", err) 2881 } 2882 2883 exp := uint64(2000) 2884 if err := state.UpsertJob(exp, mock.Job()); err != nil { 2885 t.Fatalf("err: %v", err) 2886 } 2887 2888 latest, err := state.LatestIndex() 2889 if err != nil { 2890 t.Fatalf("err: %v", err) 2891 } 2892 2893 if latest != exp { 2894 t.Fatalf("LatestIndex() returned %d; want %d", latest, exp) 2895 } 2896 } 2897 2898 func TestStateStore_RestoreIndex(t *testing.T) { 2899 t.Parallel() 2900 2901 state := testStateStore(t) 2902 2903 restore, err := state.Restore() 2904 if err != nil { 2905 t.Fatalf("err: %v", err) 2906 } 2907 2908 index := &IndexEntry{"jobs", 1000} 2909 err = restore.IndexRestore(index) 2910 if err != nil { 2911 t.Fatalf("err: %v", err) 2912 } 2913 2914 restore.Commit() 2915 2916 out, err := state.Index("jobs") 2917 if err != nil { 2918 t.Fatalf("err: %v", err) 2919 } 2920 2921 if out != 1000 { 2922 t.Fatalf("Bad: %#v %#v", out, 1000) 2923 } 2924 } 2925 2926 func TestStateStore_UpsertEvals_Eval(t *testing.T) { 2927 t.Parallel() 2928 2929 state := testStateStore(t) 2930 eval := mock.Eval() 2931 2932 // Create a watchset so we can test that upsert fires the watch 2933 ws := memdb.NewWatchSet() 2934 if _, err := state.EvalByID(ws, eval.ID); err != nil { 2935 t.Fatalf("bad: %v", err) 2936 } 2937 2938 err := state.UpsertEvals(1000, []*structs.Evaluation{eval}) 2939 if err != nil { 2940 t.Fatalf("err: %v", err) 2941 } 2942 2943 if !watchFired(ws) { 2944 t.Fatalf("bad") 2945 } 2946 2947 ws = memdb.NewWatchSet() 2948 out, err := state.EvalByID(ws, eval.ID) 2949 if err != nil { 2950 t.Fatalf("err: %v", err) 2951 } 2952 2953 if !reflect.DeepEqual(eval, out) { 2954 t.Fatalf("bad: %#v %#v", eval, out) 2955 } 2956 2957 index, err := state.Index("evals") 2958 if err != nil { 2959 t.Fatalf("err: %v", err) 2960 } 2961 if index != 1000 { 2962 t.Fatalf("bad: %d", index) 2963 } 2964 2965 if watchFired(ws) { 2966 t.Fatalf("bad") 2967 } 2968 } 2969 2970 func TestStateStore_UpsertEvals_CancelBlocked(t *testing.T) { 2971 t.Parallel() 2972 2973 state := testStateStore(t) 2974 2975 // Create two blocked evals for the same job 2976 j := "test-job" 2977 b1, b2 := mock.Eval(), mock.Eval() 2978 b1.JobID = j 2979 b1.Status = structs.EvalStatusBlocked 2980 b2.JobID = j 2981 b2.Status = structs.EvalStatusBlocked 2982 2983 err := state.UpsertEvals(999, []*structs.Evaluation{b1, b2}) 2984 if err != nil { 2985 t.Fatalf("err: %v", err) 2986 } 2987 2988 // Create one complete and successful eval for the job 2989 eval := mock.Eval() 2990 eval.JobID = j 2991 eval.Status = structs.EvalStatusComplete 2992 2993 // Create a watchset so we can test that the upsert of the complete eval 2994 // fires the watch 2995 ws := memdb.NewWatchSet() 2996 if _, err := state.EvalByID(ws, b1.ID); err != nil { 2997 t.Fatalf("bad: %v", err) 2998 } 2999 if _, err := state.EvalByID(ws, b2.ID); err != nil { 3000 t.Fatalf("bad: %v", err) 3001 } 3002 3003 if err := state.UpsertEvals(1000, []*structs.Evaluation{eval}); err != nil { 3004 t.Fatalf("err: %v", err) 3005 } 3006 3007 if !watchFired(ws) { 3008 t.Fatalf("bad") 3009 } 3010 3011 ws = memdb.NewWatchSet() 3012 out, err := state.EvalByID(ws, eval.ID) 3013 if err != nil { 3014 t.Fatalf("err: %v", err) 3015 } 3016 3017 if !reflect.DeepEqual(eval, out) { 3018 t.Fatalf("bad: %#v %#v", eval, out) 3019 } 3020 3021 index, err := state.Index("evals") 3022 if err != nil { 3023 t.Fatalf("err: %v", err) 3024 } 3025 if index != 1000 { 3026 t.Fatalf("bad: %d", index) 3027 } 3028 3029 // Get b1/b2 and check they are cancelled 3030 out1, err := state.EvalByID(ws, b1.ID) 3031 if err != nil { 3032 t.Fatalf("err: %v", err) 3033 } 3034 3035 out2, err := state.EvalByID(ws, b2.ID) 3036 if err != nil { 3037 t.Fatalf("err: %v", err) 3038 } 3039 3040 if out1.Status != structs.EvalStatusCancelled || out2.Status != structs.EvalStatusCancelled { 3041 t.Fatalf("bad: %#v %#v", out1, out2) 3042 } 3043 3044 if watchFired(ws) { 3045 t.Fatalf("bad") 3046 } 3047 } 3048 3049 func TestStateStore_Update_UpsertEvals_Eval(t *testing.T) { 3050 t.Parallel() 3051 3052 state := testStateStore(t) 3053 eval := mock.Eval() 3054 3055 err := state.UpsertEvals(1000, []*structs.Evaluation{eval}) 3056 if err != nil { 3057 t.Fatalf("err: %v", err) 3058 } 3059 3060 // Create a watchset so we can test that delete fires the watch 3061 ws := memdb.NewWatchSet() 3062 ws2 := memdb.NewWatchSet() 3063 if _, err := state.EvalByID(ws, eval.ID); err != nil { 3064 t.Fatalf("bad: %v", err) 3065 } 3066 3067 if _, err := state.EvalsByJob(ws2, eval.Namespace, eval.JobID); err != nil { 3068 t.Fatalf("bad: %v", err) 3069 } 3070 3071 eval2 := mock.Eval() 3072 eval2.ID = eval.ID 3073 eval2.JobID = eval.JobID 3074 err = state.UpsertEvals(1001, []*structs.Evaluation{eval2}) 3075 if err != nil { 3076 t.Fatalf("err: %v", err) 3077 } 3078 3079 if !watchFired(ws) { 3080 t.Fatalf("bad") 3081 } 3082 if !watchFired(ws2) { 3083 t.Fatalf("bad") 3084 } 3085 3086 ws = memdb.NewWatchSet() 3087 out, err := state.EvalByID(ws, eval.ID) 3088 if err != nil { 3089 t.Fatalf("err: %v", err) 3090 } 3091 3092 if !reflect.DeepEqual(eval2, out) { 3093 t.Fatalf("bad: %#v %#v", eval2, out) 3094 } 3095 3096 if out.CreateIndex != 1000 { 3097 t.Fatalf("bad: %#v", out) 3098 } 3099 if out.ModifyIndex != 1001 { 3100 t.Fatalf("bad: %#v", out) 3101 } 3102 3103 index, err := state.Index("evals") 3104 if err != nil { 3105 t.Fatalf("err: %v", err) 3106 } 3107 if index != 1001 { 3108 t.Fatalf("bad: %d", index) 3109 } 3110 3111 if watchFired(ws) { 3112 t.Fatalf("bad") 3113 } 3114 } 3115 3116 func TestStateStore_UpsertEvals_Eval_ChildJob(t *testing.T) { 3117 t.Parallel() 3118 3119 state := testStateStore(t) 3120 3121 parent := mock.Job() 3122 if err := state.UpsertJob(998, parent); err != nil { 3123 t.Fatalf("err: %v", err) 3124 } 3125 3126 child := mock.Job() 3127 child.ParentID = parent.ID 3128 3129 if err := state.UpsertJob(999, child); err != nil { 3130 t.Fatalf("err: %v", err) 3131 } 3132 3133 eval := mock.Eval() 3134 eval.Status = structs.EvalStatusComplete 3135 eval.JobID = child.ID 3136 3137 // Create watchsets so we can test that upsert fires the watch 3138 ws := memdb.NewWatchSet() 3139 ws2 := memdb.NewWatchSet() 3140 ws3 := memdb.NewWatchSet() 3141 if _, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID); err != nil { 3142 t.Fatalf("bad: %v", err) 3143 } 3144 if _, err := state.EvalByID(ws2, eval.ID); err != nil { 3145 t.Fatalf("bad: %v", err) 3146 } 3147 if _, err := state.EvalsByJob(ws3, eval.Namespace, eval.JobID); err != nil { 3148 t.Fatalf("bad: %v", err) 3149 } 3150 3151 err := state.UpsertEvals(1000, []*structs.Evaluation{eval}) 3152 if err != nil { 3153 t.Fatalf("err: %v", err) 3154 } 3155 3156 if !watchFired(ws) { 3157 t.Fatalf("bad") 3158 } 3159 if !watchFired(ws2) { 3160 t.Fatalf("bad") 3161 } 3162 if !watchFired(ws3) { 3163 t.Fatalf("bad") 3164 } 3165 3166 ws = memdb.NewWatchSet() 3167 out, err := state.EvalByID(ws, eval.ID) 3168 if err != nil { 3169 t.Fatalf("err: %v", err) 3170 } 3171 3172 if !reflect.DeepEqual(eval, out) { 3173 t.Fatalf("bad: %#v %#v", eval, out) 3174 } 3175 3176 index, err := state.Index("evals") 3177 if err != nil { 3178 t.Fatalf("err: %v", err) 3179 } 3180 if index != 1000 { 3181 t.Fatalf("bad: %d", index) 3182 } 3183 3184 summary, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID) 3185 if err != nil { 3186 t.Fatalf("err: %v", err) 3187 } 3188 if summary == nil { 3189 t.Fatalf("nil summary") 3190 } 3191 if summary.JobID != parent.ID { 3192 t.Fatalf("bad summary id: %v", parent.ID) 3193 } 3194 if summary.Children == nil { 3195 t.Fatalf("nil children summary") 3196 } 3197 if summary.Children.Pending != 0 || summary.Children.Running != 0 || summary.Children.Dead != 1 { 3198 t.Fatalf("bad children summary: %v", summary.Children) 3199 } 3200 3201 if watchFired(ws) { 3202 t.Fatalf("bad") 3203 } 3204 } 3205 3206 func TestStateStore_DeleteEval_Eval(t *testing.T) { 3207 t.Parallel() 3208 3209 state := testStateStore(t) 3210 eval1 := mock.Eval() 3211 eval2 := mock.Eval() 3212 alloc1 := mock.Alloc() 3213 alloc2 := mock.Alloc() 3214 3215 // Create watchsets so we can test that upsert fires the watch 3216 watches := make([]memdb.WatchSet, 12) 3217 for i := 0; i < 12; i++ { 3218 watches[i] = memdb.NewWatchSet() 3219 } 3220 if _, err := state.EvalByID(watches[0], eval1.ID); err != nil { 3221 t.Fatalf("bad: %v", err) 3222 } 3223 if _, err := state.EvalByID(watches[1], eval2.ID); err != nil { 3224 t.Fatalf("bad: %v", err) 3225 } 3226 if _, err := state.EvalsByJob(watches[2], eval1.Namespace, eval1.JobID); err != nil { 3227 t.Fatalf("bad: %v", err) 3228 } 3229 if _, err := state.EvalsByJob(watches[3], eval2.Namespace, eval2.JobID); err != nil { 3230 t.Fatalf("bad: %v", err) 3231 } 3232 if _, err := state.AllocByID(watches[4], alloc1.ID); err != nil { 3233 t.Fatalf("bad: %v", err) 3234 } 3235 if _, err := state.AllocByID(watches[5], alloc2.ID); err != nil { 3236 t.Fatalf("bad: %v", err) 3237 } 3238 if _, err := state.AllocsByEval(watches[6], alloc1.EvalID); err != nil { 3239 t.Fatalf("bad: %v", err) 3240 } 3241 if _, err := state.AllocsByEval(watches[7], alloc2.EvalID); err != nil { 3242 t.Fatalf("bad: %v", err) 3243 } 3244 if _, err := state.AllocsByJob(watches[8], alloc1.Namespace, alloc1.JobID, false); err != nil { 3245 t.Fatalf("bad: %v", err) 3246 } 3247 if _, err := state.AllocsByJob(watches[9], alloc2.Namespace, alloc2.JobID, false); err != nil { 3248 t.Fatalf("bad: %v", err) 3249 } 3250 if _, err := state.AllocsByNode(watches[10], alloc1.NodeID); err != nil { 3251 t.Fatalf("bad: %v", err) 3252 } 3253 if _, err := state.AllocsByNode(watches[11], alloc2.NodeID); err != nil { 3254 t.Fatalf("bad: %v", err) 3255 } 3256 3257 state.UpsertJobSummary(900, mock.JobSummary(eval1.JobID)) 3258 state.UpsertJobSummary(901, mock.JobSummary(eval2.JobID)) 3259 state.UpsertJobSummary(902, mock.JobSummary(alloc1.JobID)) 3260 state.UpsertJobSummary(903, mock.JobSummary(alloc2.JobID)) 3261 err := state.UpsertEvals(1000, []*structs.Evaluation{eval1, eval2}) 3262 if err != nil { 3263 t.Fatalf("err: %v", err) 3264 } 3265 3266 err = state.UpsertAllocs(1001, []*structs.Allocation{alloc1, alloc2}) 3267 if err != nil { 3268 t.Fatalf("err: %v", err) 3269 } 3270 3271 err = state.DeleteEval(1002, []string{eval1.ID, eval2.ID}, []string{alloc1.ID, alloc2.ID}) 3272 if err != nil { 3273 t.Fatalf("err: %v", err) 3274 } 3275 3276 for i, ws := range watches { 3277 if !watchFired(ws) { 3278 t.Fatalf("bad %d", i) 3279 } 3280 } 3281 3282 ws := memdb.NewWatchSet() 3283 out, err := state.EvalByID(ws, eval1.ID) 3284 if err != nil { 3285 t.Fatalf("err: %v", err) 3286 } 3287 3288 if out != nil { 3289 t.Fatalf("bad: %#v %#v", eval1, out) 3290 } 3291 3292 out, err = state.EvalByID(ws, eval2.ID) 3293 if err != nil { 3294 t.Fatalf("err: %v", err) 3295 } 3296 3297 if out != nil { 3298 t.Fatalf("bad: %#v %#v", eval1, out) 3299 } 3300 3301 outA, err := state.AllocByID(ws, alloc1.ID) 3302 if err != nil { 3303 t.Fatalf("err: %v", err) 3304 } 3305 3306 if out != nil { 3307 t.Fatalf("bad: %#v %#v", alloc1, outA) 3308 } 3309 3310 outA, err = state.AllocByID(ws, alloc2.ID) 3311 if err != nil { 3312 t.Fatalf("err: %v", err) 3313 } 3314 3315 if out != nil { 3316 t.Fatalf("bad: %#v %#v", alloc1, outA) 3317 } 3318 3319 index, err := state.Index("evals") 3320 if err != nil { 3321 t.Fatalf("err: %v", err) 3322 } 3323 if index != 1002 { 3324 t.Fatalf("bad: %d", index) 3325 } 3326 3327 index, err = state.Index("allocs") 3328 if err != nil { 3329 t.Fatalf("err: %v", err) 3330 } 3331 if index != 1002 { 3332 t.Fatalf("bad: %d", index) 3333 } 3334 3335 if watchFired(ws) { 3336 t.Fatalf("bad") 3337 } 3338 } 3339 3340 func TestStateStore_DeleteEval_ChildJob(t *testing.T) { 3341 t.Parallel() 3342 3343 state := testStateStore(t) 3344 3345 parent := mock.Job() 3346 if err := state.UpsertJob(998, parent); err != nil { 3347 t.Fatalf("err: %v", err) 3348 } 3349 3350 child := mock.Job() 3351 child.ParentID = parent.ID 3352 3353 if err := state.UpsertJob(999, child); err != nil { 3354 t.Fatalf("err: %v", err) 3355 } 3356 3357 eval1 := mock.Eval() 3358 eval1.JobID = child.ID 3359 alloc1 := mock.Alloc() 3360 alloc1.JobID = child.ID 3361 3362 err := state.UpsertEvals(1000, []*structs.Evaluation{eval1}) 3363 if err != nil { 3364 t.Fatalf("err: %v", err) 3365 } 3366 3367 err = state.UpsertAllocs(1001, []*structs.Allocation{alloc1}) 3368 if err != nil { 3369 t.Fatalf("err: %v", err) 3370 } 3371 3372 // Create watchsets so we can test that delete fires the watch 3373 ws := memdb.NewWatchSet() 3374 if _, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID); err != nil { 3375 t.Fatalf("bad: %v", err) 3376 } 3377 3378 err = state.DeleteEval(1002, []string{eval1.ID}, []string{alloc1.ID}) 3379 if err != nil { 3380 t.Fatalf("err: %v", err) 3381 } 3382 3383 if !watchFired(ws) { 3384 t.Fatalf("bad") 3385 } 3386 3387 ws = memdb.NewWatchSet() 3388 summary, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID) 3389 if err != nil { 3390 t.Fatalf("err: %v", err) 3391 } 3392 if summary == nil { 3393 t.Fatalf("nil summary") 3394 } 3395 if summary.JobID != parent.ID { 3396 t.Fatalf("bad summary id: %v", parent.ID) 3397 } 3398 if summary.Children == nil { 3399 t.Fatalf("nil children summary") 3400 } 3401 if summary.Children.Pending != 0 || summary.Children.Running != 0 || summary.Children.Dead != 1 { 3402 t.Fatalf("bad children summary: %v", summary.Children) 3403 } 3404 3405 if watchFired(ws) { 3406 t.Fatalf("bad") 3407 } 3408 } 3409 3410 func TestStateStore_EvalsByJob(t *testing.T) { 3411 t.Parallel() 3412 3413 state := testStateStore(t) 3414 3415 eval1 := mock.Eval() 3416 eval2 := mock.Eval() 3417 eval2.JobID = eval1.JobID 3418 eval3 := mock.Eval() 3419 evals := []*structs.Evaluation{eval1, eval2} 3420 3421 err := state.UpsertEvals(1000, evals) 3422 if err != nil { 3423 t.Fatalf("err: %v", err) 3424 } 3425 err = state.UpsertEvals(1001, []*structs.Evaluation{eval3}) 3426 if err != nil { 3427 t.Fatalf("err: %v", err) 3428 } 3429 3430 ws := memdb.NewWatchSet() 3431 out, err := state.EvalsByJob(ws, eval1.Namespace, eval1.JobID) 3432 if err != nil { 3433 t.Fatalf("err: %v", err) 3434 } 3435 3436 sort.Sort(EvalIDSort(evals)) 3437 sort.Sort(EvalIDSort(out)) 3438 3439 if !reflect.DeepEqual(evals, out) { 3440 t.Fatalf("bad: %#v %#v", evals, out) 3441 } 3442 3443 if watchFired(ws) { 3444 t.Fatalf("bad") 3445 } 3446 } 3447 3448 func TestStateStore_Evals(t *testing.T) { 3449 t.Parallel() 3450 3451 state := testStateStore(t) 3452 var evals []*structs.Evaluation 3453 3454 for i := 0; i < 10; i++ { 3455 eval := mock.Eval() 3456 evals = append(evals, eval) 3457 3458 err := state.UpsertEvals(1000+uint64(i), []*structs.Evaluation{eval}) 3459 if err != nil { 3460 t.Fatalf("err: %v", err) 3461 } 3462 } 3463 3464 ws := memdb.NewWatchSet() 3465 iter, err := state.Evals(ws) 3466 if err != nil { 3467 t.Fatalf("err: %v", err) 3468 } 3469 3470 var out []*structs.Evaluation 3471 for { 3472 raw := iter.Next() 3473 if raw == nil { 3474 break 3475 } 3476 out = append(out, raw.(*structs.Evaluation)) 3477 } 3478 3479 sort.Sort(EvalIDSort(evals)) 3480 sort.Sort(EvalIDSort(out)) 3481 3482 if !reflect.DeepEqual(evals, out) { 3483 t.Fatalf("bad: %#v %#v", evals, out) 3484 } 3485 3486 if watchFired(ws) { 3487 t.Fatalf("bad") 3488 } 3489 } 3490 3491 func TestStateStore_EvalsByIDPrefix(t *testing.T) { 3492 t.Parallel() 3493 3494 state := testStateStore(t) 3495 var evals []*structs.Evaluation 3496 3497 ids := []string{ 3498 "aaaaaaaa-7bfb-395d-eb95-0685af2176b2", 3499 "aaaaaaab-7bfb-395d-eb95-0685af2176b2", 3500 "aaaaaabb-7bfb-395d-eb95-0685af2176b2", 3501 "aaaaabbb-7bfb-395d-eb95-0685af2176b2", 3502 "aaaabbbb-7bfb-395d-eb95-0685af2176b2", 3503 "aaabbbbb-7bfb-395d-eb95-0685af2176b2", 3504 "aabbbbbb-7bfb-395d-eb95-0685af2176b2", 3505 "abbbbbbb-7bfb-395d-eb95-0685af2176b2", 3506 "bbbbbbbb-7bfb-395d-eb95-0685af2176b2", 3507 } 3508 for i := 0; i < 9; i++ { 3509 eval := mock.Eval() 3510 eval.ID = ids[i] 3511 evals = append(evals, eval) 3512 } 3513 3514 err := state.UpsertEvals(1000, evals) 3515 if err != nil { 3516 t.Fatalf("err: %v", err) 3517 } 3518 3519 ws := memdb.NewWatchSet() 3520 iter, err := state.EvalsByIDPrefix(ws, structs.DefaultNamespace, "aaaa") 3521 if err != nil { 3522 t.Fatalf("err: %v", err) 3523 } 3524 3525 gatherEvals := func(iter memdb.ResultIterator) []*structs.Evaluation { 3526 var evals []*structs.Evaluation 3527 for { 3528 raw := iter.Next() 3529 if raw == nil { 3530 break 3531 } 3532 evals = append(evals, raw.(*structs.Evaluation)) 3533 } 3534 return evals 3535 } 3536 3537 out := gatherEvals(iter) 3538 if len(out) != 5 { 3539 t.Fatalf("bad: expected five evaluations, got: %#v", out) 3540 } 3541 3542 sort.Sort(EvalIDSort(evals)) 3543 3544 for index, eval := range out { 3545 if ids[index] != eval.ID { 3546 t.Fatalf("bad: got unexpected id: %s", eval.ID) 3547 } 3548 } 3549 3550 iter, err = state.EvalsByIDPrefix(ws, structs.DefaultNamespace, "b-a7bfb") 3551 if err != nil { 3552 t.Fatalf("err: %v", err) 3553 } 3554 3555 out = gatherEvals(iter) 3556 if len(out) != 0 { 3557 t.Fatalf("bad: unexpected zero evaluations, got: %#v", out) 3558 } 3559 3560 if watchFired(ws) { 3561 t.Fatalf("bad") 3562 } 3563 } 3564 3565 func TestStateStore_RestoreEval(t *testing.T) { 3566 t.Parallel() 3567 3568 state := testStateStore(t) 3569 eval := mock.Eval() 3570 3571 restore, err := state.Restore() 3572 if err != nil { 3573 t.Fatalf("err: %v", err) 3574 } 3575 3576 err = restore.EvalRestore(eval) 3577 if err != nil { 3578 t.Fatalf("err: %v", err) 3579 } 3580 restore.Commit() 3581 3582 ws := memdb.NewWatchSet() 3583 out, err := state.EvalByID(ws, eval.ID) 3584 if err != nil { 3585 t.Fatalf("err: %v", err) 3586 } 3587 3588 if !reflect.DeepEqual(out, eval) { 3589 t.Fatalf("Bad: %#v %#v", out, eval) 3590 } 3591 } 3592 3593 func TestStateStore_UpdateAllocsFromClient(t *testing.T) { 3594 t.Parallel() 3595 3596 state := testStateStore(t) 3597 parent := mock.Job() 3598 if err := state.UpsertJob(998, parent); err != nil { 3599 t.Fatalf("err: %v", err) 3600 } 3601 3602 child := mock.Job() 3603 child.ParentID = parent.ID 3604 if err := state.UpsertJob(999, child); err != nil { 3605 t.Fatalf("err: %v", err) 3606 } 3607 3608 alloc := mock.Alloc() 3609 alloc.JobID = child.ID 3610 alloc.Job = child 3611 3612 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 3613 if err != nil { 3614 t.Fatalf("err: %v", err) 3615 } 3616 3617 ws := memdb.NewWatchSet() 3618 summary, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID) 3619 if err != nil { 3620 t.Fatalf("err: %v", err) 3621 } 3622 if summary == nil { 3623 t.Fatalf("nil summary") 3624 } 3625 if summary.JobID != parent.ID { 3626 t.Fatalf("bad summary id: %v", parent.ID) 3627 } 3628 if summary.Children == nil { 3629 t.Fatalf("nil children summary") 3630 } 3631 if summary.Children.Pending != 0 || summary.Children.Running != 1 || summary.Children.Dead != 0 { 3632 t.Fatalf("bad children summary: %v", summary.Children) 3633 } 3634 3635 // Create watchsets so we can test that update fires the watch 3636 ws = memdb.NewWatchSet() 3637 if _, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID); err != nil { 3638 t.Fatalf("bad: %v", err) 3639 } 3640 3641 // Create the delta updates 3642 ts := map[string]*structs.TaskState{"web": {State: structs.TaskStateRunning}} 3643 update := &structs.Allocation{ 3644 ID: alloc.ID, 3645 ClientStatus: structs.AllocClientStatusComplete, 3646 TaskStates: ts, 3647 JobID: alloc.JobID, 3648 TaskGroup: alloc.TaskGroup, 3649 } 3650 err = state.UpdateAllocsFromClient(1001, []*structs.Allocation{update}) 3651 if err != nil { 3652 t.Fatalf("err: %v", err) 3653 } 3654 3655 if !watchFired(ws) { 3656 t.Fatalf("bad") 3657 } 3658 3659 ws = memdb.NewWatchSet() 3660 summary, err = state.JobSummaryByID(ws, parent.Namespace, parent.ID) 3661 if err != nil { 3662 t.Fatalf("err: %v", err) 3663 } 3664 if summary == nil { 3665 t.Fatalf("nil summary") 3666 } 3667 if summary.JobID != parent.ID { 3668 t.Fatalf("bad summary id: %v", parent.ID) 3669 } 3670 if summary.Children == nil { 3671 t.Fatalf("nil children summary") 3672 } 3673 if summary.Children.Pending != 0 || summary.Children.Running != 0 || summary.Children.Dead != 1 { 3674 t.Fatalf("bad children summary: %v", summary.Children) 3675 } 3676 3677 if watchFired(ws) { 3678 t.Fatalf("bad") 3679 } 3680 } 3681 3682 func TestStateStore_UpdateAllocsFromClient_ChildJob(t *testing.T) { 3683 t.Parallel() 3684 3685 state := testStateStore(t) 3686 alloc1 := mock.Alloc() 3687 alloc2 := mock.Alloc() 3688 3689 if err := state.UpsertJob(999, alloc1.Job); err != nil { 3690 t.Fatalf("err: %v", err) 3691 } 3692 if err := state.UpsertJob(999, alloc2.Job); err != nil { 3693 t.Fatalf("err: %v", err) 3694 } 3695 3696 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc1, alloc2}) 3697 if err != nil { 3698 t.Fatalf("err: %v", err) 3699 } 3700 3701 // Create watchsets so we can test that update fires the watch 3702 watches := make([]memdb.WatchSet, 8) 3703 for i := 0; i < 8; i++ { 3704 watches[i] = memdb.NewWatchSet() 3705 } 3706 if _, err := state.AllocByID(watches[0], alloc1.ID); err != nil { 3707 t.Fatalf("bad: %v", err) 3708 } 3709 if _, err := state.AllocByID(watches[1], alloc2.ID); err != nil { 3710 t.Fatalf("bad: %v", err) 3711 } 3712 if _, err := state.AllocsByEval(watches[2], alloc1.EvalID); err != nil { 3713 t.Fatalf("bad: %v", err) 3714 } 3715 if _, err := state.AllocsByEval(watches[3], alloc2.EvalID); err != nil { 3716 t.Fatalf("bad: %v", err) 3717 } 3718 if _, err := state.AllocsByJob(watches[4], alloc1.Namespace, alloc1.JobID, false); err != nil { 3719 t.Fatalf("bad: %v", err) 3720 } 3721 if _, err := state.AllocsByJob(watches[5], alloc2.Namespace, alloc2.JobID, false); err != nil { 3722 t.Fatalf("bad: %v", err) 3723 } 3724 if _, err := state.AllocsByNode(watches[6], alloc1.NodeID); err != nil { 3725 t.Fatalf("bad: %v", err) 3726 } 3727 if _, err := state.AllocsByNode(watches[7], alloc2.NodeID); err != nil { 3728 t.Fatalf("bad: %v", err) 3729 } 3730 3731 // Create the delta updates 3732 ts := map[string]*structs.TaskState{"web": {State: structs.TaskStatePending}} 3733 update := &structs.Allocation{ 3734 ID: alloc1.ID, 3735 ClientStatus: structs.AllocClientStatusFailed, 3736 TaskStates: ts, 3737 JobID: alloc1.JobID, 3738 TaskGroup: alloc1.TaskGroup, 3739 } 3740 update2 := &structs.Allocation{ 3741 ID: alloc2.ID, 3742 ClientStatus: structs.AllocClientStatusRunning, 3743 TaskStates: ts, 3744 JobID: alloc2.JobID, 3745 TaskGroup: alloc2.TaskGroup, 3746 } 3747 3748 err = state.UpdateAllocsFromClient(1001, []*structs.Allocation{update, update2}) 3749 if err != nil { 3750 t.Fatalf("err: %v", err) 3751 } 3752 3753 for i, ws := range watches { 3754 if !watchFired(ws) { 3755 t.Fatalf("bad %d", i) 3756 } 3757 } 3758 3759 ws := memdb.NewWatchSet() 3760 out, err := state.AllocByID(ws, alloc1.ID) 3761 if err != nil { 3762 t.Fatalf("err: %v", err) 3763 } 3764 3765 alloc1.CreateIndex = 1000 3766 alloc1.ModifyIndex = 1001 3767 alloc1.TaskStates = ts 3768 alloc1.ClientStatus = structs.AllocClientStatusFailed 3769 if !reflect.DeepEqual(alloc1, out) { 3770 t.Fatalf("bad: %#v %#v", alloc1, out) 3771 } 3772 3773 out, err = state.AllocByID(ws, alloc2.ID) 3774 if err != nil { 3775 t.Fatalf("err: %v", err) 3776 } 3777 3778 alloc2.ModifyIndex = 1000 3779 alloc2.ModifyIndex = 1001 3780 alloc2.ClientStatus = structs.AllocClientStatusRunning 3781 alloc2.TaskStates = ts 3782 if !reflect.DeepEqual(alloc2, out) { 3783 t.Fatalf("bad: %#v %#v", alloc2, out) 3784 } 3785 3786 index, err := state.Index("allocs") 3787 if err != nil { 3788 t.Fatalf("err: %v", err) 3789 } 3790 if index != 1001 { 3791 t.Fatalf("bad: %d", index) 3792 } 3793 3794 // Ensure summaries have been updated 3795 summary, err := state.JobSummaryByID(ws, alloc1.Namespace, alloc1.JobID) 3796 if err != nil { 3797 t.Fatalf("err: %v", err) 3798 } 3799 tgSummary := summary.Summary["web"] 3800 if tgSummary.Failed != 1 { 3801 t.Fatalf("expected failed: %v, actual: %v, summary: %#v", 1, tgSummary.Failed, tgSummary) 3802 } 3803 3804 summary2, err := state.JobSummaryByID(ws, alloc2.Namespace, alloc2.JobID) 3805 if err != nil { 3806 t.Fatalf("err: %v", err) 3807 } 3808 tgSummary2 := summary2.Summary["web"] 3809 if tgSummary2.Running != 1 { 3810 t.Fatalf("expected running: %v, actual: %v", 1, tgSummary2.Running) 3811 } 3812 3813 if watchFired(ws) { 3814 t.Fatalf("bad") 3815 } 3816 } 3817 3818 func TestStateStore_UpdateMultipleAllocsFromClient(t *testing.T) { 3819 t.Parallel() 3820 3821 state := testStateStore(t) 3822 alloc := mock.Alloc() 3823 3824 if err := state.UpsertJob(999, alloc.Job); err != nil { 3825 t.Fatalf("err: %v", err) 3826 } 3827 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 3828 if err != nil { 3829 t.Fatalf("err: %v", err) 3830 } 3831 3832 // Create the delta updates 3833 ts := map[string]*structs.TaskState{"web": {State: structs.TaskStatePending}} 3834 update := &structs.Allocation{ 3835 ID: alloc.ID, 3836 ClientStatus: structs.AllocClientStatusRunning, 3837 TaskStates: ts, 3838 JobID: alloc.JobID, 3839 TaskGroup: alloc.TaskGroup, 3840 } 3841 update2 := &structs.Allocation{ 3842 ID: alloc.ID, 3843 ClientStatus: structs.AllocClientStatusPending, 3844 TaskStates: ts, 3845 JobID: alloc.JobID, 3846 TaskGroup: alloc.TaskGroup, 3847 } 3848 3849 err = state.UpdateAllocsFromClient(1001, []*structs.Allocation{update, update2}) 3850 if err != nil { 3851 t.Fatalf("err: %v", err) 3852 } 3853 3854 ws := memdb.NewWatchSet() 3855 out, err := state.AllocByID(ws, alloc.ID) 3856 if err != nil { 3857 t.Fatalf("err: %v", err) 3858 } 3859 3860 alloc.CreateIndex = 1000 3861 alloc.ModifyIndex = 1001 3862 alloc.TaskStates = ts 3863 alloc.ClientStatus = structs.AllocClientStatusPending 3864 if !reflect.DeepEqual(alloc, out) { 3865 t.Fatalf("bad: %#v , actual:%#v", alloc, out) 3866 } 3867 3868 summary, err := state.JobSummaryByID(ws, alloc.Namespace, alloc.JobID) 3869 expectedSummary := &structs.JobSummary{ 3870 JobID: alloc.JobID, 3871 Namespace: alloc.Namespace, 3872 Summary: map[string]structs.TaskGroupSummary{ 3873 "web": { 3874 Starting: 1, 3875 }, 3876 }, 3877 Children: new(structs.JobChildrenSummary), 3878 CreateIndex: 999, 3879 ModifyIndex: 1001, 3880 } 3881 if err != nil { 3882 t.Fatalf("err: %v", err) 3883 } 3884 if !reflect.DeepEqual(summary, expectedSummary) { 3885 t.Fatalf("expected: %#v, actual: %#v", expectedSummary, summary) 3886 } 3887 } 3888 3889 func TestStateStore_UpdateAllocsFromClient_Deployment(t *testing.T) { 3890 t.Parallel() 3891 require := require.New(t) 3892 3893 state := testStateStore(t) 3894 3895 alloc := mock.Alloc() 3896 now := time.Now() 3897 alloc.CreateTime = now.UnixNano() 3898 pdeadline := 5 * time.Minute 3899 deployment := mock.Deployment() 3900 deployment.TaskGroups[alloc.TaskGroup].ProgressDeadline = pdeadline 3901 alloc.DeploymentID = deployment.ID 3902 3903 require.Nil(state.UpsertJob(999, alloc.Job)) 3904 require.Nil(state.UpsertDeployment(1000, deployment)) 3905 require.Nil(state.UpsertAllocs(1001, []*structs.Allocation{alloc})) 3906 3907 healthy := now.Add(time.Second) 3908 update := &structs.Allocation{ 3909 ID: alloc.ID, 3910 ClientStatus: structs.AllocClientStatusRunning, 3911 JobID: alloc.JobID, 3912 TaskGroup: alloc.TaskGroup, 3913 DeploymentStatus: &structs.AllocDeploymentStatus{ 3914 Healthy: helper.BoolToPtr(true), 3915 Timestamp: healthy, 3916 }, 3917 } 3918 require.Nil(state.UpdateAllocsFromClient(1001, []*structs.Allocation{update})) 3919 3920 // Check that the deployment state was updated because the healthy 3921 // deployment 3922 dout, err := state.DeploymentByID(nil, deployment.ID) 3923 require.Nil(err) 3924 require.NotNil(dout) 3925 require.Len(dout.TaskGroups, 1) 3926 dstate := dout.TaskGroups[alloc.TaskGroup] 3927 require.NotNil(dstate) 3928 require.Equal(1, dstate.PlacedAllocs) 3929 require.True(healthy.Add(pdeadline).Equal(dstate.RequireProgressBy)) 3930 } 3931 3932 // This tests that the deployment state is merged correctly 3933 func TestStateStore_UpdateAllocsFromClient_DeploymentStateMerges(t *testing.T) { 3934 t.Parallel() 3935 require := require.New(t) 3936 3937 state := testStateStore(t) 3938 alloc := mock.Alloc() 3939 now := time.Now() 3940 alloc.CreateTime = now.UnixNano() 3941 pdeadline := 5 * time.Minute 3942 deployment := mock.Deployment() 3943 deployment.TaskGroups[alloc.TaskGroup].ProgressDeadline = pdeadline 3944 alloc.DeploymentID = deployment.ID 3945 alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ 3946 Canary: true, 3947 } 3948 3949 require.Nil(state.UpsertJob(999, alloc.Job)) 3950 require.Nil(state.UpsertDeployment(1000, deployment)) 3951 require.Nil(state.UpsertAllocs(1001, []*structs.Allocation{alloc})) 3952 3953 update := &structs.Allocation{ 3954 ID: alloc.ID, 3955 ClientStatus: structs.AllocClientStatusRunning, 3956 JobID: alloc.JobID, 3957 TaskGroup: alloc.TaskGroup, 3958 DeploymentStatus: &structs.AllocDeploymentStatus{ 3959 Healthy: helper.BoolToPtr(true), 3960 Canary: false, 3961 }, 3962 } 3963 require.Nil(state.UpdateAllocsFromClient(1001, []*structs.Allocation{update})) 3964 3965 // Check that the merging of the deployment status was correct 3966 out, err := state.AllocByID(nil, alloc.ID) 3967 require.Nil(err) 3968 require.NotNil(out) 3969 require.True(out.DeploymentStatus.Canary) 3970 require.NotNil(out.DeploymentStatus.Healthy) 3971 require.True(*out.DeploymentStatus.Healthy) 3972 } 3973 3974 func TestStateStore_UpsertAlloc_Alloc(t *testing.T) { 3975 t.Parallel() 3976 3977 state := testStateStore(t) 3978 alloc := mock.Alloc() 3979 3980 if err := state.UpsertJob(999, alloc.Job); err != nil { 3981 t.Fatalf("err: %v", err) 3982 } 3983 3984 // Create watchsets so we can test that update fires the watch 3985 watches := make([]memdb.WatchSet, 4) 3986 for i := 0; i < 4; i++ { 3987 watches[i] = memdb.NewWatchSet() 3988 } 3989 if _, err := state.AllocByID(watches[0], alloc.ID); err != nil { 3990 t.Fatalf("bad: %v", err) 3991 } 3992 if _, err := state.AllocsByEval(watches[1], alloc.EvalID); err != nil { 3993 t.Fatalf("bad: %v", err) 3994 } 3995 if _, err := state.AllocsByJob(watches[2], alloc.Namespace, alloc.JobID, false); err != nil { 3996 t.Fatalf("bad: %v", err) 3997 } 3998 if _, err := state.AllocsByNode(watches[3], alloc.NodeID); err != nil { 3999 t.Fatalf("bad: %v", err) 4000 } 4001 4002 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 4003 if err != nil { 4004 t.Fatalf("err: %v", err) 4005 } 4006 4007 for i, ws := range watches { 4008 if !watchFired(ws) { 4009 t.Fatalf("bad %d", i) 4010 } 4011 } 4012 4013 ws := memdb.NewWatchSet() 4014 out, err := state.AllocByID(ws, alloc.ID) 4015 if err != nil { 4016 t.Fatalf("err: %v", err) 4017 } 4018 4019 if !reflect.DeepEqual(alloc, out) { 4020 t.Fatalf("bad: %#v %#v", alloc, out) 4021 } 4022 4023 index, err := state.Index("allocs") 4024 if err != nil { 4025 t.Fatalf("err: %v", err) 4026 } 4027 if index != 1000 { 4028 t.Fatalf("bad: %d", index) 4029 } 4030 4031 summary, err := state.JobSummaryByID(ws, alloc.Namespace, alloc.JobID) 4032 if err != nil { 4033 t.Fatalf("err: %v", err) 4034 } 4035 4036 tgSummary, ok := summary.Summary["web"] 4037 if !ok { 4038 t.Fatalf("no summary for task group web") 4039 } 4040 if tgSummary.Starting != 1 { 4041 t.Fatalf("expected queued: %v, actual: %v", 1, tgSummary.Starting) 4042 } 4043 4044 if watchFired(ws) { 4045 t.Fatalf("bad") 4046 } 4047 } 4048 4049 func TestStateStore_UpsertAlloc_Deployment(t *testing.T) { 4050 t.Parallel() 4051 require := require.New(t) 4052 4053 state := testStateStore(t) 4054 alloc := mock.Alloc() 4055 now := time.Now() 4056 alloc.CreateTime = now.UnixNano() 4057 alloc.ModifyTime = now.UnixNano() 4058 pdeadline := 5 * time.Minute 4059 deployment := mock.Deployment() 4060 deployment.TaskGroups[alloc.TaskGroup].ProgressDeadline = pdeadline 4061 alloc.DeploymentID = deployment.ID 4062 4063 require.Nil(state.UpsertJob(999, alloc.Job)) 4064 require.Nil(state.UpsertDeployment(1000, deployment)) 4065 4066 // Create a watch set so we can test that update fires the watch 4067 ws := memdb.NewWatchSet() 4068 require.Nil(state.AllocsByDeployment(ws, alloc.DeploymentID)) 4069 4070 err := state.UpsertAllocs(1001, []*structs.Allocation{alloc}) 4071 require.Nil(err) 4072 4073 if !watchFired(ws) { 4074 t.Fatalf("watch not fired") 4075 } 4076 4077 ws = memdb.NewWatchSet() 4078 allocs, err := state.AllocsByDeployment(ws, alloc.DeploymentID) 4079 require.Nil(err) 4080 require.Len(allocs, 1) 4081 require.EqualValues(alloc, allocs[0]) 4082 4083 index, err := state.Index("allocs") 4084 require.Nil(err) 4085 require.EqualValues(1001, index) 4086 if watchFired(ws) { 4087 t.Fatalf("bad") 4088 } 4089 4090 // Check that the deployment state was updated 4091 dout, err := state.DeploymentByID(nil, deployment.ID) 4092 require.Nil(err) 4093 require.NotNil(dout) 4094 require.Len(dout.TaskGroups, 1) 4095 dstate := dout.TaskGroups[alloc.TaskGroup] 4096 require.NotNil(dstate) 4097 require.Equal(1, dstate.PlacedAllocs) 4098 require.True(now.Add(pdeadline).Equal(dstate.RequireProgressBy)) 4099 } 4100 4101 // Testing to ensure we keep issue 4102 // https://github.com/hashicorp/nomad/issues/2583 fixed 4103 func TestStateStore_UpsertAlloc_No_Job(t *testing.T) { 4104 t.Parallel() 4105 4106 state := testStateStore(t) 4107 alloc := mock.Alloc() 4108 alloc.Job = nil 4109 4110 err := state.UpsertAllocs(999, []*structs.Allocation{alloc}) 4111 if err == nil || !strings.Contains(err.Error(), "without a job") { 4112 t.Fatalf("expect err: %v", err) 4113 } 4114 } 4115 4116 func TestStateStore_UpsertAlloc_ChildJob(t *testing.T) { 4117 t.Parallel() 4118 4119 state := testStateStore(t) 4120 4121 parent := mock.Job() 4122 if err := state.UpsertJob(998, parent); err != nil { 4123 t.Fatalf("err: %v", err) 4124 } 4125 4126 child := mock.Job() 4127 child.ParentID = parent.ID 4128 4129 if err := state.UpsertJob(999, child); err != nil { 4130 t.Fatalf("err: %v", err) 4131 } 4132 4133 alloc := mock.Alloc() 4134 alloc.JobID = child.ID 4135 alloc.Job = child 4136 4137 // Create watchsets so we can test that delete fires the watch 4138 ws := memdb.NewWatchSet() 4139 if _, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID); err != nil { 4140 t.Fatalf("bad: %v", err) 4141 } 4142 4143 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 4144 if err != nil { 4145 t.Fatalf("err: %v", err) 4146 } 4147 4148 if !watchFired(ws) { 4149 t.Fatalf("bad") 4150 } 4151 4152 ws = memdb.NewWatchSet() 4153 summary, err := state.JobSummaryByID(ws, parent.Namespace, parent.ID) 4154 if err != nil { 4155 t.Fatalf("err: %v", err) 4156 } 4157 if summary == nil { 4158 t.Fatalf("nil summary") 4159 } 4160 if summary.JobID != parent.ID { 4161 t.Fatalf("bad summary id: %v", parent.ID) 4162 } 4163 if summary.Children == nil { 4164 t.Fatalf("nil children summary") 4165 } 4166 if summary.Children.Pending != 0 || summary.Children.Running != 1 || summary.Children.Dead != 0 { 4167 t.Fatalf("bad children summary: %v", summary.Children) 4168 } 4169 4170 if watchFired(ws) { 4171 t.Fatalf("bad") 4172 } 4173 } 4174 4175 func TestStateStore_UpdateAlloc_Alloc(t *testing.T) { 4176 t.Parallel() 4177 4178 state := testStateStore(t) 4179 alloc := mock.Alloc() 4180 4181 if err := state.UpsertJob(999, alloc.Job); err != nil { 4182 t.Fatalf("err: %v", err) 4183 } 4184 4185 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 4186 if err != nil { 4187 t.Fatalf("err: %v", err) 4188 } 4189 4190 ws := memdb.NewWatchSet() 4191 summary, err := state.JobSummaryByID(ws, alloc.Namespace, alloc.JobID) 4192 if err != nil { 4193 t.Fatalf("err: %v", err) 4194 } 4195 tgSummary := summary.Summary["web"] 4196 if tgSummary.Starting != 1 { 4197 t.Fatalf("expected starting: %v, actual: %v", 1, tgSummary.Starting) 4198 } 4199 4200 alloc2 := mock.Alloc() 4201 alloc2.ID = alloc.ID 4202 alloc2.NodeID = alloc.NodeID + ".new" 4203 state.UpsertJobSummary(1001, mock.JobSummary(alloc2.JobID)) 4204 4205 // Create watchsets so we can test that update fires the watch 4206 watches := make([]memdb.WatchSet, 4) 4207 for i := 0; i < 4; i++ { 4208 watches[i] = memdb.NewWatchSet() 4209 } 4210 if _, err := state.AllocByID(watches[0], alloc2.ID); err != nil { 4211 t.Fatalf("bad: %v", err) 4212 } 4213 if _, err := state.AllocsByEval(watches[1], alloc2.EvalID); err != nil { 4214 t.Fatalf("bad: %v", err) 4215 } 4216 if _, err := state.AllocsByJob(watches[2], alloc2.Namespace, alloc2.JobID, false); err != nil { 4217 t.Fatalf("bad: %v", err) 4218 } 4219 if _, err := state.AllocsByNode(watches[3], alloc2.NodeID); err != nil { 4220 t.Fatalf("bad: %v", err) 4221 } 4222 4223 err = state.UpsertAllocs(1002, []*structs.Allocation{alloc2}) 4224 if err != nil { 4225 t.Fatalf("err: %v", err) 4226 } 4227 4228 for i, ws := range watches { 4229 if !watchFired(ws) { 4230 t.Fatalf("bad %d", i) 4231 } 4232 } 4233 4234 ws = memdb.NewWatchSet() 4235 out, err := state.AllocByID(ws, alloc.ID) 4236 if err != nil { 4237 t.Fatalf("err: %v", err) 4238 } 4239 4240 if !reflect.DeepEqual(alloc2, out) { 4241 t.Fatalf("bad: %#v %#v", alloc2, out) 4242 } 4243 4244 if out.CreateIndex != 1000 { 4245 t.Fatalf("bad: %#v", out) 4246 } 4247 if out.ModifyIndex != 1002 { 4248 t.Fatalf("bad: %#v", out) 4249 } 4250 4251 index, err := state.Index("allocs") 4252 if err != nil { 4253 t.Fatalf("err: %v", err) 4254 } 4255 if index != 1002 { 4256 t.Fatalf("bad: %d", index) 4257 } 4258 4259 // Ensure that summary hasb't changed 4260 summary, err = state.JobSummaryByID(ws, alloc.Namespace, alloc.JobID) 4261 if err != nil { 4262 t.Fatalf("err: %v", err) 4263 } 4264 tgSummary = summary.Summary["web"] 4265 if tgSummary.Starting != 1 { 4266 t.Fatalf("expected starting: %v, actual: %v", 1, tgSummary.Starting) 4267 } 4268 4269 if watchFired(ws) { 4270 t.Fatalf("bad") 4271 } 4272 } 4273 4274 // This test ensures that the state store will mark the clients status as lost 4275 // when set rather than preferring the existing status. 4276 func TestStateStore_UpdateAlloc_Lost(t *testing.T) { 4277 t.Parallel() 4278 4279 state := testStateStore(t) 4280 alloc := mock.Alloc() 4281 alloc.ClientStatus = "foo" 4282 4283 if err := state.UpsertJob(999, alloc.Job); err != nil { 4284 t.Fatalf("err: %v", err) 4285 } 4286 4287 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 4288 if err != nil { 4289 t.Fatalf("err: %v", err) 4290 } 4291 4292 alloc2 := new(structs.Allocation) 4293 *alloc2 = *alloc 4294 alloc2.ClientStatus = structs.AllocClientStatusLost 4295 if err := state.UpsertAllocs(1001, []*structs.Allocation{alloc2}); err != nil { 4296 t.Fatalf("err: %v", err) 4297 } 4298 4299 ws := memdb.NewWatchSet() 4300 out, err := state.AllocByID(ws, alloc2.ID) 4301 if err != nil { 4302 t.Fatalf("err: %v", err) 4303 } 4304 4305 if out.ClientStatus != structs.AllocClientStatusLost { 4306 t.Fatalf("bad: %#v", out) 4307 } 4308 } 4309 4310 // This test ensures an allocation can be updated when there is no job 4311 // associated with it. This will happen when a job is stopped by an user which 4312 // has non-terminal allocations on clients 4313 func TestStateStore_UpdateAlloc_NoJob(t *testing.T) { 4314 t.Parallel() 4315 4316 state := testStateStore(t) 4317 alloc := mock.Alloc() 4318 4319 // Upsert a job 4320 state.UpsertJobSummary(998, mock.JobSummary(alloc.JobID)) 4321 if err := state.UpsertJob(999, alloc.Job); err != nil { 4322 t.Fatalf("err: %v", err) 4323 } 4324 4325 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 4326 if err != nil { 4327 t.Fatalf("err: %v", err) 4328 } 4329 4330 if err := state.DeleteJob(1001, alloc.Namespace, alloc.JobID); err != nil { 4331 t.Fatalf("err: %v", err) 4332 } 4333 4334 // Update the desired state of the allocation to stop 4335 allocCopy := alloc.Copy() 4336 allocCopy.DesiredStatus = structs.AllocDesiredStatusStop 4337 if err := state.UpsertAllocs(1002, []*structs.Allocation{allocCopy}); err != nil { 4338 t.Fatalf("err: %v", err) 4339 } 4340 4341 // Update the client state of the allocation to complete 4342 allocCopy1 := allocCopy.Copy() 4343 allocCopy1.ClientStatus = structs.AllocClientStatusComplete 4344 if err := state.UpdateAllocsFromClient(1003, []*structs.Allocation{allocCopy1}); err != nil { 4345 t.Fatalf("err: %v", err) 4346 } 4347 4348 ws := memdb.NewWatchSet() 4349 out, _ := state.AllocByID(ws, alloc.ID) 4350 // Update the modify index of the alloc before comparing 4351 allocCopy1.ModifyIndex = 1003 4352 if !reflect.DeepEqual(out, allocCopy1) { 4353 t.Fatalf("expected: %#v \n actual: %#v", allocCopy1, out) 4354 } 4355 } 4356 4357 func TestStateStore_UpdateAllocDesiredTransition(t *testing.T) { 4358 t.Parallel() 4359 require := require.New(t) 4360 4361 state := testStateStore(t) 4362 alloc := mock.Alloc() 4363 4364 require.Nil(state.UpsertJob(999, alloc.Job)) 4365 require.Nil(state.UpsertAllocs(1000, []*structs.Allocation{alloc})) 4366 4367 t1 := &structs.DesiredTransition{ 4368 Migrate: helper.BoolToPtr(true), 4369 } 4370 t2 := &structs.DesiredTransition{ 4371 Migrate: helper.BoolToPtr(false), 4372 } 4373 eval := &structs.Evaluation{ 4374 ID: uuid.Generate(), 4375 Namespace: alloc.Namespace, 4376 Priority: alloc.Job.Priority, 4377 Type: alloc.Job.Type, 4378 TriggeredBy: structs.EvalTriggerNodeDrain, 4379 JobID: alloc.Job.ID, 4380 JobModifyIndex: alloc.Job.ModifyIndex, 4381 Status: structs.EvalStatusPending, 4382 } 4383 evals := []*structs.Evaluation{eval} 4384 4385 m := map[string]*structs.DesiredTransition{alloc.ID: t1} 4386 require.Nil(state.UpdateAllocsDesiredTransitions(1001, m, evals)) 4387 4388 ws := memdb.NewWatchSet() 4389 out, err := state.AllocByID(ws, alloc.ID) 4390 require.Nil(err) 4391 require.NotNil(out.DesiredTransition.Migrate) 4392 require.True(*out.DesiredTransition.Migrate) 4393 require.EqualValues(1000, out.CreateIndex) 4394 require.EqualValues(1001, out.ModifyIndex) 4395 4396 index, err := state.Index("allocs") 4397 require.Nil(err) 4398 require.EqualValues(1001, index) 4399 4400 // Check the eval is created 4401 eout, err := state.EvalByID(nil, eval.ID) 4402 require.Nil(err) 4403 require.NotNil(eout) 4404 4405 m = map[string]*structs.DesiredTransition{alloc.ID: t2} 4406 require.Nil(state.UpdateAllocsDesiredTransitions(1002, m, evals)) 4407 4408 ws = memdb.NewWatchSet() 4409 out, err = state.AllocByID(ws, alloc.ID) 4410 require.Nil(err) 4411 require.NotNil(out.DesiredTransition.Migrate) 4412 require.False(*out.DesiredTransition.Migrate) 4413 require.EqualValues(1000, out.CreateIndex) 4414 require.EqualValues(1002, out.ModifyIndex) 4415 4416 index, err = state.Index("allocs") 4417 require.Nil(err) 4418 require.EqualValues(1002, index) 4419 4420 // Try with a bogus alloc id 4421 m = map[string]*structs.DesiredTransition{uuid.Generate(): t2} 4422 require.Nil(state.UpdateAllocsDesiredTransitions(1003, m, evals)) 4423 } 4424 4425 func TestStateStore_JobSummary(t *testing.T) { 4426 t.Parallel() 4427 4428 state := testStateStore(t) 4429 4430 // Add a job 4431 job := mock.Job() 4432 state.UpsertJob(900, job) 4433 4434 // Get the job back 4435 ws := memdb.NewWatchSet() 4436 outJob, _ := state.JobByID(ws, job.Namespace, job.ID) 4437 if outJob.CreateIndex != 900 { 4438 t.Fatalf("bad create index: %v", outJob.CreateIndex) 4439 } 4440 summary, _ := state.JobSummaryByID(ws, job.Namespace, job.ID) 4441 if summary.CreateIndex != 900 { 4442 t.Fatalf("bad create index: %v", summary.CreateIndex) 4443 } 4444 4445 // Upsert an allocation 4446 alloc := mock.Alloc() 4447 alloc.JobID = job.ID 4448 alloc.Job = job 4449 state.UpsertAllocs(910, []*structs.Allocation{alloc}) 4450 4451 // Update the alloc from client 4452 alloc1 := alloc.Copy() 4453 alloc1.ClientStatus = structs.AllocClientStatusPending 4454 alloc1.DesiredStatus = "" 4455 state.UpdateAllocsFromClient(920, []*structs.Allocation{alloc}) 4456 4457 alloc3 := alloc.Copy() 4458 alloc3.ClientStatus = structs.AllocClientStatusRunning 4459 alloc3.DesiredStatus = "" 4460 state.UpdateAllocsFromClient(930, []*structs.Allocation{alloc3}) 4461 4462 // Upsert the alloc 4463 alloc4 := alloc.Copy() 4464 alloc4.ClientStatus = structs.AllocClientStatusPending 4465 alloc4.DesiredStatus = structs.AllocDesiredStatusRun 4466 state.UpsertAllocs(950, []*structs.Allocation{alloc4}) 4467 4468 // Again upsert the alloc 4469 alloc5 := alloc.Copy() 4470 alloc5.ClientStatus = structs.AllocClientStatusPending 4471 alloc5.DesiredStatus = structs.AllocDesiredStatusRun 4472 state.UpsertAllocs(970, []*structs.Allocation{alloc5}) 4473 4474 if !watchFired(ws) { 4475 t.Fatalf("bad") 4476 } 4477 4478 expectedSummary := structs.JobSummary{ 4479 JobID: job.ID, 4480 Namespace: job.Namespace, 4481 Summary: map[string]structs.TaskGroupSummary{ 4482 "web": { 4483 Running: 1, 4484 }, 4485 }, 4486 Children: new(structs.JobChildrenSummary), 4487 CreateIndex: 900, 4488 ModifyIndex: 930, 4489 } 4490 4491 summary, _ = state.JobSummaryByID(ws, job.Namespace, job.ID) 4492 if !reflect.DeepEqual(&expectedSummary, summary) { 4493 t.Fatalf("expected: %#v, actual: %v", expectedSummary, summary) 4494 } 4495 4496 // De-register the job. 4497 state.DeleteJob(980, job.Namespace, job.ID) 4498 4499 // Shouldn't have any effect on the summary 4500 alloc6 := alloc.Copy() 4501 alloc6.ClientStatus = structs.AllocClientStatusRunning 4502 alloc6.DesiredStatus = "" 4503 state.UpdateAllocsFromClient(990, []*structs.Allocation{alloc6}) 4504 4505 // We shouldn't have any summary at this point 4506 summary, _ = state.JobSummaryByID(ws, job.Namespace, job.ID) 4507 if summary != nil { 4508 t.Fatalf("expected nil, actual: %#v", summary) 4509 } 4510 4511 // Re-register the same job 4512 job1 := mock.Job() 4513 job1.ID = job.ID 4514 state.UpsertJob(1000, job1) 4515 outJob2, _ := state.JobByID(ws, job1.Namespace, job1.ID) 4516 if outJob2.CreateIndex != 1000 { 4517 t.Fatalf("bad create index: %v", outJob2.CreateIndex) 4518 } 4519 summary, _ = state.JobSummaryByID(ws, job1.Namespace, job1.ID) 4520 if summary.CreateIndex != 1000 { 4521 t.Fatalf("bad create index: %v", summary.CreateIndex) 4522 } 4523 4524 // Upsert an allocation 4525 alloc7 := alloc.Copy() 4526 alloc7.JobID = outJob.ID 4527 alloc7.Job = outJob 4528 alloc7.ClientStatus = structs.AllocClientStatusComplete 4529 alloc7.DesiredStatus = structs.AllocDesiredStatusRun 4530 state.UpdateAllocsFromClient(1020, []*structs.Allocation{alloc7}) 4531 4532 expectedSummary = structs.JobSummary{ 4533 JobID: job.ID, 4534 Namespace: job.Namespace, 4535 Summary: map[string]structs.TaskGroupSummary{ 4536 "web": {}, 4537 }, 4538 Children: new(structs.JobChildrenSummary), 4539 CreateIndex: 1000, 4540 ModifyIndex: 1000, 4541 } 4542 4543 summary, _ = state.JobSummaryByID(ws, job1.Namespace, job1.ID) 4544 if !reflect.DeepEqual(&expectedSummary, summary) { 4545 t.Fatalf("expected: %#v, actual: %#v", expectedSummary, summary) 4546 } 4547 } 4548 4549 func TestStateStore_ReconcileJobSummary(t *testing.T) { 4550 t.Parallel() 4551 4552 state := testStateStore(t) 4553 4554 // Create an alloc 4555 alloc := mock.Alloc() 4556 4557 // Add another task group to the job 4558 tg2 := alloc.Job.TaskGroups[0].Copy() 4559 tg2.Name = "db" 4560 alloc.Job.TaskGroups = append(alloc.Job.TaskGroups, tg2) 4561 state.UpsertJob(100, alloc.Job) 4562 4563 // Create one more alloc for the db task group 4564 alloc2 := mock.Alloc() 4565 alloc2.TaskGroup = "db" 4566 alloc2.JobID = alloc.JobID 4567 alloc2.Job = alloc.Job 4568 4569 // Upserts the alloc 4570 state.UpsertAllocs(110, []*structs.Allocation{alloc, alloc2}) 4571 4572 // Change the state of the first alloc to running 4573 alloc3 := alloc.Copy() 4574 alloc3.ClientStatus = structs.AllocClientStatusRunning 4575 state.UpdateAllocsFromClient(120, []*structs.Allocation{alloc3}) 4576 4577 //Add some more allocs to the second tg 4578 alloc4 := mock.Alloc() 4579 alloc4.JobID = alloc.JobID 4580 alloc4.Job = alloc.Job 4581 alloc4.TaskGroup = "db" 4582 alloc5 := alloc4.Copy() 4583 alloc5.ClientStatus = structs.AllocClientStatusRunning 4584 4585 alloc6 := mock.Alloc() 4586 alloc6.JobID = alloc.JobID 4587 alloc6.Job = alloc.Job 4588 alloc6.TaskGroup = "db" 4589 alloc7 := alloc6.Copy() 4590 alloc7.ClientStatus = structs.AllocClientStatusComplete 4591 4592 alloc8 := mock.Alloc() 4593 alloc8.JobID = alloc.JobID 4594 alloc8.Job = alloc.Job 4595 alloc8.TaskGroup = "db" 4596 alloc9 := alloc8.Copy() 4597 alloc9.ClientStatus = structs.AllocClientStatusFailed 4598 4599 alloc10 := mock.Alloc() 4600 alloc10.JobID = alloc.JobID 4601 alloc10.Job = alloc.Job 4602 alloc10.TaskGroup = "db" 4603 alloc11 := alloc10.Copy() 4604 alloc11.ClientStatus = structs.AllocClientStatusLost 4605 4606 state.UpsertAllocs(130, []*structs.Allocation{alloc4, alloc6, alloc8, alloc10}) 4607 4608 state.UpdateAllocsFromClient(150, []*structs.Allocation{alloc5, alloc7, alloc9, alloc11}) 4609 4610 // DeleteJobSummary is a helper method and doesn't modify the indexes table 4611 state.DeleteJobSummary(130, alloc.Namespace, alloc.Job.ID) 4612 4613 state.ReconcileJobSummaries(120) 4614 4615 ws := memdb.NewWatchSet() 4616 summary, _ := state.JobSummaryByID(ws, alloc.Namespace, alloc.Job.ID) 4617 expectedSummary := structs.JobSummary{ 4618 JobID: alloc.Job.ID, 4619 Namespace: alloc.Namespace, 4620 Summary: map[string]structs.TaskGroupSummary{ 4621 "web": { 4622 Running: 1, 4623 }, 4624 "db": { 4625 Starting: 1, 4626 Running: 1, 4627 Failed: 1, 4628 Complete: 1, 4629 Lost: 1, 4630 }, 4631 }, 4632 CreateIndex: 100, 4633 ModifyIndex: 120, 4634 } 4635 if !reflect.DeepEqual(&expectedSummary, summary) { 4636 t.Fatalf("expected: %v, actual: %v", expectedSummary, summary) 4637 } 4638 } 4639 4640 func TestStateStore_ReconcileParentJobSummary(t *testing.T) { 4641 t.Parallel() 4642 require := require.New(t) 4643 4644 state := testStateStore(t) 4645 4646 // Add a node 4647 node := mock.Node() 4648 state.UpsertNode(80, node) 4649 4650 // Make a parameterized job 4651 job1 := mock.BatchJob() 4652 job1.ID = "test" 4653 job1.ParameterizedJob = &structs.ParameterizedJobConfig{ 4654 Payload: "random", 4655 } 4656 job1.TaskGroups[0].Count = 1 4657 state.UpsertJob(100, job1) 4658 4659 // Make a child job 4660 childJob := job1.Copy() 4661 childJob.ID = job1.ID + "dispatch-23423423" 4662 childJob.ParentID = job1.ID 4663 childJob.Dispatched = true 4664 childJob.Status = structs.JobStatusRunning 4665 4666 // Make some allocs for child job 4667 alloc := mock.Alloc() 4668 alloc.NodeID = node.ID 4669 alloc.Job = childJob 4670 alloc.JobID = childJob.ID 4671 alloc.ClientStatus = structs.AllocClientStatusRunning 4672 4673 alloc2 := mock.Alloc() 4674 alloc2.NodeID = node.ID 4675 alloc2.Job = childJob 4676 alloc2.JobID = childJob.ID 4677 alloc2.ClientStatus = structs.AllocClientStatusFailed 4678 4679 require.Nil(state.UpsertJob(110, childJob)) 4680 require.Nil(state.UpsertAllocs(111, []*structs.Allocation{alloc, alloc2})) 4681 4682 // Make the summary incorrect in the state store 4683 summary, err := state.JobSummaryByID(nil, job1.Namespace, job1.ID) 4684 require.Nil(err) 4685 4686 summary.Children = nil 4687 summary.Summary = make(map[string]structs.TaskGroupSummary) 4688 summary.Summary["web"] = structs.TaskGroupSummary{ 4689 Queued: 1, 4690 } 4691 4692 // Delete the child job summary 4693 state.DeleteJobSummary(125, childJob.Namespace, childJob.ID) 4694 4695 state.ReconcileJobSummaries(120) 4696 4697 ws := memdb.NewWatchSet() 4698 4699 // Verify parent summary is corrected 4700 summary, _ = state.JobSummaryByID(ws, alloc.Namespace, job1.ID) 4701 expectedSummary := structs.JobSummary{ 4702 JobID: job1.ID, 4703 Namespace: job1.Namespace, 4704 Summary: make(map[string]structs.TaskGroupSummary), 4705 Children: &structs.JobChildrenSummary{ 4706 Running: 1, 4707 }, 4708 CreateIndex: 100, 4709 ModifyIndex: 120, 4710 } 4711 require.Equal(&expectedSummary, summary) 4712 4713 // Verify child job summary is also correct 4714 childSummary, _ := state.JobSummaryByID(ws, childJob.Namespace, childJob.ID) 4715 expectedChildSummary := structs.JobSummary{ 4716 JobID: childJob.ID, 4717 Namespace: childJob.Namespace, 4718 Summary: map[string]structs.TaskGroupSummary{ 4719 "web": { 4720 Running: 1, 4721 Failed: 1, 4722 }, 4723 }, 4724 CreateIndex: 110, 4725 ModifyIndex: 120, 4726 } 4727 require.Equal(&expectedChildSummary, childSummary) 4728 } 4729 4730 func TestStateStore_UpdateAlloc_JobNotPresent(t *testing.T) { 4731 t.Parallel() 4732 4733 state := testStateStore(t) 4734 4735 alloc := mock.Alloc() 4736 state.UpsertJob(100, alloc.Job) 4737 state.UpsertAllocs(200, []*structs.Allocation{alloc}) 4738 4739 // Delete the job 4740 state.DeleteJob(300, alloc.Namespace, alloc.Job.ID) 4741 4742 // Update the alloc 4743 alloc1 := alloc.Copy() 4744 alloc1.ClientStatus = structs.AllocClientStatusRunning 4745 4746 // Updating allocation should not throw any error 4747 if err := state.UpdateAllocsFromClient(400, []*structs.Allocation{alloc1}); err != nil { 4748 t.Fatalf("expect err: %v", err) 4749 } 4750 4751 // Re-Register the job 4752 state.UpsertJob(500, alloc.Job) 4753 4754 // Update the alloc again 4755 alloc2 := alloc.Copy() 4756 alloc2.ClientStatus = structs.AllocClientStatusComplete 4757 if err := state.UpdateAllocsFromClient(400, []*structs.Allocation{alloc1}); err != nil { 4758 t.Fatalf("expect err: %v", err) 4759 } 4760 4761 // Job Summary of the newly registered job shouldn't account for the 4762 // allocation update for the older job 4763 expectedSummary := structs.JobSummary{ 4764 JobID: alloc1.JobID, 4765 Namespace: alloc1.Namespace, 4766 Summary: map[string]structs.TaskGroupSummary{ 4767 "web": {}, 4768 }, 4769 Children: new(structs.JobChildrenSummary), 4770 CreateIndex: 500, 4771 ModifyIndex: 500, 4772 } 4773 4774 ws := memdb.NewWatchSet() 4775 summary, _ := state.JobSummaryByID(ws, alloc.Namespace, alloc.Job.ID) 4776 if !reflect.DeepEqual(&expectedSummary, summary) { 4777 t.Fatalf("expected: %v, actual: %v", expectedSummary, summary) 4778 } 4779 } 4780 4781 func TestStateStore_EvictAlloc_Alloc(t *testing.T) { 4782 t.Parallel() 4783 4784 state := testStateStore(t) 4785 alloc := mock.Alloc() 4786 4787 state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)) 4788 err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 4789 if err != nil { 4790 t.Fatalf("err: %v", err) 4791 } 4792 4793 alloc2 := new(structs.Allocation) 4794 *alloc2 = *alloc 4795 alloc2.DesiredStatus = structs.AllocDesiredStatusEvict 4796 err = state.UpsertAllocs(1001, []*structs.Allocation{alloc2}) 4797 if err != nil { 4798 t.Fatalf("err: %v", err) 4799 } 4800 4801 ws := memdb.NewWatchSet() 4802 out, err := state.AllocByID(ws, alloc.ID) 4803 if err != nil { 4804 t.Fatalf("err: %v", err) 4805 } 4806 4807 if out.DesiredStatus != structs.AllocDesiredStatusEvict { 4808 t.Fatalf("bad: %#v %#v", alloc, out) 4809 } 4810 4811 index, err := state.Index("allocs") 4812 if err != nil { 4813 t.Fatalf("err: %v", err) 4814 } 4815 if index != 1001 { 4816 t.Fatalf("bad: %d", index) 4817 } 4818 } 4819 4820 func TestStateStore_AllocsByNode(t *testing.T) { 4821 t.Parallel() 4822 4823 state := testStateStore(t) 4824 var allocs []*structs.Allocation 4825 4826 for i := 0; i < 10; i++ { 4827 alloc := mock.Alloc() 4828 alloc.NodeID = "foo" 4829 allocs = append(allocs, alloc) 4830 } 4831 4832 for idx, alloc := range allocs { 4833 state.UpsertJobSummary(uint64(900+idx), mock.JobSummary(alloc.JobID)) 4834 } 4835 4836 err := state.UpsertAllocs(1000, allocs) 4837 if err != nil { 4838 t.Fatalf("err: %v", err) 4839 } 4840 4841 ws := memdb.NewWatchSet() 4842 out, err := state.AllocsByNode(ws, "foo") 4843 if err != nil { 4844 t.Fatalf("err: %v", err) 4845 } 4846 4847 sort.Sort(AllocIDSort(allocs)) 4848 sort.Sort(AllocIDSort(out)) 4849 4850 if !reflect.DeepEqual(allocs, out) { 4851 t.Fatalf("bad: %#v %#v", allocs, out) 4852 } 4853 4854 if watchFired(ws) { 4855 t.Fatalf("bad") 4856 } 4857 } 4858 4859 func TestStateStore_AllocsByNodeTerminal(t *testing.T) { 4860 t.Parallel() 4861 4862 state := testStateStore(t) 4863 var allocs, term, nonterm []*structs.Allocation 4864 4865 for i := 0; i < 10; i++ { 4866 alloc := mock.Alloc() 4867 alloc.NodeID = "foo" 4868 if i%2 == 0 { 4869 alloc.DesiredStatus = structs.AllocDesiredStatusStop 4870 term = append(term, alloc) 4871 } else { 4872 nonterm = append(nonterm, alloc) 4873 } 4874 allocs = append(allocs, alloc) 4875 } 4876 4877 for idx, alloc := range allocs { 4878 state.UpsertJobSummary(uint64(900+idx), mock.JobSummary(alloc.JobID)) 4879 } 4880 4881 err := state.UpsertAllocs(1000, allocs) 4882 if err != nil { 4883 t.Fatalf("err: %v", err) 4884 } 4885 4886 // Verify the terminal allocs 4887 ws := memdb.NewWatchSet() 4888 out, err := state.AllocsByNodeTerminal(ws, "foo", true) 4889 if err != nil { 4890 t.Fatalf("err: %v", err) 4891 } 4892 4893 sort.Sort(AllocIDSort(term)) 4894 sort.Sort(AllocIDSort(out)) 4895 4896 if !reflect.DeepEqual(term, out) { 4897 t.Fatalf("bad: %#v %#v", term, out) 4898 } 4899 4900 // Verify the non-terminal allocs 4901 out, err = state.AllocsByNodeTerminal(ws, "foo", false) 4902 if err != nil { 4903 t.Fatalf("err: %v", err) 4904 } 4905 4906 sort.Sort(AllocIDSort(nonterm)) 4907 sort.Sort(AllocIDSort(out)) 4908 4909 if !reflect.DeepEqual(nonterm, out) { 4910 t.Fatalf("bad: %#v %#v", nonterm, out) 4911 } 4912 4913 if watchFired(ws) { 4914 t.Fatalf("bad") 4915 } 4916 } 4917 4918 func TestStateStore_AllocsByJob(t *testing.T) { 4919 t.Parallel() 4920 4921 state := testStateStore(t) 4922 var allocs []*structs.Allocation 4923 4924 for i := 0; i < 10; i++ { 4925 alloc := mock.Alloc() 4926 alloc.JobID = "foo" 4927 allocs = append(allocs, alloc) 4928 } 4929 4930 for i, alloc := range allocs { 4931 state.UpsertJobSummary(uint64(900+i), mock.JobSummary(alloc.JobID)) 4932 } 4933 4934 err := state.UpsertAllocs(1000, allocs) 4935 if err != nil { 4936 t.Fatalf("err: %v", err) 4937 } 4938 4939 ws := memdb.NewWatchSet() 4940 out, err := state.AllocsByJob(ws, mock.Alloc().Namespace, "foo", false) 4941 if err != nil { 4942 t.Fatalf("err: %v", err) 4943 } 4944 4945 sort.Sort(AllocIDSort(allocs)) 4946 sort.Sort(AllocIDSort(out)) 4947 4948 if !reflect.DeepEqual(allocs, out) { 4949 t.Fatalf("bad: %#v %#v", allocs, out) 4950 } 4951 4952 if watchFired(ws) { 4953 t.Fatalf("bad") 4954 } 4955 } 4956 4957 func TestStateStore_AllocsForRegisteredJob(t *testing.T) { 4958 t.Parallel() 4959 4960 state := testStateStore(t) 4961 var allocs []*structs.Allocation 4962 var allocs1 []*structs.Allocation 4963 4964 job := mock.Job() 4965 job.ID = "foo" 4966 state.UpsertJob(100, job) 4967 for i := 0; i < 3; i++ { 4968 alloc := mock.Alloc() 4969 alloc.Job = job 4970 alloc.JobID = job.ID 4971 allocs = append(allocs, alloc) 4972 } 4973 if err := state.UpsertAllocs(200, allocs); err != nil { 4974 t.Fatalf("err: %v", err) 4975 } 4976 4977 if err := state.DeleteJob(250, job.Namespace, job.ID); err != nil { 4978 t.Fatalf("err: %v", err) 4979 } 4980 4981 job1 := mock.Job() 4982 job1.ID = "foo" 4983 job1.CreateIndex = 50 4984 state.UpsertJob(300, job1) 4985 for i := 0; i < 4; i++ { 4986 alloc := mock.Alloc() 4987 alloc.Job = job1 4988 alloc.JobID = job1.ID 4989 allocs1 = append(allocs1, alloc) 4990 } 4991 4992 if err := state.UpsertAllocs(1000, allocs1); err != nil { 4993 t.Fatalf("err: %v", err) 4994 } 4995 4996 ws := memdb.NewWatchSet() 4997 out, err := state.AllocsByJob(ws, job1.Namespace, job1.ID, true) 4998 if err != nil { 4999 t.Fatalf("err: %v", err) 5000 } 5001 5002 expected := len(allocs) + len(allocs1) 5003 if len(out) != expected { 5004 t.Fatalf("expected: %v, actual: %v", expected, len(out)) 5005 } 5006 5007 out1, err := state.AllocsByJob(ws, job1.Namespace, job1.ID, false) 5008 if err != nil { 5009 t.Fatalf("bad: %v", err) 5010 } 5011 5012 expected = len(allocs1) 5013 if len(out1) != expected { 5014 t.Fatalf("expected: %v, actual: %v", expected, len(out1)) 5015 } 5016 5017 if watchFired(ws) { 5018 t.Fatalf("bad") 5019 } 5020 } 5021 5022 func TestStateStore_AllocsByIDPrefix(t *testing.T) { 5023 t.Parallel() 5024 5025 state := testStateStore(t) 5026 var allocs []*structs.Allocation 5027 5028 ids := []string{ 5029 "aaaaaaaa-7bfb-395d-eb95-0685af2176b2", 5030 "aaaaaaab-7bfb-395d-eb95-0685af2176b2", 5031 "aaaaaabb-7bfb-395d-eb95-0685af2176b2", 5032 "aaaaabbb-7bfb-395d-eb95-0685af2176b2", 5033 "aaaabbbb-7bfb-395d-eb95-0685af2176b2", 5034 "aaabbbbb-7bfb-395d-eb95-0685af2176b2", 5035 "aabbbbbb-7bfb-395d-eb95-0685af2176b2", 5036 "abbbbbbb-7bfb-395d-eb95-0685af2176b2", 5037 "bbbbbbbb-7bfb-395d-eb95-0685af2176b2", 5038 } 5039 for i := 0; i < 9; i++ { 5040 alloc := mock.Alloc() 5041 alloc.ID = ids[i] 5042 allocs = append(allocs, alloc) 5043 } 5044 5045 for i, alloc := range allocs { 5046 state.UpsertJobSummary(uint64(900+i), mock.JobSummary(alloc.JobID)) 5047 } 5048 5049 err := state.UpsertAllocs(1000, allocs) 5050 if err != nil { 5051 t.Fatalf("err: %v", err) 5052 } 5053 5054 ws := memdb.NewWatchSet() 5055 iter, err := state.AllocsByIDPrefix(ws, structs.DefaultNamespace, "aaaa") 5056 if err != nil { 5057 t.Fatalf("err: %v", err) 5058 } 5059 5060 gatherAllocs := func(iter memdb.ResultIterator) []*structs.Allocation { 5061 var allocs []*structs.Allocation 5062 for { 5063 raw := iter.Next() 5064 if raw == nil { 5065 break 5066 } 5067 allocs = append(allocs, raw.(*structs.Allocation)) 5068 } 5069 return allocs 5070 } 5071 5072 out := gatherAllocs(iter) 5073 if len(out) != 5 { 5074 t.Fatalf("bad: expected five allocations, got: %#v", out) 5075 } 5076 5077 sort.Sort(AllocIDSort(allocs)) 5078 5079 for index, alloc := range out { 5080 if ids[index] != alloc.ID { 5081 t.Fatalf("bad: got unexpected id: %s", alloc.ID) 5082 } 5083 } 5084 5085 iter, err = state.AllocsByIDPrefix(ws, structs.DefaultNamespace, "b-a7bfb") 5086 if err != nil { 5087 t.Fatalf("err: %v", err) 5088 } 5089 5090 out = gatherAllocs(iter) 5091 if len(out) != 0 { 5092 t.Fatalf("bad: unexpected zero allocations, got: %#v", out) 5093 } 5094 5095 if watchFired(ws) { 5096 t.Fatalf("bad") 5097 } 5098 } 5099 5100 func TestStateStore_Allocs(t *testing.T) { 5101 t.Parallel() 5102 5103 state := testStateStore(t) 5104 var allocs []*structs.Allocation 5105 5106 for i := 0; i < 10; i++ { 5107 alloc := mock.Alloc() 5108 allocs = append(allocs, alloc) 5109 } 5110 for i, alloc := range allocs { 5111 state.UpsertJobSummary(uint64(900+i), mock.JobSummary(alloc.JobID)) 5112 } 5113 5114 err := state.UpsertAllocs(1000, allocs) 5115 if err != nil { 5116 t.Fatalf("err: %v", err) 5117 } 5118 5119 ws := memdb.NewWatchSet() 5120 iter, err := state.Allocs(ws) 5121 if err != nil { 5122 t.Fatalf("err: %v", err) 5123 } 5124 5125 var out []*structs.Allocation 5126 for { 5127 raw := iter.Next() 5128 if raw == nil { 5129 break 5130 } 5131 out = append(out, raw.(*structs.Allocation)) 5132 } 5133 5134 sort.Sort(AllocIDSort(allocs)) 5135 sort.Sort(AllocIDSort(out)) 5136 5137 if !reflect.DeepEqual(allocs, out) { 5138 t.Fatalf("bad: %#v %#v", allocs, out) 5139 } 5140 5141 if watchFired(ws) { 5142 t.Fatalf("bad") 5143 } 5144 } 5145 5146 func TestStateStore_Allocs_PrevAlloc(t *testing.T) { 5147 t.Parallel() 5148 5149 state := testStateStore(t) 5150 var allocs []*structs.Allocation 5151 5152 require := require.New(t) 5153 for i := 0; i < 5; i++ { 5154 alloc := mock.Alloc() 5155 allocs = append(allocs, alloc) 5156 } 5157 for i, alloc := range allocs { 5158 state.UpsertJobSummary(uint64(900+i), mock.JobSummary(alloc.JobID)) 5159 } 5160 // Set some previous alloc ids 5161 allocs[1].PreviousAllocation = allocs[0].ID 5162 allocs[2].PreviousAllocation = allocs[1].ID 5163 5164 err := state.UpsertAllocs(1000, allocs) 5165 require.Nil(err) 5166 5167 ws := memdb.NewWatchSet() 5168 iter, err := state.Allocs(ws) 5169 require.Nil(err) 5170 5171 var out []*structs.Allocation 5172 for { 5173 raw := iter.Next() 5174 if raw == nil { 5175 break 5176 } 5177 out = append(out, raw.(*structs.Allocation)) 5178 } 5179 5180 // Set expected NextAllocation fields 5181 allocs[0].NextAllocation = allocs[1].ID 5182 allocs[1].NextAllocation = allocs[2].ID 5183 5184 sort.Sort(AllocIDSort(allocs)) 5185 sort.Sort(AllocIDSort(out)) 5186 5187 require.Equal(allocs, out) 5188 require.False(watchFired(ws)) 5189 5190 // Insert another alloc, verify index of previous alloc also got updated 5191 alloc := mock.Alloc() 5192 alloc.PreviousAllocation = allocs[0].ID 5193 err = state.UpsertAllocs(1001, []*structs.Allocation{alloc}) 5194 require.Nil(err) 5195 alloc0, err := state.AllocByID(nil, allocs[0].ID) 5196 require.Nil(err) 5197 require.Equal(alloc0.ModifyIndex, uint64(1001)) 5198 } 5199 5200 func TestStateStore_RestoreAlloc(t *testing.T) { 5201 t.Parallel() 5202 5203 state := testStateStore(t) 5204 alloc := mock.Alloc() 5205 5206 restore, err := state.Restore() 5207 if err != nil { 5208 t.Fatalf("err: %v", err) 5209 } 5210 5211 err = restore.AllocRestore(alloc) 5212 if err != nil { 5213 t.Fatalf("err: %v", err) 5214 } 5215 5216 restore.Commit() 5217 5218 ws := memdb.NewWatchSet() 5219 out, err := state.AllocByID(ws, alloc.ID) 5220 if err != nil { 5221 t.Fatalf("err: %v", err) 5222 } 5223 5224 if !reflect.DeepEqual(out, alloc) { 5225 t.Fatalf("Bad: %#v %#v", out, alloc) 5226 } 5227 5228 if watchFired(ws) { 5229 t.Fatalf("bad") 5230 } 5231 } 5232 5233 func TestStateStore_SetJobStatus_ForceStatus(t *testing.T) { 5234 t.Parallel() 5235 5236 state := testStateStore(t) 5237 txn := state.db.Txn(true) 5238 5239 // Create and insert a mock job. 5240 job := mock.Job() 5241 job.Status = "" 5242 job.ModifyIndex = 0 5243 if err := txn.Insert("jobs", job); err != nil { 5244 t.Fatalf("job insert failed: %v", err) 5245 } 5246 5247 exp := "foobar" 5248 index := uint64(1000) 5249 if err := state.setJobStatus(index, txn, job, false, exp); err != nil { 5250 t.Fatalf("setJobStatus() failed: %v", err) 5251 } 5252 5253 i, err := txn.First("jobs", "id", job.Namespace, job.ID) 5254 if err != nil { 5255 t.Fatalf("job lookup failed: %v", err) 5256 } 5257 updated := i.(*structs.Job) 5258 5259 if updated.Status != exp { 5260 t.Fatalf("setJobStatus() set %v; expected %v", updated.Status, exp) 5261 } 5262 5263 if updated.ModifyIndex != index { 5264 t.Fatalf("setJobStatus() set %d; expected %d", updated.ModifyIndex, index) 5265 } 5266 } 5267 5268 func TestStateStore_SetJobStatus_NoOp(t *testing.T) { 5269 t.Parallel() 5270 5271 state := testStateStore(t) 5272 txn := state.db.Txn(true) 5273 5274 // Create and insert a mock job that should be pending. 5275 job := mock.Job() 5276 job.Status = structs.JobStatusPending 5277 job.ModifyIndex = 10 5278 if err := txn.Insert("jobs", job); err != nil { 5279 t.Fatalf("job insert failed: %v", err) 5280 } 5281 5282 index := uint64(1000) 5283 if err := state.setJobStatus(index, txn, job, false, ""); err != nil { 5284 t.Fatalf("setJobStatus() failed: %v", err) 5285 } 5286 5287 i, err := txn.First("jobs", "id", job.Namespace, job.ID) 5288 if err != nil { 5289 t.Fatalf("job lookup failed: %v", err) 5290 } 5291 updated := i.(*structs.Job) 5292 5293 if updated.ModifyIndex == index { 5294 t.Fatalf("setJobStatus() should have been a no-op") 5295 } 5296 } 5297 5298 func TestStateStore_SetJobStatus(t *testing.T) { 5299 t.Parallel() 5300 5301 state := testStateStore(t) 5302 txn := state.db.Txn(true) 5303 5304 // Create and insert a mock job that should be pending but has an incorrect 5305 // status. 5306 job := mock.Job() 5307 job.Status = "foobar" 5308 job.ModifyIndex = 10 5309 if err := txn.Insert("jobs", job); err != nil { 5310 t.Fatalf("job insert failed: %v", err) 5311 } 5312 5313 index := uint64(1000) 5314 if err := state.setJobStatus(index, txn, job, false, ""); err != nil { 5315 t.Fatalf("setJobStatus() failed: %v", err) 5316 } 5317 5318 i, err := txn.First("jobs", "id", job.Namespace, job.ID) 5319 if err != nil { 5320 t.Fatalf("job lookup failed: %v", err) 5321 } 5322 updated := i.(*structs.Job) 5323 5324 if updated.Status != structs.JobStatusPending { 5325 t.Fatalf("setJobStatus() set %v; expected %v", updated.Status, structs.JobStatusPending) 5326 } 5327 5328 if updated.ModifyIndex != index { 5329 t.Fatalf("setJobStatus() set %d; expected %d", updated.ModifyIndex, index) 5330 } 5331 } 5332 5333 func TestStateStore_GetJobStatus_NoEvalsOrAllocs(t *testing.T) { 5334 t.Parallel() 5335 5336 job := mock.Job() 5337 state := testStateStore(t) 5338 txn := state.db.Txn(false) 5339 status, err := state.getJobStatus(txn, job, false) 5340 if err != nil { 5341 t.Fatalf("getJobStatus() failed: %v", err) 5342 } 5343 5344 if status != structs.JobStatusPending { 5345 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusPending) 5346 } 5347 } 5348 5349 func TestStateStore_GetJobStatus_NoEvalsOrAllocs_Periodic(t *testing.T) { 5350 t.Parallel() 5351 5352 job := mock.PeriodicJob() 5353 state := testStateStore(t) 5354 txn := state.db.Txn(false) 5355 status, err := state.getJobStatus(txn, job, false) 5356 if err != nil { 5357 t.Fatalf("getJobStatus() failed: %v", err) 5358 } 5359 5360 if status != structs.JobStatusRunning { 5361 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusRunning) 5362 } 5363 } 5364 5365 func TestStateStore_GetJobStatus_NoEvalsOrAllocs_EvalDelete(t *testing.T) { 5366 t.Parallel() 5367 5368 job := mock.Job() 5369 state := testStateStore(t) 5370 txn := state.db.Txn(false) 5371 status, err := state.getJobStatus(txn, job, true) 5372 if err != nil { 5373 t.Fatalf("getJobStatus() failed: %v", err) 5374 } 5375 5376 if status != structs.JobStatusDead { 5377 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusDead) 5378 } 5379 } 5380 5381 func TestStateStore_GetJobStatus_DeadEvalsAndAllocs(t *testing.T) { 5382 t.Parallel() 5383 5384 state := testStateStore(t) 5385 job := mock.Job() 5386 5387 // Create a mock alloc that is dead. 5388 alloc := mock.Alloc() 5389 alloc.JobID = job.ID 5390 alloc.DesiredStatus = structs.AllocDesiredStatusStop 5391 state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)) 5392 if err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}); err != nil { 5393 t.Fatalf("err: %v", err) 5394 } 5395 5396 // Create a mock eval that is complete 5397 eval := mock.Eval() 5398 eval.JobID = job.ID 5399 eval.Status = structs.EvalStatusComplete 5400 if err := state.UpsertEvals(1001, []*structs.Evaluation{eval}); err != nil { 5401 t.Fatalf("err: %v", err) 5402 } 5403 5404 txn := state.db.Txn(false) 5405 status, err := state.getJobStatus(txn, job, false) 5406 if err != nil { 5407 t.Fatalf("getJobStatus() failed: %v", err) 5408 } 5409 5410 if status != structs.JobStatusDead { 5411 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusDead) 5412 } 5413 } 5414 5415 func TestStateStore_GetJobStatus_RunningAlloc(t *testing.T) { 5416 t.Parallel() 5417 5418 state := testStateStore(t) 5419 job := mock.Job() 5420 5421 // Create a mock alloc that is running. 5422 alloc := mock.Alloc() 5423 alloc.JobID = job.ID 5424 alloc.DesiredStatus = structs.AllocDesiredStatusRun 5425 state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)) 5426 if err := state.UpsertAllocs(1000, []*structs.Allocation{alloc}); err != nil { 5427 t.Fatalf("err: %v", err) 5428 } 5429 5430 txn := state.db.Txn(false) 5431 status, err := state.getJobStatus(txn, job, true) 5432 if err != nil { 5433 t.Fatalf("getJobStatus() failed: %v", err) 5434 } 5435 5436 if status != structs.JobStatusRunning { 5437 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusRunning) 5438 } 5439 } 5440 5441 func TestStateStore_GetJobStatus_PeriodicJob(t *testing.T) { 5442 t.Parallel() 5443 5444 state := testStateStore(t) 5445 job := mock.PeriodicJob() 5446 5447 txn := state.db.Txn(false) 5448 status, err := state.getJobStatus(txn, job, false) 5449 if err != nil { 5450 t.Fatalf("getJobStatus() failed: %v", err) 5451 } 5452 5453 if status != structs.JobStatusRunning { 5454 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusRunning) 5455 } 5456 5457 // Mark it as stopped 5458 job.Stop = true 5459 status, err = state.getJobStatus(txn, job, false) 5460 if err != nil { 5461 t.Fatalf("getJobStatus() failed: %v", err) 5462 } 5463 5464 if status != structs.JobStatusDead { 5465 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusDead) 5466 } 5467 } 5468 5469 func TestStateStore_GetJobStatus_ParameterizedJob(t *testing.T) { 5470 t.Parallel() 5471 5472 state := testStateStore(t) 5473 job := mock.Job() 5474 job.ParameterizedJob = &structs.ParameterizedJobConfig{} 5475 5476 txn := state.db.Txn(false) 5477 status, err := state.getJobStatus(txn, job, false) 5478 if err != nil { 5479 t.Fatalf("getJobStatus() failed: %v", err) 5480 } 5481 5482 if status != structs.JobStatusRunning { 5483 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusRunning) 5484 } 5485 5486 // Mark it as stopped 5487 job.Stop = true 5488 status, err = state.getJobStatus(txn, job, false) 5489 if err != nil { 5490 t.Fatalf("getJobStatus() failed: %v", err) 5491 } 5492 5493 if status != structs.JobStatusDead { 5494 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusDead) 5495 } 5496 } 5497 5498 func TestStateStore_SetJobStatus_PendingEval(t *testing.T) { 5499 t.Parallel() 5500 5501 state := testStateStore(t) 5502 job := mock.Job() 5503 5504 // Create a mock eval that is pending. 5505 eval := mock.Eval() 5506 eval.JobID = job.ID 5507 eval.Status = structs.EvalStatusPending 5508 if err := state.UpsertEvals(1000, []*structs.Evaluation{eval}); err != nil { 5509 t.Fatalf("err: %v", err) 5510 } 5511 5512 txn := state.db.Txn(false) 5513 status, err := state.getJobStatus(txn, job, true) 5514 if err != nil { 5515 t.Fatalf("getJobStatus() failed: %v", err) 5516 } 5517 5518 if status != structs.JobStatusPending { 5519 t.Fatalf("getJobStatus() returned %v; expected %v", status, structs.JobStatusPending) 5520 } 5521 } 5522 5523 // TestStateStore_SetJobStatus_SystemJob asserts that system jobs are still 5524 // considered running until explicitly stopped. 5525 func TestStateStore_SetJobStatus_SystemJob(t *testing.T) { 5526 t.Parallel() 5527 5528 state := testStateStore(t) 5529 job := mock.SystemJob() 5530 5531 // Create a mock eval that is pending. 5532 eval := mock.Eval() 5533 eval.JobID = job.ID 5534 eval.Type = job.Type 5535 eval.Status = structs.EvalStatusComplete 5536 if err := state.UpsertEvals(1000, []*structs.Evaluation{eval}); err != nil { 5537 t.Fatalf("err: %v", err) 5538 } 5539 5540 txn := state.db.Txn(false) 5541 status, err := state.getJobStatus(txn, job, true) 5542 if err != nil { 5543 t.Fatalf("getJobStatus() failed: %v", err) 5544 } 5545 5546 if expected := structs.JobStatusRunning; status != expected { 5547 t.Fatalf("getJobStatus() returned %v; expected %v", status, expected) 5548 } 5549 5550 // Stop the job 5551 job.Stop = true 5552 status, err = state.getJobStatus(txn, job, true) 5553 if err != nil { 5554 t.Fatalf("getJobStatus() failed: %v", err) 5555 } 5556 5557 if expected := structs.JobStatusDead; status != expected { 5558 t.Fatalf("getJobStatus() returned %v; expected %v", status, expected) 5559 } 5560 } 5561 5562 func TestStateJobSummary_UpdateJobCount(t *testing.T) { 5563 t.Parallel() 5564 5565 state := testStateStore(t) 5566 alloc := mock.Alloc() 5567 job := alloc.Job 5568 job.TaskGroups[0].Count = 3 5569 5570 // Create watchsets so we can test that upsert fires the watch 5571 ws := memdb.NewWatchSet() 5572 if _, err := state.JobSummaryByID(ws, job.Namespace, job.ID); err != nil { 5573 t.Fatalf("bad: %v", err) 5574 } 5575 5576 if err := state.UpsertJob(1000, job); err != nil { 5577 t.Fatalf("err: %v", err) 5578 } 5579 5580 if err := state.UpsertAllocs(1001, []*structs.Allocation{alloc}); err != nil { 5581 t.Fatalf("err: %v", err) 5582 } 5583 5584 if !watchFired(ws) { 5585 t.Fatalf("bad") 5586 } 5587 5588 ws = memdb.NewWatchSet() 5589 summary, _ := state.JobSummaryByID(ws, job.Namespace, job.ID) 5590 expectedSummary := structs.JobSummary{ 5591 JobID: job.ID, 5592 Namespace: job.Namespace, 5593 Summary: map[string]structs.TaskGroupSummary{ 5594 "web": { 5595 Starting: 1, 5596 }, 5597 }, 5598 Children: new(structs.JobChildrenSummary), 5599 CreateIndex: 1000, 5600 ModifyIndex: 1001, 5601 } 5602 if !reflect.DeepEqual(summary, &expectedSummary) { 5603 t.Fatalf("expected: %v, actual: %v", expectedSummary, summary) 5604 } 5605 5606 // Create watchsets so we can test that upsert fires the watch 5607 ws2 := memdb.NewWatchSet() 5608 if _, err := state.JobSummaryByID(ws2, job.Namespace, job.ID); err != nil { 5609 t.Fatalf("bad: %v", err) 5610 } 5611 5612 alloc2 := mock.Alloc() 5613 alloc2.Job = job 5614 alloc2.JobID = job.ID 5615 5616 alloc3 := mock.Alloc() 5617 alloc3.Job = job 5618 alloc3.JobID = job.ID 5619 5620 if err := state.UpsertAllocs(1002, []*structs.Allocation{alloc2, alloc3}); err != nil { 5621 t.Fatalf("err: %v", err) 5622 } 5623 5624 if !watchFired(ws2) { 5625 t.Fatalf("bad") 5626 } 5627 5628 outA, _ := state.AllocByID(ws, alloc3.ID) 5629 5630 summary, _ = state.JobSummaryByID(ws, job.Namespace, job.ID) 5631 expectedSummary = structs.JobSummary{ 5632 JobID: job.ID, 5633 Namespace: job.Namespace, 5634 Summary: map[string]structs.TaskGroupSummary{ 5635 "web": { 5636 Starting: 3, 5637 }, 5638 }, 5639 Children: new(structs.JobChildrenSummary), 5640 CreateIndex: job.CreateIndex, 5641 ModifyIndex: outA.ModifyIndex, 5642 } 5643 if !reflect.DeepEqual(summary, &expectedSummary) { 5644 t.Fatalf("expected summary: %v, actual: %v", expectedSummary, summary) 5645 } 5646 5647 // Create watchsets so we can test that upsert fires the watch 5648 ws3 := memdb.NewWatchSet() 5649 if _, err := state.JobSummaryByID(ws3, job.Namespace, job.ID); err != nil { 5650 t.Fatalf("bad: %v", err) 5651 } 5652 5653 alloc4 := mock.Alloc() 5654 alloc4.ID = alloc2.ID 5655 alloc4.Job = alloc2.Job 5656 alloc4.JobID = alloc2.JobID 5657 alloc4.ClientStatus = structs.AllocClientStatusComplete 5658 5659 alloc5 := mock.Alloc() 5660 alloc5.ID = alloc3.ID 5661 alloc5.Job = alloc3.Job 5662 alloc5.JobID = alloc3.JobID 5663 alloc5.ClientStatus = structs.AllocClientStatusComplete 5664 5665 if err := state.UpdateAllocsFromClient(1004, []*structs.Allocation{alloc4, alloc5}); err != nil { 5666 t.Fatalf("err: %v", err) 5667 } 5668 5669 if !watchFired(ws2) { 5670 t.Fatalf("bad") 5671 } 5672 5673 outA, _ = state.AllocByID(ws, alloc5.ID) 5674 summary, _ = state.JobSummaryByID(ws, job.Namespace, job.ID) 5675 expectedSummary = structs.JobSummary{ 5676 JobID: job.ID, 5677 Namespace: job.Namespace, 5678 Summary: map[string]structs.TaskGroupSummary{ 5679 "web": { 5680 Complete: 2, 5681 Starting: 1, 5682 }, 5683 }, 5684 Children: new(structs.JobChildrenSummary), 5685 CreateIndex: job.CreateIndex, 5686 ModifyIndex: outA.ModifyIndex, 5687 } 5688 if !reflect.DeepEqual(summary, &expectedSummary) { 5689 t.Fatalf("expected: %v, actual: %v", expectedSummary, summary) 5690 } 5691 } 5692 5693 func TestJobSummary_UpdateClientStatus(t *testing.T) { 5694 t.Parallel() 5695 5696 state := testStateStore(t) 5697 alloc := mock.Alloc() 5698 job := alloc.Job 5699 job.TaskGroups[0].Count = 3 5700 5701 alloc2 := mock.Alloc() 5702 alloc2.Job = job 5703 alloc2.JobID = job.ID 5704 5705 alloc3 := mock.Alloc() 5706 alloc3.Job = job 5707 alloc3.JobID = job.ID 5708 5709 err := state.UpsertJob(1000, job) 5710 if err != nil { 5711 t.Fatalf("err: %v", err) 5712 } 5713 5714 if err := state.UpsertAllocs(1001, []*structs.Allocation{alloc, alloc2, alloc3}); err != nil { 5715 t.Fatalf("err: %v", err) 5716 } 5717 5718 ws := memdb.NewWatchSet() 5719 summary, _ := state.JobSummaryByID(ws, job.Namespace, job.ID) 5720 if summary.Summary["web"].Starting != 3 { 5721 t.Fatalf("bad job summary: %v", summary) 5722 } 5723 5724 alloc4 := mock.Alloc() 5725 alloc4.ID = alloc2.ID 5726 alloc4.Job = alloc2.Job 5727 alloc4.JobID = alloc2.JobID 5728 alloc4.ClientStatus = structs.AllocClientStatusComplete 5729 5730 alloc5 := mock.Alloc() 5731 alloc5.ID = alloc3.ID 5732 alloc5.Job = alloc3.Job 5733 alloc5.JobID = alloc3.JobID 5734 alloc5.ClientStatus = structs.AllocClientStatusFailed 5735 5736 alloc6 := mock.Alloc() 5737 alloc6.ID = alloc.ID 5738 alloc6.Job = alloc.Job 5739 alloc6.JobID = alloc.JobID 5740 alloc6.ClientStatus = structs.AllocClientStatusRunning 5741 5742 if err := state.UpdateAllocsFromClient(1002, []*structs.Allocation{alloc4, alloc5, alloc6}); err != nil { 5743 t.Fatalf("err: %v", err) 5744 } 5745 5746 if !watchFired(ws) { 5747 t.Fatalf("bad") 5748 } 5749 5750 summary, _ = state.JobSummaryByID(ws, job.Namespace, job.ID) 5751 if summary.Summary["web"].Running != 1 || summary.Summary["web"].Failed != 1 || summary.Summary["web"].Complete != 1 { 5752 t.Fatalf("bad job summary: %v", summary) 5753 } 5754 5755 alloc7 := mock.Alloc() 5756 alloc7.Job = alloc.Job 5757 alloc7.JobID = alloc.JobID 5758 5759 if err := state.UpsertAllocs(1003, []*structs.Allocation{alloc7}); err != nil { 5760 t.Fatalf("err: %v", err) 5761 } 5762 summary, _ = state.JobSummaryByID(ws, job.Namespace, job.ID) 5763 if summary.Summary["web"].Starting != 1 || summary.Summary["web"].Running != 1 || summary.Summary["web"].Failed != 1 || summary.Summary["web"].Complete != 1 { 5764 t.Fatalf("bad job summary: %v", summary) 5765 } 5766 } 5767 5768 // Test that nonexistent deployment can't be updated 5769 func TestStateStore_UpsertDeploymentStatusUpdate_Nonexistent(t *testing.T) { 5770 t.Parallel() 5771 5772 state := testStateStore(t) 5773 5774 // Update the nonexistent deployment 5775 req := &structs.DeploymentStatusUpdateRequest{ 5776 DeploymentUpdate: &structs.DeploymentStatusUpdate{ 5777 DeploymentID: uuid.Generate(), 5778 Status: structs.DeploymentStatusRunning, 5779 }, 5780 } 5781 err := state.UpdateDeploymentStatus(2, req) 5782 if err == nil || !strings.Contains(err.Error(), "does not exist") { 5783 t.Fatalf("expected error updating the status because the deployment doesn't exist") 5784 } 5785 } 5786 5787 // Test that terminal deployment can't be updated 5788 func TestStateStore_UpsertDeploymentStatusUpdate_Terminal(t *testing.T) { 5789 t.Parallel() 5790 5791 state := testStateStore(t) 5792 5793 // Insert a terminal deployment 5794 d := mock.Deployment() 5795 d.Status = structs.DeploymentStatusFailed 5796 5797 if err := state.UpsertDeployment(1, d); err != nil { 5798 t.Fatalf("bad: %v", err) 5799 } 5800 5801 // Update the deployment 5802 req := &structs.DeploymentStatusUpdateRequest{ 5803 DeploymentUpdate: &structs.DeploymentStatusUpdate{ 5804 DeploymentID: d.ID, 5805 Status: structs.DeploymentStatusRunning, 5806 }, 5807 } 5808 err := state.UpdateDeploymentStatus(2, req) 5809 if err == nil || !strings.Contains(err.Error(), "has terminal status") { 5810 t.Fatalf("expected error updating the status because the deployment is terminal") 5811 } 5812 } 5813 5814 // Test that a non terminal deployment is updated and that a job and eval are 5815 // created. 5816 func TestStateStore_UpsertDeploymentStatusUpdate_NonTerminal(t *testing.T) { 5817 t.Parallel() 5818 5819 state := testStateStore(t) 5820 5821 // Insert a deployment 5822 d := mock.Deployment() 5823 if err := state.UpsertDeployment(1, d); err != nil { 5824 t.Fatalf("bad: %v", err) 5825 } 5826 5827 // Create an eval and a job 5828 e := mock.Eval() 5829 j := mock.Job() 5830 5831 // Update the deployment 5832 status, desc := structs.DeploymentStatusFailed, "foo" 5833 req := &structs.DeploymentStatusUpdateRequest{ 5834 DeploymentUpdate: &structs.DeploymentStatusUpdate{ 5835 DeploymentID: d.ID, 5836 Status: status, 5837 StatusDescription: desc, 5838 }, 5839 Job: j, 5840 Eval: e, 5841 } 5842 err := state.UpdateDeploymentStatus(2, req) 5843 if err != nil { 5844 t.Fatalf("bad: %v", err) 5845 } 5846 5847 // Check that the status was updated properly 5848 ws := memdb.NewWatchSet() 5849 dout, err := state.DeploymentByID(ws, d.ID) 5850 if err != nil { 5851 t.Fatalf("bad: %v", err) 5852 } 5853 if dout.Status != status || dout.StatusDescription != desc { 5854 t.Fatalf("bad: %#v", dout) 5855 } 5856 5857 // Check that the evaluation was created 5858 eout, _ := state.EvalByID(ws, e.ID) 5859 if err != nil { 5860 t.Fatalf("bad: %v", err) 5861 } 5862 if eout == nil { 5863 t.Fatalf("bad: %#v", eout) 5864 } 5865 5866 // Check that the job was created 5867 jout, _ := state.JobByID(ws, j.Namespace, j.ID) 5868 if err != nil { 5869 t.Fatalf("bad: %v", err) 5870 } 5871 if jout == nil { 5872 t.Fatalf("bad: %#v", jout) 5873 } 5874 } 5875 5876 // Test that when a deployment is updated to successful the job is updated to 5877 // stable 5878 func TestStateStore_UpsertDeploymentStatusUpdate_Successful(t *testing.T) { 5879 t.Parallel() 5880 5881 state := testStateStore(t) 5882 5883 // Insert a job 5884 job := mock.Job() 5885 if err := state.UpsertJob(1, job); err != nil { 5886 t.Fatalf("bad: %v", err) 5887 } 5888 5889 // Insert a deployment 5890 d := structs.NewDeployment(job) 5891 if err := state.UpsertDeployment(2, d); err != nil { 5892 t.Fatalf("bad: %v", err) 5893 } 5894 5895 // Update the deployment 5896 req := &structs.DeploymentStatusUpdateRequest{ 5897 DeploymentUpdate: &structs.DeploymentStatusUpdate{ 5898 DeploymentID: d.ID, 5899 Status: structs.DeploymentStatusSuccessful, 5900 StatusDescription: structs.DeploymentStatusDescriptionSuccessful, 5901 }, 5902 } 5903 err := state.UpdateDeploymentStatus(3, req) 5904 if err != nil { 5905 t.Fatalf("bad: %v", err) 5906 } 5907 5908 // Check that the status was updated properly 5909 ws := memdb.NewWatchSet() 5910 dout, err := state.DeploymentByID(ws, d.ID) 5911 if err != nil { 5912 t.Fatalf("bad: %v", err) 5913 } 5914 if dout.Status != structs.DeploymentStatusSuccessful || 5915 dout.StatusDescription != structs.DeploymentStatusDescriptionSuccessful { 5916 t.Fatalf("bad: %#v", dout) 5917 } 5918 5919 // Check that the job was created 5920 jout, _ := state.JobByID(ws, job.Namespace, job.ID) 5921 if err != nil { 5922 t.Fatalf("bad: %v", err) 5923 } 5924 if jout == nil { 5925 t.Fatalf("bad: %#v", jout) 5926 } 5927 if !jout.Stable { 5928 t.Fatalf("job not marked stable %#v", jout) 5929 } 5930 if jout.Version != d.JobVersion { 5931 t.Fatalf("job version changed; got %d; want %d", jout.Version, d.JobVersion) 5932 } 5933 } 5934 5935 func TestStateStore_UpdateJobStability(t *testing.T) { 5936 t.Parallel() 5937 5938 state := testStateStore(t) 5939 5940 // Insert a job twice to get two versions 5941 job := mock.Job() 5942 if err := state.UpsertJob(1, job); err != nil { 5943 t.Fatalf("bad: %v", err) 5944 } 5945 5946 if err := state.UpsertJob(2, job); err != nil { 5947 t.Fatalf("bad: %v", err) 5948 } 5949 5950 // Update the stability to true 5951 err := state.UpdateJobStability(3, job.Namespace, job.ID, 0, true) 5952 if err != nil { 5953 t.Fatalf("bad: %v", err) 5954 } 5955 5956 // Check that the job was updated properly 5957 ws := memdb.NewWatchSet() 5958 jout, _ := state.JobByIDAndVersion(ws, job.Namespace, job.ID, 0) 5959 if err != nil { 5960 t.Fatalf("bad: %v", err) 5961 } 5962 if jout == nil { 5963 t.Fatalf("bad: %#v", jout) 5964 } 5965 if !jout.Stable { 5966 t.Fatalf("job not marked stable %#v", jout) 5967 } 5968 5969 // Update the stability to false 5970 err = state.UpdateJobStability(3, job.Namespace, job.ID, 0, false) 5971 if err != nil { 5972 t.Fatalf("bad: %v", err) 5973 } 5974 5975 // Check that the job was updated properly 5976 jout, _ = state.JobByIDAndVersion(ws, job.Namespace, job.ID, 0) 5977 if err != nil { 5978 t.Fatalf("bad: %v", err) 5979 } 5980 if jout == nil { 5981 t.Fatalf("bad: %#v", jout) 5982 } 5983 if jout.Stable { 5984 t.Fatalf("job marked stable %#v", jout) 5985 } 5986 } 5987 5988 // Test that nonexistent deployment can't be promoted 5989 func TestStateStore_UpsertDeploymentPromotion_Nonexistent(t *testing.T) { 5990 t.Parallel() 5991 5992 state := testStateStore(t) 5993 5994 // Promote the nonexistent deployment 5995 req := &structs.ApplyDeploymentPromoteRequest{ 5996 DeploymentPromoteRequest: structs.DeploymentPromoteRequest{ 5997 DeploymentID: uuid.Generate(), 5998 All: true, 5999 }, 6000 } 6001 err := state.UpdateDeploymentPromotion(2, req) 6002 if err == nil || !strings.Contains(err.Error(), "does not exist") { 6003 t.Fatalf("expected error promoting because the deployment doesn't exist") 6004 } 6005 } 6006 6007 // Test that terminal deployment can't be updated 6008 func TestStateStore_UpsertDeploymentPromotion_Terminal(t *testing.T) { 6009 t.Parallel() 6010 6011 state := testStateStore(t) 6012 6013 // Insert a terminal deployment 6014 d := mock.Deployment() 6015 d.Status = structs.DeploymentStatusFailed 6016 6017 if err := state.UpsertDeployment(1, d); err != nil { 6018 t.Fatalf("bad: %v", err) 6019 } 6020 6021 // Promote the deployment 6022 req := &structs.ApplyDeploymentPromoteRequest{ 6023 DeploymentPromoteRequest: structs.DeploymentPromoteRequest{ 6024 DeploymentID: d.ID, 6025 All: true, 6026 }, 6027 } 6028 err := state.UpdateDeploymentPromotion(2, req) 6029 if err == nil || !strings.Contains(err.Error(), "has terminal status") { 6030 t.Fatalf("expected error updating the status because the deployment is terminal: %v", err) 6031 } 6032 } 6033 6034 // Test promoting unhealthy canaries in a deployment. 6035 func TestStateStore_UpsertDeploymentPromotion_Unhealthy(t *testing.T) { 6036 t.Parallel() 6037 6038 state := testStateStore(t) 6039 require := require.New(t) 6040 6041 // Create a job 6042 j := mock.Job() 6043 require.Nil(state.UpsertJob(1, j)) 6044 6045 // Create a deployment 6046 d := mock.Deployment() 6047 d.JobID = j.ID 6048 d.TaskGroups["web"].DesiredCanaries = 2 6049 require.Nil(state.UpsertDeployment(2, d)) 6050 6051 // Create a set of allocations 6052 c1 := mock.Alloc() 6053 c1.JobID = j.ID 6054 c1.DeploymentID = d.ID 6055 d.TaskGroups[c1.TaskGroup].PlacedCanaries = append(d.TaskGroups[c1.TaskGroup].PlacedCanaries, c1.ID) 6056 c2 := mock.Alloc() 6057 c2.JobID = j.ID 6058 c2.DeploymentID = d.ID 6059 d.TaskGroups[c2.TaskGroup].PlacedCanaries = append(d.TaskGroups[c2.TaskGroup].PlacedCanaries, c2.ID) 6060 6061 // Create a healthy but terminal alloc 6062 c3 := mock.Alloc() 6063 c3.JobID = j.ID 6064 c3.DeploymentID = d.ID 6065 c3.DesiredStatus = structs.AllocDesiredStatusStop 6066 c3.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} 6067 d.TaskGroups[c3.TaskGroup].PlacedCanaries = append(d.TaskGroups[c3.TaskGroup].PlacedCanaries, c3.ID) 6068 6069 require.Nil(state.UpsertAllocs(3, []*structs.Allocation{c1, c2, c3})) 6070 6071 // Promote the canaries 6072 req := &structs.ApplyDeploymentPromoteRequest{ 6073 DeploymentPromoteRequest: structs.DeploymentPromoteRequest{ 6074 DeploymentID: d.ID, 6075 All: true, 6076 }, 6077 } 6078 err := state.UpdateDeploymentPromotion(4, req) 6079 require.NotNil(err) 6080 require.Contains(err.Error(), `Task group "web" has 0/2 healthy allocations`) 6081 } 6082 6083 // Test promoting a deployment with no canaries 6084 func TestStateStore_UpsertDeploymentPromotion_NoCanaries(t *testing.T) { 6085 t.Parallel() 6086 6087 state := testStateStore(t) 6088 require := require.New(t) 6089 6090 // Create a job 6091 j := mock.Job() 6092 require.Nil(state.UpsertJob(1, j)) 6093 6094 // Create a deployment 6095 d := mock.Deployment() 6096 d.TaskGroups["web"].DesiredCanaries = 2 6097 d.JobID = j.ID 6098 require.Nil(state.UpsertDeployment(2, d)) 6099 6100 // Promote the canaries 6101 req := &structs.ApplyDeploymentPromoteRequest{ 6102 DeploymentPromoteRequest: structs.DeploymentPromoteRequest{ 6103 DeploymentID: d.ID, 6104 All: true, 6105 }, 6106 } 6107 err := state.UpdateDeploymentPromotion(4, req) 6108 require.NotNil(err) 6109 require.Contains(err.Error(), `Task group "web" has 0/2 healthy allocations`) 6110 } 6111 6112 // Test promoting all canaries in a deployment. 6113 func TestStateStore_UpsertDeploymentPromotion_All(t *testing.T) { 6114 t.Parallel() 6115 6116 state := testStateStore(t) 6117 6118 // Create a job with two task groups 6119 j := mock.Job() 6120 tg1 := j.TaskGroups[0] 6121 tg2 := tg1.Copy() 6122 tg2.Name = "foo" 6123 j.TaskGroups = append(j.TaskGroups, tg2) 6124 if err := state.UpsertJob(1, j); err != nil { 6125 t.Fatalf("bad: %v", err) 6126 } 6127 6128 // Create a deployment 6129 d := mock.Deployment() 6130 d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion 6131 d.JobID = j.ID 6132 d.TaskGroups = map[string]*structs.DeploymentState{ 6133 "web": { 6134 DesiredTotal: 10, 6135 DesiredCanaries: 1, 6136 }, 6137 "foo": { 6138 DesiredTotal: 10, 6139 DesiredCanaries: 1, 6140 }, 6141 } 6142 if err := state.UpsertDeployment(2, d); err != nil { 6143 t.Fatalf("bad: %v", err) 6144 } 6145 6146 // Create a set of allocations 6147 c1 := mock.Alloc() 6148 c1.JobID = j.ID 6149 c1.DeploymentID = d.ID 6150 d.TaskGroups[c1.TaskGroup].PlacedCanaries = append(d.TaskGroups[c1.TaskGroup].PlacedCanaries, c1.ID) 6151 c1.DeploymentStatus = &structs.AllocDeploymentStatus{ 6152 Healthy: helper.BoolToPtr(true), 6153 } 6154 c2 := mock.Alloc() 6155 c2.JobID = j.ID 6156 c2.DeploymentID = d.ID 6157 d.TaskGroups[c2.TaskGroup].PlacedCanaries = append(d.TaskGroups[c2.TaskGroup].PlacedCanaries, c2.ID) 6158 c2.TaskGroup = tg2.Name 6159 c2.DeploymentStatus = &structs.AllocDeploymentStatus{ 6160 Healthy: helper.BoolToPtr(true), 6161 } 6162 6163 if err := state.UpsertAllocs(3, []*structs.Allocation{c1, c2}); err != nil { 6164 t.Fatalf("err: %v", err) 6165 } 6166 6167 // Create an eval 6168 e := mock.Eval() 6169 6170 // Promote the canaries 6171 req := &structs.ApplyDeploymentPromoteRequest{ 6172 DeploymentPromoteRequest: structs.DeploymentPromoteRequest{ 6173 DeploymentID: d.ID, 6174 All: true, 6175 }, 6176 Eval: e, 6177 } 6178 err := state.UpdateDeploymentPromotion(4, req) 6179 if err != nil { 6180 t.Fatalf("bad: %v", err) 6181 } 6182 6183 // Check that the status per task group was updated properly 6184 ws := memdb.NewWatchSet() 6185 dout, err := state.DeploymentByID(ws, d.ID) 6186 if err != nil { 6187 t.Fatalf("bad: %v", err) 6188 } 6189 if dout.StatusDescription != structs.DeploymentStatusDescriptionRunning { 6190 t.Fatalf("status description not updated: got %v; want %v", dout.StatusDescription, structs.DeploymentStatusDescriptionRunning) 6191 } 6192 if len(dout.TaskGroups) != 2 { 6193 t.Fatalf("bad: %#v", dout.TaskGroups) 6194 } 6195 for tg, state := range dout.TaskGroups { 6196 if !state.Promoted { 6197 t.Fatalf("bad: group %q not promoted %#v", tg, state) 6198 } 6199 } 6200 6201 // Check that the evaluation was created 6202 eout, _ := state.EvalByID(ws, e.ID) 6203 if err != nil { 6204 t.Fatalf("bad: %v", err) 6205 } 6206 if eout == nil { 6207 t.Fatalf("bad: %#v", eout) 6208 } 6209 } 6210 6211 // Test promoting a subset of canaries in a deployment. 6212 func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) { 6213 t.Parallel() 6214 require := require.New(t) 6215 6216 state := testStateStore(t) 6217 6218 // Create a job with two task groups 6219 j := mock.Job() 6220 tg1 := j.TaskGroups[0] 6221 tg2 := tg1.Copy() 6222 tg2.Name = "foo" 6223 j.TaskGroups = append(j.TaskGroups, tg2) 6224 require.Nil(state.UpsertJob(1, j)) 6225 6226 // Create a deployment 6227 d := mock.Deployment() 6228 d.JobID = j.ID 6229 d.TaskGroups = map[string]*structs.DeploymentState{ 6230 "web": { 6231 DesiredTotal: 10, 6232 DesiredCanaries: 1, 6233 }, 6234 "foo": { 6235 DesiredTotal: 10, 6236 DesiredCanaries: 1, 6237 }, 6238 } 6239 require.Nil(state.UpsertDeployment(2, d)) 6240 6241 // Create a set of allocations for both groups, including an unhealthy one 6242 c1 := mock.Alloc() 6243 c1.JobID = j.ID 6244 c1.DeploymentID = d.ID 6245 d.TaskGroups[c1.TaskGroup].PlacedCanaries = append(d.TaskGroups[c1.TaskGroup].PlacedCanaries, c1.ID) 6246 c1.DeploymentStatus = &structs.AllocDeploymentStatus{ 6247 Healthy: helper.BoolToPtr(true), 6248 Canary: true, 6249 } 6250 6251 // Should still be a canary 6252 c2 := mock.Alloc() 6253 c2.JobID = j.ID 6254 c2.DeploymentID = d.ID 6255 d.TaskGroups[c2.TaskGroup].PlacedCanaries = append(d.TaskGroups[c2.TaskGroup].PlacedCanaries, c2.ID) 6256 c2.TaskGroup = tg2.Name 6257 c2.DeploymentStatus = &structs.AllocDeploymentStatus{ 6258 Healthy: helper.BoolToPtr(true), 6259 Canary: true, 6260 } 6261 6262 c3 := mock.Alloc() 6263 c3.JobID = j.ID 6264 c3.DeploymentID = d.ID 6265 d.TaskGroups[c3.TaskGroup].PlacedCanaries = append(d.TaskGroups[c3.TaskGroup].PlacedCanaries, c3.ID) 6266 c3.DeploymentStatus = &structs.AllocDeploymentStatus{ 6267 Healthy: helper.BoolToPtr(false), 6268 Canary: true, 6269 } 6270 6271 require.Nil(state.UpsertAllocs(3, []*structs.Allocation{c1, c2, c3})) 6272 6273 // Create an eval 6274 e := mock.Eval() 6275 6276 // Promote the canaries 6277 req := &structs.ApplyDeploymentPromoteRequest{ 6278 DeploymentPromoteRequest: structs.DeploymentPromoteRequest{ 6279 DeploymentID: d.ID, 6280 Groups: []string{"web"}, 6281 }, 6282 Eval: e, 6283 } 6284 require.Nil(state.UpdateDeploymentPromotion(4, req)) 6285 6286 // Check that the status per task group was updated properly 6287 ws := memdb.NewWatchSet() 6288 dout, err := state.DeploymentByID(ws, d.ID) 6289 require.Nil(err) 6290 require.Len(dout.TaskGroups, 2) 6291 require.Contains(dout.TaskGroups, "web") 6292 require.True(dout.TaskGroups["web"].Promoted) 6293 6294 // Check that the evaluation was created 6295 eout, err := state.EvalByID(ws, e.ID) 6296 require.Nil(err) 6297 require.NotNil(eout) 6298 6299 // Check the canary field was set properly 6300 aout1, err1 := state.AllocByID(ws, c1.ID) 6301 aout2, err2 := state.AllocByID(ws, c2.ID) 6302 aout3, err3 := state.AllocByID(ws, c3.ID) 6303 require.Nil(err1) 6304 require.Nil(err2) 6305 require.Nil(err3) 6306 require.NotNil(aout1) 6307 require.NotNil(aout2) 6308 require.NotNil(aout3) 6309 require.False(aout1.DeploymentStatus.Canary) 6310 require.True(aout2.DeploymentStatus.Canary) 6311 require.True(aout3.DeploymentStatus.Canary) 6312 } 6313 6314 // Test that allocation health can't be set against a nonexistent deployment 6315 func TestStateStore_UpsertDeploymentAllocHealth_Nonexistent(t *testing.T) { 6316 t.Parallel() 6317 6318 state := testStateStore(t) 6319 6320 // Set health against the nonexistent deployment 6321 req := &structs.ApplyDeploymentAllocHealthRequest{ 6322 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 6323 DeploymentID: uuid.Generate(), 6324 HealthyAllocationIDs: []string{uuid.Generate()}, 6325 }, 6326 } 6327 err := state.UpdateDeploymentAllocHealth(2, req) 6328 if err == nil || !strings.Contains(err.Error(), "does not exist") { 6329 t.Fatalf("expected error because the deployment doesn't exist: %v", err) 6330 } 6331 } 6332 6333 // Test that allocation health can't be set against a terminal deployment 6334 func TestStateStore_UpsertDeploymentAllocHealth_Terminal(t *testing.T) { 6335 t.Parallel() 6336 6337 state := testStateStore(t) 6338 6339 // Insert a terminal deployment 6340 d := mock.Deployment() 6341 d.Status = structs.DeploymentStatusFailed 6342 6343 if err := state.UpsertDeployment(1, d); err != nil { 6344 t.Fatalf("bad: %v", err) 6345 } 6346 6347 // Set health against the terminal deployment 6348 req := &structs.ApplyDeploymentAllocHealthRequest{ 6349 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 6350 DeploymentID: d.ID, 6351 HealthyAllocationIDs: []string{uuid.Generate()}, 6352 }, 6353 } 6354 err := state.UpdateDeploymentAllocHealth(2, req) 6355 if err == nil || !strings.Contains(err.Error(), "has terminal status") { 6356 t.Fatalf("expected error because the deployment is terminal: %v", err) 6357 } 6358 } 6359 6360 // Test that allocation health can't be set against a nonexistent alloc 6361 func TestStateStore_UpsertDeploymentAllocHealth_BadAlloc_Nonexistent(t *testing.T) { 6362 t.Parallel() 6363 6364 state := testStateStore(t) 6365 6366 // Insert a deployment 6367 d := mock.Deployment() 6368 if err := state.UpsertDeployment(1, d); err != nil { 6369 t.Fatalf("bad: %v", err) 6370 } 6371 6372 // Set health against the terminal deployment 6373 req := &structs.ApplyDeploymentAllocHealthRequest{ 6374 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 6375 DeploymentID: d.ID, 6376 HealthyAllocationIDs: []string{uuid.Generate()}, 6377 }, 6378 } 6379 err := state.UpdateDeploymentAllocHealth(2, req) 6380 if err == nil || !strings.Contains(err.Error(), "unknown alloc") { 6381 t.Fatalf("expected error because the alloc doesn't exist: %v", err) 6382 } 6383 } 6384 6385 // Test that allocation health can't be set for an alloc with mismatched 6386 // deployment ids 6387 func TestStateStore_UpsertDeploymentAllocHealth_BadAlloc_MismatchDeployment(t *testing.T) { 6388 t.Parallel() 6389 6390 state := testStateStore(t) 6391 6392 // Insert two deployment 6393 d1 := mock.Deployment() 6394 d2 := mock.Deployment() 6395 if err := state.UpsertDeployment(1, d1); err != nil { 6396 t.Fatalf("bad: %v", err) 6397 } 6398 if err := state.UpsertDeployment(2, d2); err != nil { 6399 t.Fatalf("bad: %v", err) 6400 } 6401 6402 // Insert an alloc for a random deployment 6403 a := mock.Alloc() 6404 a.DeploymentID = d1.ID 6405 if err := state.UpsertAllocs(3, []*structs.Allocation{a}); err != nil { 6406 t.Fatalf("bad: %v", err) 6407 } 6408 6409 // Set health against the terminal deployment 6410 req := &structs.ApplyDeploymentAllocHealthRequest{ 6411 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 6412 DeploymentID: d2.ID, 6413 HealthyAllocationIDs: []string{a.ID}, 6414 }, 6415 } 6416 err := state.UpdateDeploymentAllocHealth(4, req) 6417 if err == nil || !strings.Contains(err.Error(), "not part of deployment") { 6418 t.Fatalf("expected error because the alloc isn't part of the deployment: %v", err) 6419 } 6420 } 6421 6422 // Test that allocation health is properly set 6423 func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) { 6424 t.Parallel() 6425 6426 state := testStateStore(t) 6427 6428 // Insert a deployment 6429 d := mock.Deployment() 6430 d.TaskGroups["web"].ProgressDeadline = 5 * time.Minute 6431 if err := state.UpsertDeployment(1, d); err != nil { 6432 t.Fatalf("bad: %v", err) 6433 } 6434 6435 // Insert two allocations 6436 a1 := mock.Alloc() 6437 a1.DeploymentID = d.ID 6438 a2 := mock.Alloc() 6439 a2.DeploymentID = d.ID 6440 if err := state.UpsertAllocs(2, []*structs.Allocation{a1, a2}); err != nil { 6441 t.Fatalf("bad: %v", err) 6442 } 6443 6444 // Create a job to roll back to 6445 j := mock.Job() 6446 6447 // Create an eval that should be upserted 6448 e := mock.Eval() 6449 6450 // Create a status update for the deployment 6451 status, desc := structs.DeploymentStatusFailed, "foo" 6452 u := &structs.DeploymentStatusUpdate{ 6453 DeploymentID: d.ID, 6454 Status: status, 6455 StatusDescription: desc, 6456 } 6457 6458 // Capture the time for the update 6459 ts := time.Now() 6460 6461 // Set health against the deployment 6462 req := &structs.ApplyDeploymentAllocHealthRequest{ 6463 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 6464 DeploymentID: d.ID, 6465 HealthyAllocationIDs: []string{a1.ID}, 6466 UnhealthyAllocationIDs: []string{a2.ID}, 6467 }, 6468 Job: j, 6469 Eval: e, 6470 DeploymentUpdate: u, 6471 Timestamp: ts, 6472 } 6473 err := state.UpdateDeploymentAllocHealth(3, req) 6474 if err != nil { 6475 t.Fatalf("bad: %v", err) 6476 } 6477 6478 // Check that the status was updated properly 6479 ws := memdb.NewWatchSet() 6480 dout, err := state.DeploymentByID(ws, d.ID) 6481 if err != nil { 6482 t.Fatalf("bad: %v", err) 6483 } 6484 if dout.Status != status || dout.StatusDescription != desc { 6485 t.Fatalf("bad: %#v", dout) 6486 } 6487 6488 // Check that the evaluation was created 6489 eout, _ := state.EvalByID(ws, e.ID) 6490 if err != nil { 6491 t.Fatalf("bad: %v", err) 6492 } 6493 if eout == nil { 6494 t.Fatalf("bad: %#v", eout) 6495 } 6496 6497 // Check that the job was created 6498 jout, _ := state.JobByID(ws, j.Namespace, j.ID) 6499 if err != nil { 6500 t.Fatalf("bad: %v", err) 6501 } 6502 if jout == nil { 6503 t.Fatalf("bad: %#v", jout) 6504 } 6505 6506 // Check the status of the allocs 6507 out1, err := state.AllocByID(ws, a1.ID) 6508 if err != nil { 6509 t.Fatalf("err: %v", err) 6510 } 6511 out2, err := state.AllocByID(ws, a2.ID) 6512 if err != nil { 6513 t.Fatalf("err: %v", err) 6514 } 6515 6516 if !out1.DeploymentStatus.IsHealthy() { 6517 t.Fatalf("bad: alloc %q not healthy", out1.ID) 6518 } 6519 if !out2.DeploymentStatus.IsUnhealthy() { 6520 t.Fatalf("bad: alloc %q not unhealthy", out2.ID) 6521 } 6522 6523 if !out1.DeploymentStatus.Timestamp.Equal(ts) { 6524 t.Fatalf("bad: alloc %q had timestamp %v; want %v", out1.ID, out1.DeploymentStatus.Timestamp, ts) 6525 } 6526 if !out2.DeploymentStatus.Timestamp.Equal(ts) { 6527 t.Fatalf("bad: alloc %q had timestamp %v; want %v", out2.ID, out2.DeploymentStatus.Timestamp, ts) 6528 } 6529 } 6530 6531 func TestStateStore_UpsertVaultAccessors(t *testing.T) { 6532 t.Parallel() 6533 6534 state := testStateStore(t) 6535 a := mock.VaultAccessor() 6536 a2 := mock.VaultAccessor() 6537 6538 ws := memdb.NewWatchSet() 6539 if _, err := state.VaultAccessor(ws, a.Accessor); err != nil { 6540 t.Fatalf("err: %v", err) 6541 } 6542 6543 if _, err := state.VaultAccessor(ws, a2.Accessor); err != nil { 6544 t.Fatalf("err: %v", err) 6545 } 6546 6547 err := state.UpsertVaultAccessor(1000, []*structs.VaultAccessor{a, a2}) 6548 if err != nil { 6549 t.Fatalf("err: %v", err) 6550 } 6551 6552 if !watchFired(ws) { 6553 t.Fatalf("bad") 6554 } 6555 6556 ws = memdb.NewWatchSet() 6557 out, err := state.VaultAccessor(ws, a.Accessor) 6558 if err != nil { 6559 t.Fatalf("err: %v", err) 6560 } 6561 6562 if !reflect.DeepEqual(a, out) { 6563 t.Fatalf("bad: %#v %#v", a, out) 6564 } 6565 6566 out, err = state.VaultAccessor(ws, a2.Accessor) 6567 if err != nil { 6568 t.Fatalf("err: %v", err) 6569 } 6570 6571 if !reflect.DeepEqual(a2, out) { 6572 t.Fatalf("bad: %#v %#v", a2, out) 6573 } 6574 6575 iter, err := state.VaultAccessors(ws) 6576 if err != nil { 6577 t.Fatalf("err: %v", err) 6578 } 6579 6580 count := 0 6581 for { 6582 raw := iter.Next() 6583 if raw == nil { 6584 break 6585 } 6586 6587 count++ 6588 accessor := raw.(*structs.VaultAccessor) 6589 6590 if !reflect.DeepEqual(accessor, a) && !reflect.DeepEqual(accessor, a2) { 6591 t.Fatalf("bad: %#v", accessor) 6592 } 6593 } 6594 6595 if count != 2 { 6596 t.Fatalf("bad: %d", count) 6597 } 6598 6599 index, err := state.Index("vault_accessors") 6600 if err != nil { 6601 t.Fatalf("err: %v", err) 6602 } 6603 if index != 1000 { 6604 t.Fatalf("bad: %d", index) 6605 } 6606 6607 if watchFired(ws) { 6608 t.Fatalf("bad") 6609 } 6610 } 6611 6612 func TestStateStore_DeleteVaultAccessors(t *testing.T) { 6613 t.Parallel() 6614 6615 state := testStateStore(t) 6616 a1 := mock.VaultAccessor() 6617 a2 := mock.VaultAccessor() 6618 accessors := []*structs.VaultAccessor{a1, a2} 6619 6620 err := state.UpsertVaultAccessor(1000, accessors) 6621 if err != nil { 6622 t.Fatalf("err: %v", err) 6623 } 6624 6625 ws := memdb.NewWatchSet() 6626 if _, err := state.VaultAccessor(ws, a1.Accessor); err != nil { 6627 t.Fatalf("err: %v", err) 6628 } 6629 6630 err = state.DeleteVaultAccessors(1001, accessors) 6631 if err != nil { 6632 t.Fatalf("err: %v", err) 6633 } 6634 6635 if !watchFired(ws) { 6636 t.Fatalf("bad") 6637 } 6638 6639 ws = memdb.NewWatchSet() 6640 out, err := state.VaultAccessor(ws, a1.Accessor) 6641 if err != nil { 6642 t.Fatalf("err: %v", err) 6643 } 6644 if out != nil { 6645 t.Fatalf("bad: %#v %#v", a1, out) 6646 } 6647 out, err = state.VaultAccessor(ws, a2.Accessor) 6648 if err != nil { 6649 t.Fatalf("err: %v", err) 6650 } 6651 if out != nil { 6652 t.Fatalf("bad: %#v %#v", a2, out) 6653 } 6654 6655 index, err := state.Index("vault_accessors") 6656 if err != nil { 6657 t.Fatalf("err: %v", err) 6658 } 6659 if index != 1001 { 6660 t.Fatalf("bad: %d", index) 6661 } 6662 6663 if watchFired(ws) { 6664 t.Fatalf("bad") 6665 } 6666 } 6667 6668 func TestStateStore_VaultAccessorsByAlloc(t *testing.T) { 6669 t.Parallel() 6670 6671 state := testStateStore(t) 6672 alloc := mock.Alloc() 6673 var accessors []*structs.VaultAccessor 6674 var expected []*structs.VaultAccessor 6675 6676 for i := 0; i < 5; i++ { 6677 accessor := mock.VaultAccessor() 6678 accessor.AllocID = alloc.ID 6679 expected = append(expected, accessor) 6680 accessors = append(accessors, accessor) 6681 } 6682 6683 for i := 0; i < 10; i++ { 6684 accessor := mock.VaultAccessor() 6685 accessors = append(accessors, accessor) 6686 } 6687 6688 err := state.UpsertVaultAccessor(1000, accessors) 6689 if err != nil { 6690 t.Fatalf("err: %v", err) 6691 } 6692 6693 ws := memdb.NewWatchSet() 6694 out, err := state.VaultAccessorsByAlloc(ws, alloc.ID) 6695 if err != nil { 6696 t.Fatalf("err: %v", err) 6697 } 6698 6699 if len(expected) != len(out) { 6700 t.Fatalf("bad: %#v %#v", len(expected), len(out)) 6701 } 6702 6703 index, err := state.Index("vault_accessors") 6704 if err != nil { 6705 t.Fatalf("err: %v", err) 6706 } 6707 if index != 1000 { 6708 t.Fatalf("bad: %d", index) 6709 } 6710 6711 if watchFired(ws) { 6712 t.Fatalf("bad") 6713 } 6714 } 6715 6716 func TestStateStore_VaultAccessorsByNode(t *testing.T) { 6717 t.Parallel() 6718 6719 state := testStateStore(t) 6720 node := mock.Node() 6721 var accessors []*structs.VaultAccessor 6722 var expected []*structs.VaultAccessor 6723 6724 for i := 0; i < 5; i++ { 6725 accessor := mock.VaultAccessor() 6726 accessor.NodeID = node.ID 6727 expected = append(expected, accessor) 6728 accessors = append(accessors, accessor) 6729 } 6730 6731 for i := 0; i < 10; i++ { 6732 accessor := mock.VaultAccessor() 6733 accessors = append(accessors, accessor) 6734 } 6735 6736 err := state.UpsertVaultAccessor(1000, accessors) 6737 if err != nil { 6738 t.Fatalf("err: %v", err) 6739 } 6740 6741 ws := memdb.NewWatchSet() 6742 out, err := state.VaultAccessorsByNode(ws, node.ID) 6743 if err != nil { 6744 t.Fatalf("err: %v", err) 6745 } 6746 6747 if len(expected) != len(out) { 6748 t.Fatalf("bad: %#v %#v", len(expected), len(out)) 6749 } 6750 6751 index, err := state.Index("vault_accessors") 6752 if err != nil { 6753 t.Fatalf("err: %v", err) 6754 } 6755 if index != 1000 { 6756 t.Fatalf("bad: %d", index) 6757 } 6758 6759 if watchFired(ws) { 6760 t.Fatalf("bad") 6761 } 6762 } 6763 6764 func TestStateStore_RestoreVaultAccessor(t *testing.T) { 6765 t.Parallel() 6766 6767 state := testStateStore(t) 6768 a := mock.VaultAccessor() 6769 6770 restore, err := state.Restore() 6771 if err != nil { 6772 t.Fatalf("err: %v", err) 6773 } 6774 6775 err = restore.VaultAccessorRestore(a) 6776 if err != nil { 6777 t.Fatalf("err: %v", err) 6778 } 6779 restore.Commit() 6780 6781 ws := memdb.NewWatchSet() 6782 out, err := state.VaultAccessor(ws, a.Accessor) 6783 if err != nil { 6784 t.Fatalf("err: %v", err) 6785 } 6786 6787 if !reflect.DeepEqual(out, a) { 6788 t.Fatalf("Bad: %#v %#v", out, a) 6789 } 6790 6791 if watchFired(ws) { 6792 t.Fatalf("bad") 6793 } 6794 } 6795 6796 func TestStateStore_UpsertACLPolicy(t *testing.T) { 6797 t.Parallel() 6798 6799 state := testStateStore(t) 6800 policy := mock.ACLPolicy() 6801 policy2 := mock.ACLPolicy() 6802 6803 ws := memdb.NewWatchSet() 6804 if _, err := state.ACLPolicyByName(ws, policy.Name); err != nil { 6805 t.Fatalf("err: %v", err) 6806 } 6807 if _, err := state.ACLPolicyByName(ws, policy2.Name); err != nil { 6808 t.Fatalf("err: %v", err) 6809 } 6810 6811 if err := state.UpsertACLPolicies(1000, 6812 []*structs.ACLPolicy{policy, policy2}); err != nil { 6813 t.Fatalf("err: %v", err) 6814 } 6815 if !watchFired(ws) { 6816 t.Fatalf("bad") 6817 } 6818 6819 ws = memdb.NewWatchSet() 6820 out, err := state.ACLPolicyByName(ws, policy.Name) 6821 assert.Equal(t, nil, err) 6822 assert.Equal(t, policy, out) 6823 6824 out, err = state.ACLPolicyByName(ws, policy2.Name) 6825 assert.Equal(t, nil, err) 6826 assert.Equal(t, policy2, out) 6827 6828 iter, err := state.ACLPolicies(ws) 6829 if err != nil { 6830 t.Fatalf("err: %v", err) 6831 } 6832 6833 // Ensure we see both policies 6834 count := 0 6835 for { 6836 raw := iter.Next() 6837 if raw == nil { 6838 break 6839 } 6840 count++ 6841 } 6842 if count != 2 { 6843 t.Fatalf("bad: %d", count) 6844 } 6845 6846 index, err := state.Index("acl_policy") 6847 if err != nil { 6848 t.Fatalf("err: %v", err) 6849 } 6850 if index != 1000 { 6851 t.Fatalf("bad: %d", index) 6852 } 6853 6854 if watchFired(ws) { 6855 t.Fatalf("bad") 6856 } 6857 } 6858 6859 func TestStateStore_DeleteACLPolicy(t *testing.T) { 6860 t.Parallel() 6861 6862 state := testStateStore(t) 6863 policy := mock.ACLPolicy() 6864 policy2 := mock.ACLPolicy() 6865 6866 // Create the policy 6867 if err := state.UpsertACLPolicies(1000, 6868 []*structs.ACLPolicy{policy, policy2}); err != nil { 6869 t.Fatalf("err: %v", err) 6870 } 6871 6872 // Create a watcher 6873 ws := memdb.NewWatchSet() 6874 if _, err := state.ACLPolicyByName(ws, policy.Name); err != nil { 6875 t.Fatalf("err: %v", err) 6876 } 6877 6878 // Delete the policy 6879 if err := state.DeleteACLPolicies(1001, 6880 []string{policy.Name, policy2.Name}); err != nil { 6881 t.Fatalf("err: %v", err) 6882 } 6883 6884 // Ensure watching triggered 6885 if !watchFired(ws) { 6886 t.Fatalf("bad") 6887 } 6888 6889 // Ensure we don't get the object back 6890 ws = memdb.NewWatchSet() 6891 out, err := state.ACLPolicyByName(ws, policy.Name) 6892 assert.Equal(t, nil, err) 6893 if out != nil { 6894 t.Fatalf("bad: %#v", out) 6895 } 6896 6897 iter, err := state.ACLPolicies(ws) 6898 if err != nil { 6899 t.Fatalf("err: %v", err) 6900 } 6901 6902 // Ensure we see both policies 6903 count := 0 6904 for { 6905 raw := iter.Next() 6906 if raw == nil { 6907 break 6908 } 6909 count++ 6910 } 6911 if count != 0 { 6912 t.Fatalf("bad: %d", count) 6913 } 6914 6915 index, err := state.Index("acl_policy") 6916 if err != nil { 6917 t.Fatalf("err: %v", err) 6918 } 6919 if index != 1001 { 6920 t.Fatalf("bad: %d", index) 6921 } 6922 6923 if watchFired(ws) { 6924 t.Fatalf("bad") 6925 } 6926 } 6927 6928 func TestStateStore_ACLPolicyByNamePrefix(t *testing.T) { 6929 t.Parallel() 6930 6931 state := testStateStore(t) 6932 names := []string{ 6933 "foo", 6934 "bar", 6935 "foobar", 6936 "foozip", 6937 "zip", 6938 } 6939 6940 // Create the policies 6941 var baseIndex uint64 = 1000 6942 for _, name := range names { 6943 p := mock.ACLPolicy() 6944 p.Name = name 6945 if err := state.UpsertACLPolicies(baseIndex, []*structs.ACLPolicy{p}); err != nil { 6946 t.Fatalf("err: %v", err) 6947 } 6948 baseIndex++ 6949 } 6950 6951 // Scan by prefix 6952 iter, err := state.ACLPolicyByNamePrefix(nil, "foo") 6953 if err != nil { 6954 t.Fatalf("err: %v", err) 6955 } 6956 6957 // Ensure we see both policies 6958 count := 0 6959 out := []string{} 6960 for { 6961 raw := iter.Next() 6962 if raw == nil { 6963 break 6964 } 6965 count++ 6966 out = append(out, raw.(*structs.ACLPolicy).Name) 6967 } 6968 if count != 3 { 6969 t.Fatalf("bad: %d %v", count, out) 6970 } 6971 sort.Strings(out) 6972 6973 expect := []string{"foo", "foobar", "foozip"} 6974 assert.Equal(t, expect, out) 6975 } 6976 6977 func TestStateStore_BootstrapACLTokens(t *testing.T) { 6978 t.Parallel() 6979 6980 state := testStateStore(t) 6981 tk1 := mock.ACLToken() 6982 tk2 := mock.ACLToken() 6983 6984 ok, resetIdx, err := state.CanBootstrapACLToken() 6985 assert.Nil(t, err) 6986 assert.Equal(t, true, ok) 6987 assert.EqualValues(t, 0, resetIdx) 6988 6989 if err := state.BootstrapACLTokens(1000, 0, tk1); err != nil { 6990 t.Fatalf("err: %v", err) 6991 } 6992 6993 out, err := state.ACLTokenByAccessorID(nil, tk1.AccessorID) 6994 assert.Equal(t, nil, err) 6995 assert.Equal(t, tk1, out) 6996 6997 ok, resetIdx, err = state.CanBootstrapACLToken() 6998 assert.Nil(t, err) 6999 assert.Equal(t, false, ok) 7000 assert.EqualValues(t, 1000, resetIdx) 7001 7002 if err := state.BootstrapACLTokens(1001, 0, tk2); err == nil { 7003 t.Fatalf("expected error") 7004 } 7005 7006 iter, err := state.ACLTokens(nil) 7007 if err != nil { 7008 t.Fatalf("err: %v", err) 7009 } 7010 7011 // Ensure we see both policies 7012 count := 0 7013 for { 7014 raw := iter.Next() 7015 if raw == nil { 7016 break 7017 } 7018 count++ 7019 } 7020 if count != 1 { 7021 t.Fatalf("bad: %d", count) 7022 } 7023 7024 index, err := state.Index("acl_token") 7025 if err != nil { 7026 t.Fatalf("err: %v", err) 7027 } 7028 if index != 1000 { 7029 t.Fatalf("bad: %d", index) 7030 } 7031 index, err = state.Index("acl_token_bootstrap") 7032 if err != nil { 7033 t.Fatalf("err: %v", err) 7034 } 7035 if index != 1000 { 7036 t.Fatalf("bad: %d", index) 7037 } 7038 7039 // Should allow bootstrap with reset index 7040 if err := state.BootstrapACLTokens(1001, 1000, tk2); err != nil { 7041 t.Fatalf("err %v", err) 7042 } 7043 7044 // Check we've modified the index 7045 index, err = state.Index("acl_token") 7046 if err != nil { 7047 t.Fatalf("err: %v", err) 7048 } 7049 if index != 1001 { 7050 t.Fatalf("bad: %d", index) 7051 } 7052 index, err = state.Index("acl_token_bootstrap") 7053 if err != nil { 7054 t.Fatalf("err: %v", err) 7055 } 7056 if index != 1001 { 7057 t.Fatalf("bad: %d", index) 7058 } 7059 } 7060 7061 func TestStateStore_UpsertACLTokens(t *testing.T) { 7062 t.Parallel() 7063 7064 state := testStateStore(t) 7065 tk1 := mock.ACLToken() 7066 tk2 := mock.ACLToken() 7067 7068 ws := memdb.NewWatchSet() 7069 if _, err := state.ACLTokenByAccessorID(ws, tk1.AccessorID); err != nil { 7070 t.Fatalf("err: %v", err) 7071 } 7072 if _, err := state.ACLTokenByAccessorID(ws, tk2.AccessorID); err != nil { 7073 t.Fatalf("err: %v", err) 7074 } 7075 7076 if err := state.UpsertACLTokens(1000, 7077 []*structs.ACLToken{tk1, tk2}); err != nil { 7078 t.Fatalf("err: %v", err) 7079 } 7080 if !watchFired(ws) { 7081 t.Fatalf("bad") 7082 } 7083 7084 ws = memdb.NewWatchSet() 7085 out, err := state.ACLTokenByAccessorID(ws, tk1.AccessorID) 7086 assert.Equal(t, nil, err) 7087 assert.Equal(t, tk1, out) 7088 7089 out, err = state.ACLTokenByAccessorID(ws, tk2.AccessorID) 7090 assert.Equal(t, nil, err) 7091 assert.Equal(t, tk2, out) 7092 7093 out, err = state.ACLTokenBySecretID(ws, tk1.SecretID) 7094 assert.Equal(t, nil, err) 7095 assert.Equal(t, tk1, out) 7096 7097 out, err = state.ACLTokenBySecretID(ws, tk2.SecretID) 7098 assert.Equal(t, nil, err) 7099 assert.Equal(t, tk2, out) 7100 7101 iter, err := state.ACLTokens(ws) 7102 if err != nil { 7103 t.Fatalf("err: %v", err) 7104 } 7105 7106 // Ensure we see both policies 7107 count := 0 7108 for { 7109 raw := iter.Next() 7110 if raw == nil { 7111 break 7112 } 7113 count++ 7114 } 7115 if count != 2 { 7116 t.Fatalf("bad: %d", count) 7117 } 7118 7119 index, err := state.Index("acl_token") 7120 if err != nil { 7121 t.Fatalf("err: %v", err) 7122 } 7123 if index != 1000 { 7124 t.Fatalf("bad: %d", index) 7125 } 7126 7127 if watchFired(ws) { 7128 t.Fatalf("bad") 7129 } 7130 } 7131 7132 func TestStateStore_DeleteACLTokens(t *testing.T) { 7133 t.Parallel() 7134 7135 state := testStateStore(t) 7136 tk1 := mock.ACLToken() 7137 tk2 := mock.ACLToken() 7138 7139 // Create the tokens 7140 if err := state.UpsertACLTokens(1000, 7141 []*structs.ACLToken{tk1, tk2}); err != nil { 7142 t.Fatalf("err: %v", err) 7143 } 7144 7145 // Create a watcher 7146 ws := memdb.NewWatchSet() 7147 if _, err := state.ACLTokenByAccessorID(ws, tk1.AccessorID); err != nil { 7148 t.Fatalf("err: %v", err) 7149 } 7150 7151 // Delete the token 7152 if err := state.DeleteACLTokens(1001, 7153 []string{tk1.AccessorID, tk2.AccessorID}); err != nil { 7154 t.Fatalf("err: %v", err) 7155 } 7156 7157 // Ensure watching triggered 7158 if !watchFired(ws) { 7159 t.Fatalf("bad") 7160 } 7161 7162 // Ensure we don't get the object back 7163 ws = memdb.NewWatchSet() 7164 out, err := state.ACLTokenByAccessorID(ws, tk1.AccessorID) 7165 assert.Equal(t, nil, err) 7166 if out != nil { 7167 t.Fatalf("bad: %#v", out) 7168 } 7169 7170 iter, err := state.ACLTokens(ws) 7171 if err != nil { 7172 t.Fatalf("err: %v", err) 7173 } 7174 7175 // Ensure we see both policies 7176 count := 0 7177 for { 7178 raw := iter.Next() 7179 if raw == nil { 7180 break 7181 } 7182 count++ 7183 } 7184 if count != 0 { 7185 t.Fatalf("bad: %d", count) 7186 } 7187 7188 index, err := state.Index("acl_token") 7189 if err != nil { 7190 t.Fatalf("err: %v", err) 7191 } 7192 if index != 1001 { 7193 t.Fatalf("bad: %d", index) 7194 } 7195 7196 if watchFired(ws) { 7197 t.Fatalf("bad") 7198 } 7199 } 7200 7201 func TestStateStore_ACLTokenByAccessorIDPrefix(t *testing.T) { 7202 t.Parallel() 7203 7204 state := testStateStore(t) 7205 prefixes := []string{ 7206 "aaaa", 7207 "aabb", 7208 "bbbb", 7209 "bbcc", 7210 "ffff", 7211 } 7212 7213 // Create the tokens 7214 var baseIndex uint64 = 1000 7215 for _, prefix := range prefixes { 7216 tk := mock.ACLToken() 7217 tk.AccessorID = prefix + tk.AccessorID[4:] 7218 if err := state.UpsertACLTokens(baseIndex, []*structs.ACLToken{tk}); err != nil { 7219 t.Fatalf("err: %v", err) 7220 } 7221 baseIndex++ 7222 } 7223 7224 // Scan by prefix 7225 iter, err := state.ACLTokenByAccessorIDPrefix(nil, "aa") 7226 if err != nil { 7227 t.Fatalf("err: %v", err) 7228 } 7229 7230 // Ensure we see both tokens 7231 count := 0 7232 out := []string{} 7233 for { 7234 raw := iter.Next() 7235 if raw == nil { 7236 break 7237 } 7238 count++ 7239 out = append(out, raw.(*structs.ACLToken).AccessorID[:4]) 7240 } 7241 if count != 2 { 7242 t.Fatalf("bad: %d %v", count, out) 7243 } 7244 sort.Strings(out) 7245 7246 expect := []string{"aaaa", "aabb"} 7247 assert.Equal(t, expect, out) 7248 } 7249 7250 func TestStateStore_RestoreACLPolicy(t *testing.T) { 7251 t.Parallel() 7252 7253 state := testStateStore(t) 7254 policy := mock.ACLPolicy() 7255 7256 restore, err := state.Restore() 7257 if err != nil { 7258 t.Fatalf("err: %v", err) 7259 } 7260 7261 err = restore.ACLPolicyRestore(policy) 7262 if err != nil { 7263 t.Fatalf("err: %v", err) 7264 } 7265 restore.Commit() 7266 7267 ws := memdb.NewWatchSet() 7268 out, err := state.ACLPolicyByName(ws, policy.Name) 7269 if err != nil { 7270 t.Fatalf("err: %v", err) 7271 } 7272 assert.Equal(t, policy, out) 7273 } 7274 7275 func TestStateStore_ACLTokensByGlobal(t *testing.T) { 7276 t.Parallel() 7277 7278 state := testStateStore(t) 7279 tk1 := mock.ACLToken() 7280 tk2 := mock.ACLToken() 7281 tk3 := mock.ACLToken() 7282 tk4 := mock.ACLToken() 7283 tk3.Global = true 7284 7285 if err := state.UpsertACLTokens(1000, 7286 []*structs.ACLToken{tk1, tk2, tk3, tk4}); err != nil { 7287 t.Fatalf("err: %v", err) 7288 } 7289 7290 iter, err := state.ACLTokensByGlobal(nil, true) 7291 if err != nil { 7292 t.Fatalf("err: %v", err) 7293 } 7294 7295 // Ensure we see the one global policies 7296 count := 0 7297 for { 7298 raw := iter.Next() 7299 if raw == nil { 7300 break 7301 } 7302 count++ 7303 } 7304 if count != 1 { 7305 t.Fatalf("bad: %d", count) 7306 } 7307 } 7308 7309 func TestStateStore_RestoreACLToken(t *testing.T) { 7310 t.Parallel() 7311 7312 state := testStateStore(t) 7313 token := mock.ACLToken() 7314 7315 restore, err := state.Restore() 7316 if err != nil { 7317 t.Fatalf("err: %v", err) 7318 } 7319 7320 err = restore.ACLTokenRestore(token) 7321 if err != nil { 7322 t.Fatalf("err: %v", err) 7323 } 7324 restore.Commit() 7325 7326 ws := memdb.NewWatchSet() 7327 out, err := state.ACLTokenByAccessorID(ws, token.AccessorID) 7328 if err != nil { 7329 t.Fatalf("err: %v", err) 7330 } 7331 assert.Equal(t, token, out) 7332 } 7333 7334 func TestStateStore_SchedulerConfig(t *testing.T) { 7335 t.Parallel() 7336 7337 state := testStateStore(t) 7338 schedConfig := &structs.SchedulerConfiguration{ 7339 PreemptionConfig: structs.PreemptionConfig{ 7340 SystemSchedulerEnabled: false, 7341 }, 7342 CreateIndex: 100, 7343 ModifyIndex: 200, 7344 } 7345 7346 require := require.New(t) 7347 restore, err := state.Restore() 7348 7349 require.Nil(err) 7350 7351 err = restore.SchedulerConfigRestore(schedConfig) 7352 require.Nil(err) 7353 7354 restore.Commit() 7355 7356 modIndex, out, err := state.SchedulerConfig() 7357 require.Nil(err) 7358 require.Equal(schedConfig.ModifyIndex, modIndex) 7359 7360 require.Equal(schedConfig, out) 7361 } 7362 7363 func TestStateStore_Abandon(t *testing.T) { 7364 t.Parallel() 7365 7366 s := testStateStore(t) 7367 abandonCh := s.AbandonCh() 7368 s.Abandon() 7369 select { 7370 case <-abandonCh: 7371 default: 7372 t.Fatalf("bad") 7373 } 7374 } 7375 7376 // Verifies that an error is returned when an allocation doesn't exist in the state store. 7377 func TestStateSnapshot_DenormalizeAllocationDiffSlice_AllocDoesNotExist(t *testing.T) { 7378 t.Parallel() 7379 7380 state := testStateStore(t) 7381 alloc := mock.Alloc() 7382 require := require.New(t) 7383 7384 // Insert job 7385 err := state.UpsertJob(999, alloc.Job) 7386 require.NoError(err) 7387 7388 allocDiffs := []*structs.AllocationDiff{ 7389 { 7390 ID: alloc.ID, 7391 }, 7392 } 7393 7394 snap, err := state.Snapshot() 7395 require.NoError(err) 7396 7397 denormalizedAllocs, err := snap.DenormalizeAllocationDiffSlice(allocDiffs) 7398 7399 require.EqualError(err, fmt.Sprintf("alloc %v doesn't exist", alloc.ID)) 7400 require.Nil(denormalizedAllocs) 7401 } 7402 7403 // TestStateStore_SnapshotMinIndex_OK asserts StateStore.SnapshotMinIndex blocks 7404 // until the StateStore's latest index is >= the requested index. 7405 func TestStateStore_SnapshotMinIndex_OK(t *testing.T) { 7406 t.Parallel() 7407 7408 s := testStateStore(t) 7409 index, err := s.LatestIndex() 7410 require.NoError(t, err) 7411 7412 node := mock.Node() 7413 require.NoError(t, s.UpsertNode(index+1, node)) 7414 7415 // Assert SnapshotMinIndex returns immediately if index < latest index 7416 ctx, cancel := context.WithTimeout(context.Background(), 0) 7417 snap, err := s.SnapshotMinIndex(ctx, index) 7418 cancel() 7419 require.NoError(t, err) 7420 7421 snapIndex, err := snap.LatestIndex() 7422 require.NoError(t, err) 7423 if snapIndex <= index { 7424 require.Fail(t, "snapshot index should be greater than index") 7425 } 7426 7427 // Assert SnapshotMinIndex returns immediately if index == latest index 7428 ctx, cancel = context.WithTimeout(context.Background(), 0) 7429 snap, err = s.SnapshotMinIndex(ctx, index+1) 7430 cancel() 7431 require.NoError(t, err) 7432 7433 snapIndex, err = snap.LatestIndex() 7434 require.NoError(t, err) 7435 require.Equal(t, snapIndex, index+1) 7436 7437 // Assert SnapshotMinIndex blocks if index > latest index 7438 errCh := make(chan error, 1) 7439 ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) 7440 defer cancel() 7441 go func() { 7442 defer close(errCh) 7443 waitIndex := index + 2 7444 snap, err := s.SnapshotMinIndex(ctx, waitIndex) 7445 if err != nil { 7446 errCh <- err 7447 return 7448 } 7449 7450 snapIndex, err := snap.LatestIndex() 7451 if err != nil { 7452 errCh <- err 7453 return 7454 } 7455 7456 if snapIndex < waitIndex { 7457 errCh <- fmt.Errorf("snapshot index < wait index: %d < %d", snapIndex, waitIndex) 7458 return 7459 } 7460 }() 7461 7462 select { 7463 case err := <-errCh: 7464 require.NoError(t, err) 7465 case <-time.After(500 * time.Millisecond): 7466 // Let it block for a bit before unblocking by upserting 7467 } 7468 7469 node.Name = "hal" 7470 require.NoError(t, s.UpsertNode(index+2, node)) 7471 7472 select { 7473 case err := <-errCh: 7474 require.NoError(t, err) 7475 case <-time.After(5 * time.Second): 7476 require.Fail(t, "timed out waiting for SnapshotMinIndex to unblock") 7477 } 7478 } 7479 7480 // TestStateStore_SnapshotMinIndex_Timeout asserts StateStore.SnapshotMinIndex 7481 // returns an error if the desired index is not reached within the deadline. 7482 func TestStateStore_SnapshotMinIndex_Timeout(t *testing.T) { 7483 t.Parallel() 7484 7485 s := testStateStore(t) 7486 index, err := s.LatestIndex() 7487 require.NoError(t, err) 7488 7489 // Assert SnapshotMinIndex blocks if index > latest index 7490 ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) 7491 defer cancel() 7492 snap, err := s.SnapshotMinIndex(ctx, index+1) 7493 require.EqualError(t, err, context.DeadlineExceeded.Error()) 7494 require.Nil(t, snap) 7495 } 7496 7497 // watchFired is a helper for unit tests that returns if the given watch set 7498 // fired (it doesn't care which watch actually fired). This uses a fixed 7499 // timeout since we already expect the event happened before calling this and 7500 // just need to distinguish a fire from a timeout. We do need a little time to 7501 // allow the watch to set up any goroutines, though. 7502 func watchFired(ws memdb.WatchSet) bool { 7503 timedOut := ws.Watch(time.After(50 * time.Millisecond)) 7504 return !timedOut 7505 } 7506 7507 // NodeIDSort is used to sort nodes by ID 7508 type NodeIDSort []*structs.Node 7509 7510 func (n NodeIDSort) Len() int { 7511 return len(n) 7512 } 7513 7514 func (n NodeIDSort) Less(i, j int) bool { 7515 return n[i].ID < n[j].ID 7516 } 7517 7518 func (n NodeIDSort) Swap(i, j int) { 7519 n[i], n[j] = n[j], n[i] 7520 } 7521 7522 // JobIDis used to sort jobs by id 7523 type JobIDSort []*structs.Job 7524 7525 func (n JobIDSort) Len() int { 7526 return len(n) 7527 } 7528 7529 func (n JobIDSort) Less(i, j int) bool { 7530 return n[i].ID < n[j].ID 7531 } 7532 7533 func (n JobIDSort) Swap(i, j int) { 7534 n[i], n[j] = n[j], n[i] 7535 } 7536 7537 // EvalIDis used to sort evals by id 7538 type EvalIDSort []*structs.Evaluation 7539 7540 func (n EvalIDSort) Len() int { 7541 return len(n) 7542 } 7543 7544 func (n EvalIDSort) Less(i, j int) bool { 7545 return n[i].ID < n[j].ID 7546 } 7547 7548 func (n EvalIDSort) Swap(i, j int) { 7549 n[i], n[j] = n[j], n[i] 7550 } 7551 7552 // AllocIDsort used to sort allocations by id 7553 type AllocIDSort []*structs.Allocation 7554 7555 func (n AllocIDSort) Len() int { 7556 return len(n) 7557 } 7558 7559 func (n AllocIDSort) Less(i, j int) bool { 7560 return n[i].ID < n[j].ID 7561 } 7562 7563 func (n AllocIDSort) Swap(i, j int) { 7564 n[i], n[j] = n[j], n[i] 7565 }