github.com/aminovpavel/nomad@v0.11.8/nomad/blocked_evals_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "reflect" 6 "testing" 7 "time" 8 9 "github.com/hashicorp/nomad/helper/testlog" 10 "github.com/hashicorp/nomad/nomad/mock" 11 "github.com/hashicorp/nomad/nomad/structs" 12 "github.com/hashicorp/nomad/testutil" 13 "github.com/stretchr/testify/require" 14 ) 15 16 func testBlockedEvals(t *testing.T) (*BlockedEvals, *EvalBroker) { 17 broker := testBroker(t, 0) 18 broker.SetEnabled(true) 19 blocked := NewBlockedEvals(broker, testlog.HCLogger(t)) 20 blocked.SetEnabled(true) 21 return blocked, broker 22 } 23 24 func TestBlockedEvals_Block_Disabled(t *testing.T) { 25 t.Parallel() 26 blocked, _ := testBlockedEvals(t) 27 blocked.SetEnabled(false) 28 29 // Create an escaped eval and add it to the blocked tracker. 30 e := mock.Eval() 31 e.Status = structs.EvalStatusBlocked 32 e.EscapedComputedClass = true 33 blocked.Block(e) 34 35 // Verify block did nothing 36 bStats := blocked.Stats() 37 if bStats.TotalBlocked != 0 || bStats.TotalEscaped != 0 { 38 t.Fatalf("bad: %#v", bStats) 39 } 40 } 41 42 func TestBlockedEvals_Block_SameJob(t *testing.T) { 43 t.Parallel() 44 blocked, _ := testBlockedEvals(t) 45 46 // Create two blocked evals and add them to the blocked tracker. 47 e := mock.Eval() 48 e2 := mock.Eval() 49 e2.JobID = e.JobID 50 blocked.Block(e) 51 blocked.Block(e2) 52 53 // Verify block did track both 54 bStats := blocked.Stats() 55 if bStats.TotalBlocked != 1 || bStats.TotalEscaped != 0 { 56 t.Fatalf("bad: %#v", bStats) 57 } 58 } 59 60 func TestBlockedEvals_Block_Quota(t *testing.T) { 61 t.Parallel() 62 blocked, _ := testBlockedEvals(t) 63 64 // Create a blocked evals on quota 65 e := mock.Eval() 66 e.QuotaLimitReached = "foo" 67 blocked.Block(e) 68 69 // Verify block did track both 70 bs := blocked.Stats() 71 if bs.TotalBlocked != 1 || bs.TotalEscaped != 0 || bs.TotalQuotaLimit != 1 { 72 t.Fatalf("bad: %#v", bs) 73 } 74 } 75 76 func TestBlockedEvals_Block_PriorUnblocks(t *testing.T) { 77 t.Parallel() 78 blocked, _ := testBlockedEvals(t) 79 80 // Do unblocks prior to blocking 81 blocked.Unblock("v1:123", 1000) 82 blocked.Unblock("v1:123", 1001) 83 84 // Create two blocked evals and add them to the blocked tracker. 85 e := mock.Eval() 86 e.Status = structs.EvalStatusBlocked 87 e.ClassEligibility = map[string]bool{"v1:123": false, "v1:456": false} 88 e.SnapshotIndex = 999 89 blocked.Block(e) 90 91 // Verify block did track both 92 bStats := blocked.Stats() 93 if bStats.TotalBlocked != 1 || bStats.TotalEscaped != 0 { 94 t.Fatalf("bad: %#v", bStats) 95 } 96 } 97 98 func TestBlockedEvals_GetDuplicates(t *testing.T) { 99 t.Parallel() 100 blocked, _ := testBlockedEvals(t) 101 102 // Create duplicate blocked evals and add them to the blocked tracker. 103 e := mock.Eval() 104 e.CreateIndex = 100 105 e2 := mock.Eval() 106 e2.JobID = e.JobID 107 e2.CreateIndex = 101 108 e3 := mock.Eval() 109 e3.JobID = e.JobID 110 e3.CreateIndex = 102 111 e4 := mock.Eval() 112 e4.JobID = e.JobID 113 e4.CreateIndex = 100 114 blocked.Block(e) 115 blocked.Block(e2) 116 117 // Verify stats such that we are only tracking one 118 bStats := blocked.Stats() 119 if bStats.TotalBlocked != 1 || bStats.TotalEscaped != 0 { 120 t.Fatalf("bad: %#v", bStats) 121 } 122 123 // Get the duplicates. 124 out := blocked.GetDuplicates(0) 125 if len(out) != 1 || !reflect.DeepEqual(out[0], e) { 126 t.Fatalf("bad: %#v %#v", out, e) 127 } 128 129 // Call block again after a small sleep. 130 go func() { 131 time.Sleep(500 * time.Millisecond) 132 blocked.Block(e3) 133 }() 134 135 // Get the duplicates. 136 out = blocked.GetDuplicates(1 * time.Second) 137 if len(out) != 1 || !reflect.DeepEqual(out[0], e2) { 138 t.Fatalf("bad: %#v %#v", out, e2) 139 } 140 141 // Verify stats such that we are only tracking one 142 bStats = blocked.Stats() 143 if bStats.TotalBlocked != 1 || bStats.TotalEscaped != 0 { 144 t.Fatalf("bad: %#v", bStats) 145 } 146 147 // Add an older evaluation and assert it gets cancelled 148 blocked.Block(e4) 149 out = blocked.GetDuplicates(0) 150 if len(out) != 1 || !reflect.DeepEqual(out[0], e4) { 151 t.Fatalf("bad: %#v %#v", out, e4) 152 } 153 154 // Verify stats such that we are only tracking one 155 bStats = blocked.Stats() 156 if bStats.TotalBlocked != 1 || bStats.TotalEscaped != 0 { 157 t.Fatalf("bad: %#v", bStats) 158 } 159 } 160 161 func TestBlockedEvals_UnblockEscaped(t *testing.T) { 162 t.Parallel() 163 blocked, broker := testBlockedEvals(t) 164 165 // Create an escaped eval and add it to the blocked tracker. 166 e := mock.Eval() 167 e.Status = structs.EvalStatusBlocked 168 e.EscapedComputedClass = true 169 blocked.Block(e) 170 171 // Verify block caused the eval to be tracked 172 bStats := blocked.Stats() 173 if bStats.TotalBlocked != 1 || bStats.TotalEscaped != 1 { 174 t.Fatalf("bad: %#v", bStats) 175 } 176 177 blocked.Unblock("v1:123", 1000) 178 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 179 } 180 181 func requireBlockedEvalsEnqueued(t *testing.T, blocked *BlockedEvals, broker *EvalBroker, enqueued int) { 182 testutil.WaitForResult(func() (bool, error) { 183 // Verify Unblock caused an enqueue 184 brokerStats := broker.Stats() 185 if brokerStats.TotalReady != enqueued { 186 return false, fmt.Errorf("missing enqueued evals: %#v", brokerStats) 187 } 188 189 // Verify Unblock updates the stats 190 bStats := blocked.Stats() 191 if bStats.TotalBlocked != 0 || bStats.TotalEscaped != 0 { 192 return false, fmt.Errorf("evals still blocked: %#v", bStats) 193 } 194 return true, nil 195 }, func(err error) { 196 t.Fatalf("err: %s", err) 197 }) 198 } 199 200 func TestBlockedEvals_UnblockEligible(t *testing.T) { 201 t.Parallel() 202 blocked, broker := testBlockedEvals(t) 203 204 // Create a blocked eval that is eligible on a specific node class and add 205 // it to the blocked tracker. 206 e := mock.Eval() 207 e.Status = structs.EvalStatusBlocked 208 e.ClassEligibility = map[string]bool{"v1:123": true} 209 blocked.Block(e) 210 211 // Verify block caused the eval to be tracked 212 blockedStats := blocked.Stats() 213 if blockedStats.TotalBlocked != 1 { 214 t.Fatalf("bad: %#v", blockedStats) 215 } 216 217 blocked.Unblock("v1:123", 1000) 218 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 219 } 220 221 func TestBlockedEvals_UnblockIneligible(t *testing.T) { 222 t.Parallel() 223 blocked, broker := testBlockedEvals(t) 224 225 // Create a blocked eval that is ineligible on a specific node class and add 226 // it to the blocked tracker. 227 e := mock.Eval() 228 e.Status = structs.EvalStatusBlocked 229 e.ClassEligibility = map[string]bool{"v1:123": false} 230 blocked.Block(e) 231 232 // Verify block caused the eval to be tracked 233 blockedStats := blocked.Stats() 234 if blockedStats.TotalBlocked != 1 && blockedStats.TotalEscaped != 0 { 235 t.Fatalf("bad: %#v", blockedStats) 236 } 237 238 // Should do nothing 239 blocked.Unblock("v1:123", 1000) 240 241 testutil.WaitForResult(func() (bool, error) { 242 // Verify Unblock didn't cause an enqueue 243 brokerStats := broker.Stats() 244 if brokerStats.TotalReady != 0 { 245 return false, fmt.Errorf("bad: %#v", brokerStats) 246 } 247 248 bStats := blocked.Stats() 249 if bStats.TotalBlocked != 1 || bStats.TotalEscaped != 0 { 250 return false, fmt.Errorf("bad: %#v", bStats) 251 } 252 return true, nil 253 }, func(err error) { 254 t.Fatalf("err: %s", err) 255 }) 256 } 257 258 func TestBlockedEvals_UnblockUnknown(t *testing.T) { 259 t.Parallel() 260 blocked, broker := testBlockedEvals(t) 261 262 // Create a blocked eval that is ineligible on a specific node class and add 263 // it to the blocked tracker. 264 e := mock.Eval() 265 e.Status = structs.EvalStatusBlocked 266 e.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false} 267 blocked.Block(e) 268 269 // Verify block caused the eval to be tracked 270 blockedStats := blocked.Stats() 271 if blockedStats.TotalBlocked != 1 && blockedStats.TotalEscaped != 0 { 272 t.Fatalf("bad: %#v", blockedStats) 273 } 274 275 // Should unblock because the eval hasn't seen this node class. 276 blocked.Unblock("v1:789", 1000) 277 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 278 } 279 280 func TestBlockedEvals_UnblockEligible_Quota(t *testing.T) { 281 t.Parallel() 282 blocked, broker := testBlockedEvals(t) 283 284 // Create a blocked eval that is eligible for a particular quota 285 e := mock.Eval() 286 e.Status = structs.EvalStatusBlocked 287 e.QuotaLimitReached = "foo" 288 blocked.Block(e) 289 290 // Verify block caused the eval to be tracked 291 bs := blocked.Stats() 292 if bs.TotalBlocked != 1 || bs.TotalQuotaLimit != 1 { 293 t.Fatalf("bad: %#v", bs) 294 } 295 296 blocked.UnblockQuota("foo", 1000) 297 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 298 } 299 300 func TestBlockedEvals_UnblockIneligible_Quota(t *testing.T) { 301 t.Parallel() 302 blocked, broker := testBlockedEvals(t) 303 304 // Create a blocked eval that is eligible on a specific quota 305 e := mock.Eval() 306 e.Status = structs.EvalStatusBlocked 307 e.QuotaLimitReached = "foo" 308 blocked.Block(e) 309 310 // Verify block caused the eval to be tracked 311 bs := blocked.Stats() 312 if bs.TotalBlocked != 1 || bs.TotalQuotaLimit != 1 { 313 t.Fatalf("bad: %#v", bs) 314 } 315 316 // Should do nothing 317 blocked.UnblockQuota("bar", 1000) 318 319 testutil.WaitForResult(func() (bool, error) { 320 // Verify Unblock didn't cause an enqueue 321 brokerStats := broker.Stats() 322 if brokerStats.TotalReady != 0 { 323 return false, fmt.Errorf("bad: %#v", brokerStats) 324 } 325 326 bs := blocked.Stats() 327 if bs.TotalBlocked != 1 || bs.TotalEscaped != 0 || bs.TotalQuotaLimit != 1 { 328 return false, fmt.Errorf("bad: %#v", bs) 329 } 330 return true, nil 331 }, func(err error) { 332 t.Fatalf("err: %s", err) 333 }) 334 } 335 336 func TestBlockedEvals_Reblock(t *testing.T) { 337 t.Parallel() 338 blocked, broker := testBlockedEvals(t) 339 340 // Create an evaluation, Enqueue/Dequeue it to get a token 341 e := mock.Eval() 342 e.SnapshotIndex = 500 343 e.Status = structs.EvalStatusBlocked 344 e.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false} 345 broker.Enqueue(e) 346 347 _, token, err := broker.Dequeue([]string{e.Type}, time.Second) 348 if err != nil { 349 t.Fatalf("err: %v", err) 350 } 351 352 // Reblock the evaluation 353 blocked.Reblock(e, token) 354 355 // Verify block caused the eval to be tracked 356 blockedStats := blocked.Stats() 357 if blockedStats.TotalBlocked != 1 && blockedStats.TotalEscaped != 0 { 358 t.Fatalf("bad: %#v", blockedStats) 359 } 360 361 // Should unblock because the eval 362 blocked.Unblock("v1:123", 1000) 363 364 brokerStats := broker.Stats() 365 if brokerStats.TotalReady != 0 && brokerStats.TotalUnacked != 1 { 366 t.Fatalf("bad: %#v", brokerStats) 367 } 368 369 // Ack the evaluation which should cause the reblocked eval to transition 370 // to ready 371 if err := broker.Ack(e.ID, token); err != nil { 372 t.Fatalf("err: %v", err) 373 } 374 375 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 376 } 377 378 // Test the block case in which the eval should be immediately unblocked since 379 // it is escaped and old 380 func TestBlockedEvals_Block_ImmediateUnblock_Escaped(t *testing.T) { 381 t.Parallel() 382 blocked, broker := testBlockedEvals(t) 383 384 // Do an unblock prior to blocking 385 blocked.Unblock("v1:123", 1000) 386 387 // Create a blocked eval that is eligible on a specific node class and add 388 // it to the blocked tracker. 389 e := mock.Eval() 390 e.Status = structs.EvalStatusBlocked 391 e.EscapedComputedClass = true 392 e.SnapshotIndex = 900 393 blocked.Block(e) 394 395 // Verify block caused the eval to be immediately unblocked 396 blockedStats := blocked.Stats() 397 if blockedStats.TotalBlocked != 0 && blockedStats.TotalEscaped != 0 { 398 t.Fatalf("bad: %#v", blockedStats) 399 } 400 401 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 402 } 403 404 // Test the block case in which the eval should be immediately unblocked since 405 // there is an unblock on an unseen class that occurred while it was in the 406 // scheduler 407 func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass_After(t *testing.T) { 408 t.Parallel() 409 blocked, broker := testBlockedEvals(t) 410 411 // Do an unblock prior to blocking 412 blocked.Unblock("v1:123", 1000) 413 414 // Create a blocked eval that is eligible on a specific node class and add 415 // it to the blocked tracker. 416 e := mock.Eval() 417 e.Status = structs.EvalStatusBlocked 418 e.EscapedComputedClass = false 419 e.SnapshotIndex = 900 420 blocked.Block(e) 421 422 // Verify block caused the eval to be immediately unblocked 423 blockedStats := blocked.Stats() 424 if blockedStats.TotalBlocked != 0 && blockedStats.TotalEscaped != 0 { 425 t.Fatalf("bad: %#v", blockedStats) 426 } 427 428 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 429 } 430 431 // Test the block case in which the eval should not immediately unblock since 432 // there is an unblock on an unseen class that occurred before it was in the 433 // scheduler 434 func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass_Before(t *testing.T) { 435 t.Parallel() 436 blocked, _ := testBlockedEvals(t) 437 438 // Do an unblock prior to blocking 439 blocked.Unblock("v1:123", 500) 440 441 // Create a blocked eval that is eligible on a specific node class and add 442 // it to the blocked tracker. 443 e := mock.Eval() 444 e.Status = structs.EvalStatusBlocked 445 e.EscapedComputedClass = false 446 e.SnapshotIndex = 900 447 blocked.Block(e) 448 449 // Verify block caused the eval to be immediately unblocked 450 blockedStats := blocked.Stats() 451 if blockedStats.TotalBlocked != 1 && blockedStats.TotalEscaped != 0 { 452 t.Fatalf("bad: %#v", blockedStats) 453 } 454 } 455 456 // Test the block case in which the eval should be immediately unblocked since 457 // it a class it is eligible for has been unblocked 458 func TestBlockedEvals_Block_ImmediateUnblock_SeenClass(t *testing.T) { 459 t.Parallel() 460 blocked, broker := testBlockedEvals(t) 461 462 // Do an unblock prior to blocking 463 blocked.Unblock("v1:123", 1000) 464 465 // Create a blocked eval that is eligible on a specific node class and add 466 // it to the blocked tracker. 467 e := mock.Eval() 468 e.Status = structs.EvalStatusBlocked 469 e.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false} 470 e.SnapshotIndex = 900 471 blocked.Block(e) 472 473 // Verify block caused the eval to be immediately unblocked 474 blockedStats := blocked.Stats() 475 if blockedStats.TotalBlocked != 0 && blockedStats.TotalEscaped != 0 { 476 t.Fatalf("bad: %#v", blockedStats) 477 } 478 479 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 480 } 481 482 // Test the block case in which the eval should be immediately unblocked since 483 // it a quota has changed that it is using 484 func TestBlockedEvals_Block_ImmediateUnblock_Quota(t *testing.T) { 485 t.Parallel() 486 blocked, broker := testBlockedEvals(t) 487 488 // Do an unblock prior to blocking 489 blocked.UnblockQuota("my-quota", 1000) 490 491 // Create a blocked eval that is eligible on a specific node class and add 492 // it to the blocked tracker. 493 e := mock.Eval() 494 e.Status = structs.EvalStatusBlocked 495 e.QuotaLimitReached = "my-quota" 496 e.SnapshotIndex = 900 497 blocked.Block(e) 498 499 // Verify block caused the eval to be immediately unblocked 500 bs := blocked.Stats() 501 if bs.TotalBlocked != 0 && bs.TotalEscaped != 0 && bs.TotalQuotaLimit != 0 { 502 t.Fatalf("bad: %#v", bs) 503 } 504 505 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 506 } 507 508 func TestBlockedEvals_UnblockFailed(t *testing.T) { 509 t.Parallel() 510 blocked, broker := testBlockedEvals(t) 511 512 // Create blocked evals that are due to failures 513 e := mock.Eval() 514 e.Status = structs.EvalStatusBlocked 515 e.TriggeredBy = structs.EvalTriggerMaxPlans 516 e.EscapedComputedClass = true 517 blocked.Block(e) 518 519 e2 := mock.Eval() 520 e2.Status = structs.EvalStatusBlocked 521 e2.TriggeredBy = structs.EvalTriggerMaxPlans 522 e2.ClassEligibility = map[string]bool{"v1:123": true, "v1:456": false} 523 blocked.Block(e2) 524 525 e3 := mock.Eval() 526 e3.Status = structs.EvalStatusBlocked 527 e3.TriggeredBy = structs.EvalTriggerMaxPlans 528 e3.QuotaLimitReached = "foo" 529 blocked.Block(e3) 530 531 // Trigger an unblock fail 532 blocked.UnblockFailed() 533 534 // Verify UnblockFailed caused the eval to be immediately unblocked 535 bs := blocked.Stats() 536 if bs.TotalBlocked != 0 || bs.TotalEscaped != 0 || bs.TotalQuotaLimit != 0 { 537 t.Fatalf("bad: %#v", bs) 538 } 539 540 requireBlockedEvalsEnqueued(t, blocked, broker, 3) 541 542 // Reblock an eval for the same job and check that it gets tracked. 543 blocked.Block(e) 544 bs = blocked.Stats() 545 if bs.TotalBlocked != 1 || bs.TotalEscaped != 1 { 546 t.Fatalf("bad: %#v", bs) 547 } 548 } 549 550 func TestBlockedEvals_Untrack(t *testing.T) { 551 t.Parallel() 552 blocked, _ := testBlockedEvals(t) 553 554 // Create two blocked evals and add them to the blocked tracker. 555 e := mock.Eval() 556 e.Status = structs.EvalStatusBlocked 557 e.ClassEligibility = map[string]bool{"v1:123": false, "v1:456": false} 558 e.SnapshotIndex = 1000 559 blocked.Block(e) 560 561 // Verify block did track 562 bStats := blocked.Stats() 563 if bStats.TotalBlocked != 1 || bStats.TotalEscaped != 0 { 564 t.Fatalf("bad: %#v", bStats) 565 } 566 567 // Untrack and verify 568 blocked.Untrack(e.JobID, e.Namespace) 569 bStats = blocked.Stats() 570 if bStats.TotalBlocked != 0 || bStats.TotalEscaped != 0 { 571 t.Fatalf("bad: %#v", bStats) 572 } 573 } 574 575 func TestBlockedEvals_Untrack_Quota(t *testing.T) { 576 t.Parallel() 577 blocked, _ := testBlockedEvals(t) 578 579 // Create a blocked evals and add it to the blocked tracker. 580 e := mock.Eval() 581 e.Status = structs.EvalStatusBlocked 582 e.QuotaLimitReached = "foo" 583 e.SnapshotIndex = 1000 584 blocked.Block(e) 585 586 // Verify block did track 587 bs := blocked.Stats() 588 if bs.TotalBlocked != 1 || bs.TotalEscaped != 0 || bs.TotalQuotaLimit != 1 { 589 t.Fatalf("bad: %#v", bs) 590 } 591 592 // Untrack and verify 593 blocked.Untrack(e.JobID, e.Namespace) 594 bs = blocked.Stats() 595 if bs.TotalBlocked != 0 || bs.TotalEscaped != 0 || bs.TotalQuotaLimit != 0 { 596 t.Fatalf("bad: %#v", bs) 597 } 598 } 599 600 func TestBlockedEvals_UnblockNode(t *testing.T) { 601 t.Parallel() 602 blocked, broker := testBlockedEvals(t) 603 604 require.NotNil(t, broker) 605 606 // Create a blocked evals and add it to the blocked tracker. 607 e := mock.Eval() 608 e.Type = structs.JobTypeSystem 609 e.NodeID = "foo" 610 e.SnapshotIndex = 999 611 blocked.Block(e) 612 613 // Verify block did track 614 bs := blocked.Stats() 615 require.Equal(t, 1, bs.TotalBlocked) 616 617 blocked.UnblockNode("foo", 1000) 618 requireBlockedEvalsEnqueued(t, blocked, broker, 1) 619 bs = blocked.Stats() 620 require.Empty(t, blocked.system.byNode) 621 require.Equal(t, 0, bs.TotalBlocked) 622 } 623 624 func TestBlockedEvals_SystemUntrack(t *testing.T) { 625 t.Parallel() 626 blocked, _ := testBlockedEvals(t) 627 628 // Create a blocked evals and add it to the blocked tracker. 629 e := mock.Eval() 630 e.Type = structs.JobTypeSystem 631 e.NodeID = "foo" 632 blocked.Block(e) 633 634 // Verify block did track 635 bs := blocked.Stats() 636 require.Equal(t, 1, bs.TotalBlocked) 637 require.Equal(t, 0, bs.TotalEscaped) 638 require.Equal(t, 0, bs.TotalQuotaLimit) 639 640 // Untrack and verify 641 blocked.Untrack(e.JobID, e.Namespace) 642 bs = blocked.Stats() 643 require.Equal(t, 0, bs.TotalBlocked) 644 require.Equal(t, 0, bs.TotalEscaped) 645 require.Equal(t, 0, bs.TotalQuotaLimit) 646 } 647 648 func TestBlockedEvals_SystemDisableFlush(t *testing.T) { 649 t.Parallel() 650 blocked, _ := testBlockedEvals(t) 651 652 // Create a blocked evals and add it to the blocked tracker. 653 e := mock.Eval() 654 e.Type = structs.JobTypeSystem 655 e.NodeID = "foo" 656 blocked.Block(e) 657 658 // Verify block did track 659 bs := blocked.Stats() 660 require.Equal(t, 1, bs.TotalBlocked) 661 require.Equal(t, 0, bs.TotalEscaped) 662 require.Equal(t, 0, bs.TotalQuotaLimit) 663 664 // Disable empties 665 blocked.SetEnabled(false) 666 bs = blocked.Stats() 667 require.Equal(t, 0, bs.TotalBlocked) 668 require.Equal(t, 0, bs.TotalEscaped) 669 require.Equal(t, 0, bs.TotalQuotaLimit) 670 require.Empty(t, blocked.system.evals) 671 require.Empty(t, blocked.system.byJob) 672 require.Empty(t, blocked.system.byNode) 673 }