github.com/uchennaokeke444/nomad@v0.11.8/scheduler/spread_test.go (about) 1 package scheduler 2 3 import ( 4 "math" 5 "testing" 6 7 "fmt" 8 9 "github.com/hashicorp/nomad/helper/uuid" 10 "github.com/hashicorp/nomad/nomad/mock" 11 "github.com/hashicorp/nomad/nomad/structs" 12 "github.com/stretchr/testify/require" 13 ) 14 15 func TestSpreadIterator_SingleAttribute(t *testing.T) { 16 state, ctx := testContext(t) 17 dcs := []string{"dc1", "dc2", "dc1", "dc1"} 18 var nodes []*RankedNode 19 20 // Add these nodes to the state store 21 for i, dc := range dcs { 22 node := mock.Node() 23 node.Datacenter = dc 24 if err := state.UpsertNode(uint64(100+i), node); err != nil { 25 t.Fatalf("failed to upsert node: %v", err) 26 } 27 nodes = append(nodes, &RankedNode{Node: node}) 28 } 29 30 static := NewStaticRankIterator(ctx, nodes) 31 32 job := mock.Job() 33 tg := job.TaskGroups[0] 34 job.TaskGroups[0].Count = 10 35 // add allocs to nodes in dc1 36 upserting := []*structs.Allocation{ 37 { 38 Namespace: structs.DefaultNamespace, 39 TaskGroup: tg.Name, 40 JobID: job.ID, 41 Job: job, 42 ID: uuid.Generate(), 43 EvalID: uuid.Generate(), 44 NodeID: nodes[0].Node.ID, 45 }, 46 { 47 Namespace: structs.DefaultNamespace, 48 TaskGroup: tg.Name, 49 JobID: job.ID, 50 Job: job, 51 ID: uuid.Generate(), 52 EvalID: uuid.Generate(), 53 NodeID: nodes[2].Node.ID, 54 }, 55 } 56 57 if err := state.UpsertAllocs(1000, upserting); err != nil { 58 t.Fatalf("failed to UpsertAllocs: %v", err) 59 } 60 61 // Create spread target of 80% in dc1 62 // Implicitly, this means 20% in dc2 63 spread := &structs.Spread{ 64 Weight: 100, 65 Attribute: "${node.datacenter}", 66 SpreadTarget: []*structs.SpreadTarget{ 67 { 68 Value: "dc1", 69 Percent: 80, 70 }, 71 }, 72 } 73 tg.Spreads = []*structs.Spread{spread} 74 spreadIter := NewSpreadIterator(ctx, static) 75 spreadIter.SetJob(job) 76 spreadIter.SetTaskGroup(tg) 77 78 scoreNorm := NewScoreNormalizationIterator(ctx, spreadIter) 79 80 out := collectRanked(scoreNorm) 81 82 // Expect nodes in dc1 with existing allocs to get a boost 83 // Boost should be ((desiredCount-actual)/desired)*spreadWeight 84 // For this test, that becomes dc1 = ((8-3)/8 ) = 0.5, and dc2=(2-1)/2 85 expectedScores := map[string]float64{ 86 "dc1": 0.625, 87 "dc2": 0.5, 88 } 89 for _, rn := range out { 90 require.Equal(t, expectedScores[rn.Node.Datacenter], rn.FinalScore) 91 } 92 93 // Update the plan to add more allocs to nodes in dc1 94 // After this step there are enough allocs to meet the desired count in dc1 95 ctx.plan.NodeAllocation[nodes[0].Node.ID] = []*structs.Allocation{ 96 { 97 Namespace: structs.DefaultNamespace, 98 TaskGroup: tg.Name, 99 JobID: job.ID, 100 Job: job, 101 ID: uuid.Generate(), 102 NodeID: nodes[0].Node.ID, 103 }, 104 { 105 Namespace: structs.DefaultNamespace, 106 TaskGroup: tg.Name, 107 JobID: job.ID, 108 Job: job, 109 ID: uuid.Generate(), 110 NodeID: nodes[0].Node.ID, 111 }, 112 // Should be ignored as it is a different job. 113 { 114 Namespace: structs.DefaultNamespace, 115 TaskGroup: "bbb", 116 JobID: "ignore 2", 117 Job: job, 118 ID: uuid.Generate(), 119 NodeID: nodes[0].Node.ID, 120 }, 121 } 122 ctx.plan.NodeAllocation[nodes[3].Node.ID] = []*structs.Allocation{ 123 { 124 Namespace: structs.DefaultNamespace, 125 TaskGroup: tg.Name, 126 JobID: job.ID, 127 Job: job, 128 ID: uuid.Generate(), 129 NodeID: nodes[3].Node.ID, 130 }, 131 { 132 Namespace: structs.DefaultNamespace, 133 TaskGroup: tg.Name, 134 JobID: job.ID, 135 Job: job, 136 ID: uuid.Generate(), 137 NodeID: nodes[3].Node.ID, 138 }, 139 { 140 Namespace: structs.DefaultNamespace, 141 TaskGroup: tg.Name, 142 JobID: job.ID, 143 Job: job, 144 ID: uuid.Generate(), 145 NodeID: nodes[3].Node.ID, 146 }, 147 } 148 149 // Reset the scores 150 for _, node := range nodes { 151 node.Scores = nil 152 node.FinalScore = 0 153 } 154 static = NewStaticRankIterator(ctx, nodes) 155 spreadIter = NewSpreadIterator(ctx, static) 156 spreadIter.SetJob(job) 157 spreadIter.SetTaskGroup(tg) 158 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 159 out = collectRanked(scoreNorm) 160 161 // Expect nodes in dc2 with existing allocs to get a boost 162 // DC1 nodes are not boosted because there are enough allocs to meet 163 // the desired count 164 expectedScores = map[string]float64{ 165 "dc1": 0, 166 "dc2": 0.5, 167 } 168 for _, rn := range out { 169 require.Equal(t, expectedScores[rn.Node.Datacenter], rn.FinalScore) 170 } 171 } 172 173 func TestSpreadIterator_MultipleAttributes(t *testing.T) { 174 state, ctx := testContext(t) 175 dcs := []string{"dc1", "dc2", "dc1", "dc1"} 176 rack := []string{"r1", "r1", "r2", "r2"} 177 var nodes []*RankedNode 178 179 // Add these nodes to the state store 180 for i, dc := range dcs { 181 node := mock.Node() 182 node.Datacenter = dc 183 node.Meta["rack"] = rack[i] 184 if err := state.UpsertNode(uint64(100+i), node); err != nil { 185 t.Fatalf("failed to upsert node: %v", err) 186 } 187 nodes = append(nodes, &RankedNode{Node: node}) 188 } 189 190 static := NewStaticRankIterator(ctx, nodes) 191 192 job := mock.Job() 193 tg := job.TaskGroups[0] 194 job.TaskGroups[0].Count = 10 195 // add allocs to nodes in dc1 196 upserting := []*structs.Allocation{ 197 { 198 Namespace: structs.DefaultNamespace, 199 TaskGroup: tg.Name, 200 JobID: job.ID, 201 Job: job, 202 ID: uuid.Generate(), 203 EvalID: uuid.Generate(), 204 NodeID: nodes[0].Node.ID, 205 }, 206 { 207 Namespace: structs.DefaultNamespace, 208 TaskGroup: tg.Name, 209 JobID: job.ID, 210 Job: job, 211 ID: uuid.Generate(), 212 EvalID: uuid.Generate(), 213 NodeID: nodes[2].Node.ID, 214 }, 215 } 216 217 if err := state.UpsertAllocs(1000, upserting); err != nil { 218 t.Fatalf("failed to UpsertAllocs: %v", err) 219 } 220 221 spread1 := &structs.Spread{ 222 Weight: 100, 223 Attribute: "${node.datacenter}", 224 SpreadTarget: []*structs.SpreadTarget{ 225 { 226 Value: "dc1", 227 Percent: 60, 228 }, 229 { 230 Value: "dc2", 231 Percent: 40, 232 }, 233 }, 234 } 235 236 spread2 := &structs.Spread{ 237 Weight: 50, 238 Attribute: "${meta.rack}", 239 SpreadTarget: []*structs.SpreadTarget{ 240 { 241 Value: "r1", 242 Percent: 40, 243 }, 244 { 245 Value: "r2", 246 Percent: 60, 247 }, 248 }, 249 } 250 251 tg.Spreads = []*structs.Spread{spread1, spread2} 252 spreadIter := NewSpreadIterator(ctx, static) 253 spreadIter.SetJob(job) 254 spreadIter.SetTaskGroup(tg) 255 256 scoreNorm := NewScoreNormalizationIterator(ctx, spreadIter) 257 258 out := collectRanked(scoreNorm) 259 260 // Score comes from combining two different spread factors 261 // Second node should have the highest score because it has no allocs and its in dc2/r1 262 expectedScores := map[string]float64{ 263 nodes[0].Node.ID: 0.500, 264 nodes[1].Node.ID: 0.667, 265 nodes[2].Node.ID: 0.556, 266 nodes[3].Node.ID: 0.556, 267 } 268 for _, rn := range out { 269 require.Equal(t, fmt.Sprintf("%.3f", expectedScores[rn.Node.ID]), fmt.Sprintf("%.3f", rn.FinalScore)) 270 } 271 272 } 273 274 func TestSpreadIterator_EvenSpread(t *testing.T) { 275 state, ctx := testContext(t) 276 dcs := []string{"dc1", "dc2", "dc1", "dc2", "dc1", "dc2", "dc2", "dc1", "dc1", "dc1"} 277 var nodes []*RankedNode 278 279 // Add these nodes to the state store 280 for i, dc := range dcs { 281 node := mock.Node() 282 node.Datacenter = dc 283 if err := state.UpsertNode(uint64(100+i), node); err != nil { 284 t.Fatalf("failed to upsert node: %v", err) 285 } 286 nodes = append(nodes, &RankedNode{Node: node}) 287 } 288 289 static := NewStaticRankIterator(ctx, nodes) 290 job := mock.Job() 291 tg := job.TaskGroups[0] 292 job.TaskGroups[0].Count = 10 293 294 // Configure even spread across node.datacenter 295 spread := &structs.Spread{ 296 Weight: 100, 297 Attribute: "${node.datacenter}", 298 } 299 tg.Spreads = []*structs.Spread{spread} 300 spreadIter := NewSpreadIterator(ctx, static) 301 spreadIter.SetJob(job) 302 spreadIter.SetTaskGroup(tg) 303 304 scoreNorm := NewScoreNormalizationIterator(ctx, spreadIter) 305 306 out := collectRanked(scoreNorm) 307 308 // Nothing placed so both dc nodes get 0 as the score 309 expectedScores := map[string]float64{ 310 "dc1": 0, 311 "dc2": 0, 312 } 313 for _, rn := range out { 314 require.Equal(t, fmt.Sprintf("%.3f", expectedScores[rn.Node.Datacenter]), fmt.Sprintf("%.3f", rn.FinalScore)) 315 316 } 317 318 // Update the plan to add allocs to nodes in dc1 319 // After this step dc2 nodes should get boosted 320 ctx.plan.NodeAllocation[nodes[0].Node.ID] = []*structs.Allocation{ 321 { 322 Namespace: structs.DefaultNamespace, 323 TaskGroup: tg.Name, 324 JobID: job.ID, 325 Job: job, 326 ID: uuid.Generate(), 327 NodeID: nodes[0].Node.ID, 328 }, 329 } 330 ctx.plan.NodeAllocation[nodes[2].Node.ID] = []*structs.Allocation{ 331 { 332 Namespace: structs.DefaultNamespace, 333 TaskGroup: tg.Name, 334 JobID: job.ID, 335 Job: job, 336 ID: uuid.Generate(), 337 NodeID: nodes[2].Node.ID, 338 }, 339 } 340 341 // Reset the scores 342 for _, node := range nodes { 343 node.Scores = nil 344 node.FinalScore = 0 345 } 346 static = NewStaticRankIterator(ctx, nodes) 347 spreadIter = NewSpreadIterator(ctx, static) 348 spreadIter.SetJob(job) 349 spreadIter.SetTaskGroup(tg) 350 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 351 out = collectRanked(scoreNorm) 352 353 // Expect nodes in dc2 with existing allocs to get a boost 354 // dc1 nodes are penalized because they have allocs 355 expectedScores = map[string]float64{ 356 "dc1": -1, 357 "dc2": 1, 358 } 359 for _, rn := range out { 360 require.Equal(t, expectedScores[rn.Node.Datacenter], rn.FinalScore) 361 } 362 363 // Update the plan to add more allocs to nodes in dc2 364 // After this step dc1 nodes should get boosted 365 ctx.plan.NodeAllocation[nodes[1].Node.ID] = []*structs.Allocation{ 366 { 367 Namespace: structs.DefaultNamespace, 368 TaskGroup: tg.Name, 369 JobID: job.ID, 370 Job: job, 371 ID: uuid.Generate(), 372 NodeID: nodes[1].Node.ID, 373 }, 374 { 375 Namespace: structs.DefaultNamespace, 376 TaskGroup: tg.Name, 377 JobID: job.ID, 378 Job: job, 379 ID: uuid.Generate(), 380 NodeID: nodes[1].Node.ID, 381 }, 382 } 383 ctx.plan.NodeAllocation[nodes[3].Node.ID] = []*structs.Allocation{ 384 { 385 Namespace: structs.DefaultNamespace, 386 TaskGroup: tg.Name, 387 JobID: job.ID, 388 Job: job, 389 ID: uuid.Generate(), 390 NodeID: nodes[3].Node.ID, 391 }, 392 } 393 394 // Reset the scores 395 for _, node := range nodes { 396 node.Scores = nil 397 node.FinalScore = 0 398 } 399 static = NewStaticRankIterator(ctx, nodes) 400 spreadIter = NewSpreadIterator(ctx, static) 401 spreadIter.SetJob(job) 402 spreadIter.SetTaskGroup(tg) 403 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 404 out = collectRanked(scoreNorm) 405 406 // Expect nodes in dc2 to be penalized because there are 3 allocs there now 407 // dc1 nodes are boosted because that has 2 allocs 408 expectedScores = map[string]float64{ 409 "dc1": 0.5, 410 "dc2": -0.5, 411 } 412 for _, rn := range out { 413 require.Equal(t, fmt.Sprintf("%3.3f", expectedScores[rn.Node.Datacenter]), fmt.Sprintf("%3.3f", rn.FinalScore)) 414 } 415 416 // Add another node in dc3 417 node := mock.Node() 418 node.Datacenter = "dc3" 419 if err := state.UpsertNode(uint64(1111), node); err != nil { 420 t.Fatalf("failed to upsert node: %v", err) 421 } 422 nodes = append(nodes, &RankedNode{Node: node}) 423 424 // Add another alloc to dc1, now its count matches dc2 425 ctx.plan.NodeAllocation[nodes[4].Node.ID] = []*structs.Allocation{ 426 { 427 Namespace: structs.DefaultNamespace, 428 TaskGroup: tg.Name, 429 JobID: job.ID, 430 Job: job, 431 ID: uuid.Generate(), 432 NodeID: nodes[4].Node.ID, 433 }, 434 } 435 436 // Reset scores 437 for _, node := range nodes { 438 node.Scores = nil 439 node.FinalScore = 0 440 } 441 static = NewStaticRankIterator(ctx, nodes) 442 spreadIter = NewSpreadIterator(ctx, static) 443 spreadIter.SetJob(job) 444 spreadIter.SetTaskGroup(tg) 445 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 446 out = collectRanked(scoreNorm) 447 448 // Expect dc1 and dc2 to be penalized because they have 3 allocs 449 // dc3 should get a boost because it has 0 allocs 450 expectedScores = map[string]float64{ 451 "dc1": -1, 452 "dc2": -1, 453 "dc3": 1, 454 } 455 for _, rn := range out { 456 require.Equal(t, fmt.Sprintf("%.3f", expectedScores[rn.Node.Datacenter]), fmt.Sprintf("%.3f", rn.FinalScore)) 457 } 458 459 } 460 461 // Test scenarios where the spread iterator sets maximum penalty (-1.0) 462 func TestSpreadIterator_MaxPenalty(t *testing.T) { 463 state, ctx := testContext(t) 464 var nodes []*RankedNode 465 466 // Add nodes in dc3 to the state store 467 for i := 0; i < 5; i++ { 468 node := mock.Node() 469 node.Datacenter = "dc3" 470 if err := state.UpsertNode(uint64(100+i), node); err != nil { 471 t.Fatalf("failed to upsert node: %v", err) 472 } 473 nodes = append(nodes, &RankedNode{Node: node}) 474 } 475 476 static := NewStaticRankIterator(ctx, nodes) 477 478 job := mock.Job() 479 tg := job.TaskGroups[0] 480 job.TaskGroups[0].Count = 5 481 482 // Create spread target of 80% in dc1 483 // and 20% in dc2 484 spread := &structs.Spread{ 485 Weight: 100, 486 Attribute: "${node.datacenter}", 487 SpreadTarget: []*structs.SpreadTarget{ 488 { 489 Value: "dc1", 490 Percent: 80, 491 }, 492 { 493 Value: "dc2", 494 Percent: 20, 495 }, 496 }, 497 } 498 tg.Spreads = []*structs.Spread{spread} 499 spreadIter := NewSpreadIterator(ctx, static) 500 spreadIter.SetJob(job) 501 spreadIter.SetTaskGroup(tg) 502 503 scoreNorm := NewScoreNormalizationIterator(ctx, spreadIter) 504 505 out := collectRanked(scoreNorm) 506 507 // All nodes are in dc3 so score should be -1 508 for _, rn := range out { 509 require.Equal(t, -1.0, rn.FinalScore) 510 } 511 512 // Reset scores 513 for _, node := range nodes { 514 node.Scores = nil 515 node.FinalScore = 0 516 } 517 518 // Create spread on attribute that doesn't exist on any nodes 519 spread = &structs.Spread{ 520 Weight: 100, 521 Attribute: "${meta.foo}", 522 SpreadTarget: []*structs.SpreadTarget{ 523 { 524 Value: "bar", 525 Percent: 80, 526 }, 527 { 528 Value: "baz", 529 Percent: 20, 530 }, 531 }, 532 } 533 534 tg.Spreads = []*structs.Spread{spread} 535 static = NewStaticRankIterator(ctx, nodes) 536 spreadIter = NewSpreadIterator(ctx, static) 537 spreadIter.SetJob(job) 538 spreadIter.SetTaskGroup(tg) 539 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 540 out = collectRanked(scoreNorm) 541 542 // All nodes don't have the spread attribute so score should be -1 543 for _, rn := range out { 544 require.Equal(t, -1.0, rn.FinalScore) 545 } 546 547 } 548 549 func Test_evenSpreadScoreBoost(t *testing.T) { 550 pset := &propertySet{ 551 existingValues: map[string]uint64{}, 552 proposedValues: map[string]uint64{ 553 "dc2": 1, 554 "dc1": 1, 555 "dc3": 1, 556 }, 557 clearedValues: map[string]uint64{ 558 "dc2": 1, 559 "dc3": 1, 560 }, 561 targetAttribute: "${node.datacenter}", 562 } 563 564 opt := &structs.Node{ 565 Datacenter: "dc2", 566 } 567 boost := evenSpreadScoreBoost(pset, opt) 568 require.False(t, math.IsInf(boost, 1)) 569 require.Equal(t, 1.0, boost) 570 }