github.com/smithx10/nomad@v0.9.1-rc1/scheduler/spread_test.go (about) 1 package scheduler 2 3 import ( 4 "testing" 5 6 "fmt" 7 8 "github.com/hashicorp/nomad/helper/uuid" 9 "github.com/hashicorp/nomad/nomad/mock" 10 "github.com/hashicorp/nomad/nomad/structs" 11 "github.com/stretchr/testify/require" 12 ) 13 14 func TestSpreadIterator_SingleAttribute(t *testing.T) { 15 state, ctx := testContext(t) 16 dcs := []string{"dc1", "dc2", "dc1", "dc1"} 17 var nodes []*RankedNode 18 19 // Add these nodes to the state store 20 for i, dc := range dcs { 21 node := mock.Node() 22 node.Datacenter = dc 23 if err := state.UpsertNode(uint64(100+i), node); err != nil { 24 t.Fatalf("failed to upsert node: %v", err) 25 } 26 nodes = append(nodes, &RankedNode{Node: node}) 27 } 28 29 static := NewStaticRankIterator(ctx, nodes) 30 31 job := mock.Job() 32 tg := job.TaskGroups[0] 33 job.TaskGroups[0].Count = 10 34 // add allocs to nodes in dc1 35 upserting := []*structs.Allocation{ 36 { 37 Namespace: structs.DefaultNamespace, 38 TaskGroup: tg.Name, 39 JobID: job.ID, 40 Job: job, 41 ID: uuid.Generate(), 42 EvalID: uuid.Generate(), 43 NodeID: nodes[0].Node.ID, 44 }, 45 { 46 Namespace: structs.DefaultNamespace, 47 TaskGroup: tg.Name, 48 JobID: job.ID, 49 Job: job, 50 ID: uuid.Generate(), 51 EvalID: uuid.Generate(), 52 NodeID: nodes[2].Node.ID, 53 }, 54 } 55 56 if err := state.UpsertAllocs(1000, upserting); err != nil { 57 t.Fatalf("failed to UpsertAllocs: %v", err) 58 } 59 60 // Create spread target of 80% in dc1 61 // Implicitly, this means 20% in dc2 62 spread := &structs.Spread{ 63 Weight: 100, 64 Attribute: "${node.datacenter}", 65 SpreadTarget: []*structs.SpreadTarget{ 66 { 67 Value: "dc1", 68 Percent: 80, 69 }, 70 }, 71 } 72 tg.Spreads = []*structs.Spread{spread} 73 spreadIter := NewSpreadIterator(ctx, static) 74 spreadIter.SetJob(job) 75 spreadIter.SetTaskGroup(tg) 76 77 scoreNorm := NewScoreNormalizationIterator(ctx, spreadIter) 78 79 out := collectRanked(scoreNorm) 80 81 // Expect nodes in dc1 with existing allocs to get a boost 82 // Boost should be ((desiredCount-actual)/desired)*spreadWeight 83 // For this test, that becomes dc1 = ((8-3)/8 ) = 0.5, and dc2=(2-1)/2 84 expectedScores := map[string]float64{ 85 "dc1": 0.625, 86 "dc2": 0.5, 87 } 88 for _, rn := range out { 89 require.Equal(t, expectedScores[rn.Node.Datacenter], rn.FinalScore) 90 } 91 92 // Update the plan to add more allocs to nodes in dc1 93 // After this step there are enough allocs to meet the desired count in dc1 94 ctx.plan.NodeAllocation[nodes[0].Node.ID] = []*structs.Allocation{ 95 { 96 Namespace: structs.DefaultNamespace, 97 TaskGroup: tg.Name, 98 JobID: job.ID, 99 Job: job, 100 ID: uuid.Generate(), 101 NodeID: nodes[0].Node.ID, 102 }, 103 { 104 Namespace: structs.DefaultNamespace, 105 TaskGroup: tg.Name, 106 JobID: job.ID, 107 Job: job, 108 ID: uuid.Generate(), 109 NodeID: nodes[0].Node.ID, 110 }, 111 // Should be ignored as it is a different job. 112 { 113 Namespace: structs.DefaultNamespace, 114 TaskGroup: "bbb", 115 JobID: "ignore 2", 116 Job: job, 117 ID: uuid.Generate(), 118 NodeID: nodes[0].Node.ID, 119 }, 120 } 121 ctx.plan.NodeAllocation[nodes[3].Node.ID] = []*structs.Allocation{ 122 { 123 Namespace: structs.DefaultNamespace, 124 TaskGroup: tg.Name, 125 JobID: job.ID, 126 Job: job, 127 ID: uuid.Generate(), 128 NodeID: nodes[3].Node.ID, 129 }, 130 { 131 Namespace: structs.DefaultNamespace, 132 TaskGroup: tg.Name, 133 JobID: job.ID, 134 Job: job, 135 ID: uuid.Generate(), 136 NodeID: nodes[3].Node.ID, 137 }, 138 { 139 Namespace: structs.DefaultNamespace, 140 TaskGroup: tg.Name, 141 JobID: job.ID, 142 Job: job, 143 ID: uuid.Generate(), 144 NodeID: nodes[3].Node.ID, 145 }, 146 } 147 148 // Reset the scores 149 for _, node := range nodes { 150 node.Scores = nil 151 node.FinalScore = 0 152 } 153 static = NewStaticRankIterator(ctx, nodes) 154 spreadIter = NewSpreadIterator(ctx, static) 155 spreadIter.SetJob(job) 156 spreadIter.SetTaskGroup(tg) 157 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 158 out = collectRanked(scoreNorm) 159 160 // Expect nodes in dc2 with existing allocs to get a boost 161 // DC1 nodes are not boosted because there are enough allocs to meet 162 // the desired count 163 expectedScores = map[string]float64{ 164 "dc1": 0, 165 "dc2": 0.5, 166 } 167 for _, rn := range out { 168 require.Equal(t, expectedScores[rn.Node.Datacenter], rn.FinalScore) 169 } 170 } 171 172 func TestSpreadIterator_MultipleAttributes(t *testing.T) { 173 state, ctx := testContext(t) 174 dcs := []string{"dc1", "dc2", "dc1", "dc1"} 175 rack := []string{"r1", "r1", "r2", "r2"} 176 var nodes []*RankedNode 177 178 // Add these nodes to the state store 179 for i, dc := range dcs { 180 node := mock.Node() 181 node.Datacenter = dc 182 node.Meta["rack"] = rack[i] 183 if err := state.UpsertNode(uint64(100+i), node); err != nil { 184 t.Fatalf("failed to upsert node: %v", err) 185 } 186 nodes = append(nodes, &RankedNode{Node: node}) 187 } 188 189 static := NewStaticRankIterator(ctx, nodes) 190 191 job := mock.Job() 192 tg := job.TaskGroups[0] 193 job.TaskGroups[0].Count = 10 194 // add allocs to nodes in dc1 195 upserting := []*structs.Allocation{ 196 { 197 Namespace: structs.DefaultNamespace, 198 TaskGroup: tg.Name, 199 JobID: job.ID, 200 Job: job, 201 ID: uuid.Generate(), 202 EvalID: uuid.Generate(), 203 NodeID: nodes[0].Node.ID, 204 }, 205 { 206 Namespace: structs.DefaultNamespace, 207 TaskGroup: tg.Name, 208 JobID: job.ID, 209 Job: job, 210 ID: uuid.Generate(), 211 EvalID: uuid.Generate(), 212 NodeID: nodes[2].Node.ID, 213 }, 214 } 215 216 if err := state.UpsertAllocs(1000, upserting); err != nil { 217 t.Fatalf("failed to UpsertAllocs: %v", err) 218 } 219 220 spread1 := &structs.Spread{ 221 Weight: 100, 222 Attribute: "${node.datacenter}", 223 SpreadTarget: []*structs.SpreadTarget{ 224 { 225 Value: "dc1", 226 Percent: 60, 227 }, 228 { 229 Value: "dc2", 230 Percent: 40, 231 }, 232 }, 233 } 234 235 spread2 := &structs.Spread{ 236 Weight: 50, 237 Attribute: "${meta.rack}", 238 SpreadTarget: []*structs.SpreadTarget{ 239 { 240 Value: "r1", 241 Percent: 40, 242 }, 243 { 244 Value: "r2", 245 Percent: 60, 246 }, 247 }, 248 } 249 250 tg.Spreads = []*structs.Spread{spread1, spread2} 251 spreadIter := NewSpreadIterator(ctx, static) 252 spreadIter.SetJob(job) 253 spreadIter.SetTaskGroup(tg) 254 255 scoreNorm := NewScoreNormalizationIterator(ctx, spreadIter) 256 257 out := collectRanked(scoreNorm) 258 259 // Score comes from combining two different spread factors 260 // Second node should have the highest score because it has no allocs and its in dc2/r1 261 expectedScores := map[string]float64{ 262 nodes[0].Node.ID: 0.500, 263 nodes[1].Node.ID: 0.667, 264 nodes[2].Node.ID: 0.556, 265 nodes[3].Node.ID: 0.556, 266 } 267 for _, rn := range out { 268 require.Equal(t, fmt.Sprintf("%.3f", expectedScores[rn.Node.ID]), fmt.Sprintf("%.3f", rn.FinalScore)) 269 } 270 271 } 272 273 func TestSpreadIterator_EvenSpread(t *testing.T) { 274 state, ctx := testContext(t) 275 dcs := []string{"dc1", "dc2", "dc1", "dc2", "dc1", "dc2", "dc2", "dc1", "dc1", "dc1"} 276 var nodes []*RankedNode 277 278 // Add these nodes to the state store 279 for i, dc := range dcs { 280 node := mock.Node() 281 node.Datacenter = dc 282 if err := state.UpsertNode(uint64(100+i), node); err != nil { 283 t.Fatalf("failed to upsert node: %v", err) 284 } 285 nodes = append(nodes, &RankedNode{Node: node}) 286 } 287 288 static := NewStaticRankIterator(ctx, nodes) 289 job := mock.Job() 290 tg := job.TaskGroups[0] 291 job.TaskGroups[0].Count = 10 292 293 // Configure even spread across node.datacenter 294 spread := &structs.Spread{ 295 Weight: 100, 296 Attribute: "${node.datacenter}", 297 } 298 tg.Spreads = []*structs.Spread{spread} 299 spreadIter := NewSpreadIterator(ctx, static) 300 spreadIter.SetJob(job) 301 spreadIter.SetTaskGroup(tg) 302 303 scoreNorm := NewScoreNormalizationIterator(ctx, spreadIter) 304 305 out := collectRanked(scoreNorm) 306 307 // Nothing placed so both dc nodes get 0 as the score 308 expectedScores := map[string]float64{ 309 "dc1": 0, 310 "dc2": 0, 311 } 312 for _, rn := range out { 313 require.Equal(t, fmt.Sprintf("%.3f", expectedScores[rn.Node.Datacenter]), fmt.Sprintf("%.3f", rn.FinalScore)) 314 } 315 316 // Update the plan to add allocs to nodes in dc1 317 // After this step dc2 nodes should get boosted 318 ctx.plan.NodeAllocation[nodes[0].Node.ID] = []*structs.Allocation{ 319 { 320 Namespace: structs.DefaultNamespace, 321 TaskGroup: tg.Name, 322 JobID: job.ID, 323 Job: job, 324 ID: uuid.Generate(), 325 NodeID: nodes[0].Node.ID, 326 }, 327 } 328 ctx.plan.NodeAllocation[nodes[2].Node.ID] = []*structs.Allocation{ 329 { 330 Namespace: structs.DefaultNamespace, 331 TaskGroup: tg.Name, 332 JobID: job.ID, 333 Job: job, 334 ID: uuid.Generate(), 335 NodeID: nodes[2].Node.ID, 336 }, 337 } 338 339 // Reset the scores 340 for _, node := range nodes { 341 node.Scores = nil 342 node.FinalScore = 0 343 } 344 static = NewStaticRankIterator(ctx, nodes) 345 spreadIter = NewSpreadIterator(ctx, static) 346 spreadIter.SetJob(job) 347 spreadIter.SetTaskGroup(tg) 348 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 349 out = collectRanked(scoreNorm) 350 351 // Expect nodes in dc2 with existing allocs to get a boost 352 // dc1 nodes are penalized because they have allocs 353 expectedScores = map[string]float64{ 354 "dc1": -1, 355 "dc2": 1, 356 } 357 for _, rn := range out { 358 require.Equal(t, expectedScores[rn.Node.Datacenter], rn.FinalScore) 359 } 360 361 // Update the plan to add more allocs to nodes in dc2 362 // After this step dc1 nodes should get boosted 363 ctx.plan.NodeAllocation[nodes[1].Node.ID] = []*structs.Allocation{ 364 { 365 Namespace: structs.DefaultNamespace, 366 TaskGroup: tg.Name, 367 JobID: job.ID, 368 Job: job, 369 ID: uuid.Generate(), 370 NodeID: nodes[1].Node.ID, 371 }, 372 { 373 Namespace: structs.DefaultNamespace, 374 TaskGroup: tg.Name, 375 JobID: job.ID, 376 Job: job, 377 ID: uuid.Generate(), 378 NodeID: nodes[1].Node.ID, 379 }, 380 } 381 ctx.plan.NodeAllocation[nodes[3].Node.ID] = []*structs.Allocation{ 382 { 383 Namespace: structs.DefaultNamespace, 384 TaskGroup: tg.Name, 385 JobID: job.ID, 386 Job: job, 387 ID: uuid.Generate(), 388 NodeID: nodes[3].Node.ID, 389 }, 390 } 391 392 // Reset the scores 393 for _, node := range nodes { 394 node.Scores = nil 395 node.FinalScore = 0 396 } 397 static = NewStaticRankIterator(ctx, nodes) 398 spreadIter = NewSpreadIterator(ctx, static) 399 spreadIter.SetJob(job) 400 spreadIter.SetTaskGroup(tg) 401 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 402 out = collectRanked(scoreNorm) 403 404 // Expect nodes in dc2 to be penalized because there are 3 allocs there now 405 // dc1 nodes are boosted because that has 2 allocs 406 expectedScores = map[string]float64{ 407 "dc1": 0.5, 408 "dc2": -0.5, 409 } 410 for _, rn := range out { 411 require.Equal(t, fmt.Sprintf("%3.3f", expectedScores[rn.Node.Datacenter]), fmt.Sprintf("%3.3f", rn.FinalScore)) 412 } 413 414 // Add another node in dc3 415 node := mock.Node() 416 node.Datacenter = "dc3" 417 if err := state.UpsertNode(uint64(1111), node); err != nil { 418 t.Fatalf("failed to upsert node: %v", err) 419 } 420 nodes = append(nodes, &RankedNode{Node: node}) 421 422 // Add another alloc to dc1, now its count matches dc2 423 ctx.plan.NodeAllocation[nodes[4].Node.ID] = []*structs.Allocation{ 424 { 425 Namespace: structs.DefaultNamespace, 426 TaskGroup: tg.Name, 427 JobID: job.ID, 428 Job: job, 429 ID: uuid.Generate(), 430 NodeID: nodes[4].Node.ID, 431 }, 432 } 433 434 // Reset scores 435 for _, node := range nodes { 436 node.Scores = nil 437 node.FinalScore = 0 438 } 439 static = NewStaticRankIterator(ctx, nodes) 440 spreadIter = NewSpreadIterator(ctx, static) 441 spreadIter.SetJob(job) 442 spreadIter.SetTaskGroup(tg) 443 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 444 out = collectRanked(scoreNorm) 445 446 // Expect dc1 and dc2 to be penalized because they have 3 allocs 447 // dc3 should get a boost because it has 0 allocs 448 expectedScores = map[string]float64{ 449 "dc1": -1, 450 "dc2": -1, 451 "dc3": 1, 452 } 453 for _, rn := range out { 454 require.Equal(t, fmt.Sprintf("%.3f", expectedScores[rn.Node.Datacenter]), fmt.Sprintf("%.3f", rn.FinalScore)) 455 } 456 457 } 458 459 // Test scenarios where the spread iterator sets maximum penalty (-1.0) 460 func TestSpreadIterator_MaxPenalty(t *testing.T) { 461 state, ctx := testContext(t) 462 var nodes []*RankedNode 463 464 // Add nodes in dc3 to the state store 465 for i := 0; i < 5; i++ { 466 node := mock.Node() 467 node.Datacenter = "dc3" 468 if err := state.UpsertNode(uint64(100+i), node); err != nil { 469 t.Fatalf("failed to upsert node: %v", err) 470 } 471 nodes = append(nodes, &RankedNode{Node: node}) 472 } 473 474 static := NewStaticRankIterator(ctx, nodes) 475 476 job := mock.Job() 477 tg := job.TaskGroups[0] 478 job.TaskGroups[0].Count = 5 479 480 // Create spread target of 80% in dc1 481 // and 20% in dc2 482 spread := &structs.Spread{ 483 Weight: 100, 484 Attribute: "${node.datacenter}", 485 SpreadTarget: []*structs.SpreadTarget{ 486 { 487 Value: "dc1", 488 Percent: 80, 489 }, 490 { 491 Value: "dc2", 492 Percent: 20, 493 }, 494 }, 495 } 496 tg.Spreads = []*structs.Spread{spread} 497 spreadIter := NewSpreadIterator(ctx, static) 498 spreadIter.SetJob(job) 499 spreadIter.SetTaskGroup(tg) 500 501 scoreNorm := NewScoreNormalizationIterator(ctx, spreadIter) 502 503 out := collectRanked(scoreNorm) 504 505 // All nodes are in dc3 so score should be -1 506 for _, rn := range out { 507 require.Equal(t, -1.0, rn.FinalScore) 508 } 509 510 // Reset scores 511 for _, node := range nodes { 512 node.Scores = nil 513 node.FinalScore = 0 514 } 515 516 // Create spread on attribute that doesn't exist on any nodes 517 spread = &structs.Spread{ 518 Weight: 100, 519 Attribute: "${meta.foo}", 520 SpreadTarget: []*structs.SpreadTarget{ 521 { 522 Value: "bar", 523 Percent: 80, 524 }, 525 { 526 Value: "baz", 527 Percent: 20, 528 }, 529 }, 530 } 531 532 tg.Spreads = []*structs.Spread{spread} 533 static = NewStaticRankIterator(ctx, nodes) 534 spreadIter = NewSpreadIterator(ctx, static) 535 spreadIter.SetJob(job) 536 spreadIter.SetTaskGroup(tg) 537 scoreNorm = NewScoreNormalizationIterator(ctx, spreadIter) 538 out = collectRanked(scoreNorm) 539 540 // All nodes don't have the spread attribute so score should be -1 541 for _, rn := range out { 542 require.Equal(t, -1.0, rn.FinalScore) 543 } 544 545 }