github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/nomad/blocked_evals_stats_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "math/rand" 6 "reflect" 7 "testing" 8 "testing/quick" 9 "time" 10 11 "github.com/hashicorp/nomad/ci" 12 "github.com/hashicorp/nomad/nomad/mock" 13 "github.com/hashicorp/nomad/nomad/structs" 14 "github.com/stretchr/testify/require" 15 ) 16 17 func now(year int) time.Time { 18 return time.Date(2000+year, 1, 2, 3, 4, 5, 6, time.UTC) 19 } 20 21 func TestBlockedResourceSummary_Add(t *testing.T) { 22 now1 := now(1) 23 now2 := now(2) 24 a := BlockedResourcesSummary{ 25 Timestamp: now1, 26 CPU: 600, 27 MemoryMB: 256, 28 } 29 30 b := BlockedResourcesSummary{ 31 Timestamp: now2, 32 CPU: 250, 33 MemoryMB: 128, 34 } 35 36 result := a.Add(b) 37 38 // a not modified 39 require.Equal(t, 600, a.CPU) 40 require.Equal(t, 256, a.MemoryMB) 41 require.Equal(t, now1, a.Timestamp) 42 43 // b not modified 44 require.Equal(t, 250, b.CPU) 45 require.Equal(t, 128, b.MemoryMB) 46 require.Equal(t, now2, b.Timestamp) 47 48 // result is a + b, using timestamp from b 49 require.Equal(t, 850, result.CPU) 50 require.Equal(t, 384, result.MemoryMB) 51 require.Equal(t, now2, result.Timestamp) 52 } 53 54 func TestBlockedResourceSummary_Add_nil(t *testing.T) { 55 now1 := now(1) 56 b := BlockedResourcesSummary{ 57 Timestamp: now1, 58 CPU: 250, 59 MemoryMB: 128, 60 } 61 62 // zero + b == b 63 result := (BlockedResourcesSummary{}).Add(b) 64 require.Equal(t, now1, result.Timestamp) 65 require.Equal(t, 250, result.CPU) 66 require.Equal(t, 128, result.MemoryMB) 67 } 68 69 func TestBlockedResourceSummary_Subtract(t *testing.T) { 70 now1 := now(1) 71 now2 := now(2) 72 a := BlockedResourcesSummary{ 73 Timestamp: now1, 74 CPU: 600, 75 MemoryMB: 256, 76 } 77 78 b := BlockedResourcesSummary{ 79 Timestamp: now2, 80 CPU: 250, 81 MemoryMB: 120, 82 } 83 84 result := a.Subtract(b) 85 86 // a not modified 87 require.Equal(t, 600, a.CPU) 88 require.Equal(t, 256, a.MemoryMB) 89 require.Equal(t, now1, a.Timestamp) 90 91 // b not modified 92 require.Equal(t, 250, b.CPU) 93 require.Equal(t, 120, b.MemoryMB) 94 require.Equal(t, now2, b.Timestamp) 95 96 // result is a + b, using timestamp from b 97 require.Equal(t, 350, result.CPU) 98 require.Equal(t, 136, result.MemoryMB) 99 require.Equal(t, now2, result.Timestamp) 100 } 101 102 func TestBlockedResourceSummary_IsZero(t *testing.T) { 103 now1 := now(1) 104 105 // cpu and mem zero, timestamp is ignored 106 require.True(t, (&BlockedResourcesSummary{ 107 Timestamp: now1, 108 CPU: 0, 109 MemoryMB: 0, 110 }).IsZero()) 111 112 // cpu non-zero 113 require.False(t, (&BlockedResourcesSummary{ 114 Timestamp: now1, 115 CPU: 1, 116 MemoryMB: 0, 117 }).IsZero()) 118 119 // mem non-zero 120 require.False(t, (&BlockedResourcesSummary{ 121 Timestamp: now1, 122 CPU: 0, 123 MemoryMB: 1, 124 }).IsZero()) 125 } 126 127 func TestBlockedResourceStats_New(t *testing.T) { 128 a := NewBlockedResourcesStats() 129 require.NotNil(t, a.ByJob) 130 require.Empty(t, a.ByJob) 131 require.NotNil(t, a.ByClassInDC) 132 require.Empty(t, a.ByClassInDC) 133 } 134 135 var ( 136 id1 = structs.NamespacedID{ 137 ID: "1", 138 Namespace: "one", 139 } 140 141 id2 = structs.NamespacedID{ 142 ID: "2", 143 Namespace: "two", 144 } 145 146 node1 = classInDC{ 147 dc: "dc1", 148 class: "alpha", 149 } 150 151 node2 = classInDC{ 152 dc: "dc1", 153 class: "beta", 154 } 155 156 node3 = classInDC{ 157 dc: "dc1", 158 class: "", // not set 159 } 160 ) 161 162 func TestBlockedResourceStats_Copy(t *testing.T) { 163 now1 := now(1) 164 now2 := now(2) 165 166 a := NewBlockedResourcesStats() 167 a.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{ 168 id1: { 169 Timestamp: now1, 170 CPU: 100, 171 MemoryMB: 256, 172 }, 173 } 174 a.ByClassInDC = map[classInDC]BlockedResourcesSummary{ 175 node1: { 176 Timestamp: now1, 177 CPU: 300, 178 MemoryMB: 333, 179 }, 180 } 181 182 c := a.Copy() 183 c.ByJob[id1] = BlockedResourcesSummary{ 184 Timestamp: now2, 185 CPU: 888, 186 MemoryMB: 888, 187 } 188 c.ByClassInDC[node1] = BlockedResourcesSummary{ 189 Timestamp: now2, 190 CPU: 999, 191 MemoryMB: 999, 192 } 193 194 // underlying data should have been deep copied 195 require.Equal(t, 100, a.ByJob[id1].CPU) 196 require.Equal(t, 300, a.ByClassInDC[node1].CPU) 197 } 198 199 func TestBlockedResourcesStats_Add(t *testing.T) { 200 a := NewBlockedResourcesStats() 201 a.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{ 202 id1: {Timestamp: now(1), CPU: 111, MemoryMB: 222}, 203 } 204 a.ByClassInDC = map[classInDC]BlockedResourcesSummary{ 205 node1: {Timestamp: now(2), CPU: 333, MemoryMB: 444}, 206 } 207 208 b := NewBlockedResourcesStats() 209 b.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{ 210 id1: {Timestamp: now(3), CPU: 200, MemoryMB: 300}, 211 id2: {Timestamp: now(4), CPU: 400, MemoryMB: 500}, 212 } 213 b.ByClassInDC = map[classInDC]BlockedResourcesSummary{ 214 node1: {Timestamp: now(5), CPU: 600, MemoryMB: 700}, 215 node2: {Timestamp: now(6), CPU: 800, MemoryMB: 900}, 216 } 217 218 t.Run("a add b", func(t *testing.T) { 219 result := a.Add(b) 220 221 require.Equal(t, map[structs.NamespacedID]BlockedResourcesSummary{ 222 id1: {Timestamp: now(3), CPU: 311, MemoryMB: 522}, 223 id2: {Timestamp: now(4), CPU: 400, MemoryMB: 500}, 224 }, result.ByJob) 225 226 require.Equal(t, map[classInDC]BlockedResourcesSummary{ 227 node1: {Timestamp: now(5), CPU: 933, MemoryMB: 1144}, 228 node2: {Timestamp: now(6), CPU: 800, MemoryMB: 900}, 229 }, result.ByClassInDC) 230 }) 231 232 // make sure we handle zeros in both directions 233 // and timestamps originate from rhs 234 t.Run("b add a", func(t *testing.T) { 235 result := b.Add(a) 236 require.Equal(t, map[structs.NamespacedID]BlockedResourcesSummary{ 237 id1: {Timestamp: now(1), CPU: 311, MemoryMB: 522}, 238 id2: {Timestamp: now(4), CPU: 400, MemoryMB: 500}, 239 }, result.ByJob) 240 241 require.Equal(t, map[classInDC]BlockedResourcesSummary{ 242 node1: {Timestamp: now(2), CPU: 933, MemoryMB: 1144}, 243 node2: {Timestamp: now(6), CPU: 800, MemoryMB: 900}, 244 }, result.ByClassInDC) 245 }) 246 } 247 248 func TestBlockedResourcesStats_Add_NoClass(t *testing.T) { 249 a := NewBlockedResourcesStats() 250 a.ByClassInDC = map[classInDC]BlockedResourcesSummary{ 251 node3: {Timestamp: now(1), CPU: 111, MemoryMB: 1111}, 252 } 253 result := a.Add(a) 254 require.Equal(t, map[classInDC]BlockedResourcesSummary{ 255 node3: {Timestamp: now(1), CPU: 222, MemoryMB: 2222}, 256 }, result.ByClassInDC) 257 } 258 259 func TestBlockedResourcesStats_Subtract(t *testing.T) { 260 a := NewBlockedResourcesStats() 261 a.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{ 262 id1: {Timestamp: now(1), CPU: 100, MemoryMB: 100}, 263 id2: {Timestamp: now(2), CPU: 200, MemoryMB: 200}, 264 } 265 a.ByClassInDC = map[classInDC]BlockedResourcesSummary{ 266 node1: {Timestamp: now(3), CPU: 300, MemoryMB: 300}, 267 node2: {Timestamp: now(4), CPU: 400, MemoryMB: 400}, 268 } 269 270 b := NewBlockedResourcesStats() 271 b.ByJob = map[structs.NamespacedID]BlockedResourcesSummary{ 272 id1: {Timestamp: now(5), CPU: 10, MemoryMB: 11}, 273 id2: {Timestamp: now(6), CPU: 12, MemoryMB: 13}, 274 } 275 b.ByClassInDC = map[classInDC]BlockedResourcesSummary{ 276 node1: {Timestamp: now(7), CPU: 14, MemoryMB: 15}, 277 node2: {Timestamp: now(8), CPU: 16, MemoryMB: 17}, 278 } 279 280 result := a.Subtract(b) 281 282 // id1 283 require.Equal(t, now(5), result.ByJob[id1].Timestamp) 284 require.Equal(t, 90, result.ByJob[id1].CPU) 285 require.Equal(t, 89, result.ByJob[id1].MemoryMB) 286 287 // id2 288 require.Equal(t, now(6), result.ByJob[id2].Timestamp) 289 require.Equal(t, 188, result.ByJob[id2].CPU) 290 require.Equal(t, 187, result.ByJob[id2].MemoryMB) 291 292 // node1 293 require.Equal(t, now(7), result.ByClassInDC[node1].Timestamp) 294 require.Equal(t, 286, result.ByClassInDC[node1].CPU) 295 require.Equal(t, 285, result.ByClassInDC[node1].MemoryMB) 296 297 // node2 298 require.Equal(t, now(8), result.ByClassInDC[node2].Timestamp) 299 require.Equal(t, 384, result.ByClassInDC[node2].CPU) 300 require.Equal(t, 383, result.ByClassInDC[node2].MemoryMB) 301 } 302 303 // testBlockedEvalsRandomBlockedEval wraps an eval that is randomly generated. 304 type testBlockedEvalsRandomBlockedEval struct { 305 eval *structs.Evaluation 306 } 307 308 // Generate returns a random eval. 309 func (t testBlockedEvalsRandomBlockedEval) Generate(rand *rand.Rand, _ int) reflect.Value { 310 resourceTypes := []string{"cpu", "memory"} 311 312 // Start with a mock eval. 313 e := mock.BlockedEval() 314 315 // Get how many task groups, datacenters and node classes to generate. 316 // Add 1 to avoid 0. 317 jobCount := rand.Intn(3) + 1 318 tgCount := rand.Intn(10) + 1 319 dcCount := rand.Intn(3) + 1 320 nodeClassCount := rand.Intn(3) + 1 321 322 failedTGAllocs := map[string]*structs.AllocMetric{} 323 324 e.JobID = fmt.Sprintf("job-%d", jobCount) 325 for tg := 1; tg <= tgCount; tg++ { 326 tgName := fmt.Sprintf("group-%d", tg) 327 328 // Get which resource type to use for this task group. 329 // Nomad stops at the first dimension that is exhausted, so only 1 is 330 // added per task group. 331 i := rand.Int() % len(resourceTypes) 332 resourceType := resourceTypes[i] 333 334 failedTGAllocs[tgName] = &structs.AllocMetric{ 335 DimensionExhausted: map[string]int{ 336 resourceType: 1, 337 }, 338 NodesAvailable: map[string]int{}, 339 ClassExhausted: map[string]int{}, 340 } 341 342 for dc := 1; dc <= dcCount; dc++ { 343 dcName := fmt.Sprintf("dc%d", dc) 344 failedTGAllocs[tgName].NodesAvailable[dcName] = 1 345 } 346 347 for nc := 1; nc <= nodeClassCount; nc++ { 348 nodeClassName := fmt.Sprintf("node-class-%d", nc) 349 failedTGAllocs[tgName].ClassExhausted[nodeClassName] = 1 350 } 351 352 // Generate resources for each task. 353 taskCount := rand.Intn(5) + 1 354 resourcesExhausted := map[string]*structs.Resources{} 355 356 for t := 1; t <= taskCount; t++ { 357 task := fmt.Sprintf("task-%d", t) 358 resourcesExhausted[task] = &structs.Resources{} 359 360 resourceAmount := rand.Intn(1000) 361 switch resourceType { 362 case "cpu": 363 resourcesExhausted[task].CPU = resourceAmount 364 case "memory": 365 resourcesExhausted[task].MemoryMB = resourceAmount 366 } 367 } 368 failedTGAllocs[tgName].ResourcesExhausted = resourcesExhausted 369 } 370 e.FailedTGAllocs = failedTGAllocs 371 t.eval = e 372 return reflect.ValueOf(t) 373 } 374 375 // clearTimestampFromBlockedResourceStats set timestamp metrics to zero to 376 // avoid invalid comparisons. 377 func clearTimestampFromBlockedResourceStats(b *BlockedResourcesStats) { 378 for k, v := range b.ByJob { 379 v.Timestamp = time.Time{} 380 b.ByJob[k] = v 381 } 382 for k, v := range b.ByClassInDC { 383 v.Timestamp = time.Time{} 384 b.ByClassInDC[k] = v 385 } 386 } 387 388 // TestBlockedEvalsStats_BlockedResources generates random evals and processes 389 // them using the expected code paths and a manual check of the expeceted result. 390 func TestBlockedEvalsStats_BlockedResources(t *testing.T) { 391 ci.Parallel(t) 392 blocked, _ := testBlockedEvals(t) 393 394 // evalHistory stores all evals generated during the test. 395 var evalHistory []*structs.Evaluation 396 397 // blockedEvals keeps track if evals are blocked or unblocked. 398 blockedEvals := map[string]bool{} 399 400 // blockAndUntrack processes the generated evals in order using a 401 // BlockedEvals instance. 402 blockAndUntrack := func(testEval testBlockedEvalsRandomBlockedEval, block bool, unblockIdx uint16) *BlockedResourcesStats { 403 if block || len(evalHistory) == 0 { 404 blocked.Block(testEval.eval) 405 } else { 406 i := int(unblockIdx) % len(evalHistory) 407 eval := evalHistory[i] 408 blocked.Untrack(eval.JobID, eval.Namespace) 409 } 410 411 // Remove zero stats from unblocked evals. 412 blocked.pruneStats(time.Now().UTC()) 413 414 result := blocked.Stats().BlockedResources 415 clearTimestampFromBlockedResourceStats(result) 416 return result 417 } 418 419 // manualCount processes only the blocked evals and generate a 420 // BlockedResourcesStats result directly from the eval history. 421 manualCount := func(testEval testBlockedEvalsRandomBlockedEval, block bool, unblockIdx uint16) *BlockedResourcesStats { 422 if block || len(evalHistory) == 0 { 423 evalHistory = append(evalHistory, testEval.eval) 424 425 // Find and unblock evals for the same job. 426 for _, e := range evalHistory { 427 if e.Namespace == testEval.eval.Namespace && e.JobID == testEval.eval.JobID { 428 blockedEvals[e.ID] = false 429 } 430 } 431 blockedEvals[testEval.eval.ID] = true 432 } else { 433 i := int(unblockIdx) % len(evalHistory) 434 eval := evalHistory[i] 435 436 // Find and unlock all evals for this job. 437 for _, e := range evalHistory { 438 if e.Namespace == eval.Namespace && e.JobID == eval.JobID { 439 blockedEvals[e.ID] = false 440 } 441 } 442 } 443 444 result := NewBlockedResourcesStats() 445 for _, e := range evalHistory { 446 if !blockedEvals[e.ID] { 447 continue 448 } 449 result = result.Add(generateResourceStats(e)) 450 } 451 clearTimestampFromBlockedResourceStats(result) 452 return result 453 } 454 455 err := quick.CheckEqual(blockAndUntrack, manualCount, nil) 456 if err != nil { 457 t.Error(err) 458 } 459 }