github.com/kubewharf/katalyst-core@v0.5.3/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go (about) 1 //go:build linux 2 // +build linux 3 4 /* 5 Copyright 2022 The Katalyst Authors. 6 7 Licensed under the Apache License, Version 2.0 (the "License"); 8 you may not use this file except in compliance with the License. 9 You may obtain a copy of the License at 10 11 http://www.apache.org/licenses/LICENSE-2.0 12 13 Unless required by applicable law or agreed to in writing, software 14 distributed under the License is distributed on an "AS IS" BASIS, 15 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 See the License for the specific language governing permissions and 17 limitations under the License. 18 */ 19 20 package dynamicpolicy 21 22 import ( 23 "context" 24 "fmt" 25 "io/ioutil" 26 "os" 27 "path/filepath" 28 "strings" 29 "testing" 30 "time" 31 32 "github.com/stretchr/testify/require" 33 v1 "k8s.io/api/core/v1" 34 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 35 "k8s.io/apimachinery/pkg/util/uuid" 36 pluginapi "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" 37 utilfs "k8s.io/kubernetes/pkg/util/filesystem" 38 39 "github.com/kubewharf/katalyst-api/pkg/consts" 40 cpuconsts "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/consts" 41 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/calculator" 42 advisorapi "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpuadvisor" 43 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state" 44 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/validator" 45 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util" 46 "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic" 47 "github.com/kubewharf/katalyst-core/pkg/config/generic" 48 "github.com/kubewharf/katalyst-core/pkg/metaserver" 49 "github.com/kubewharf/katalyst-core/pkg/metaserver/agent" 50 "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod" 51 "github.com/kubewharf/katalyst-core/pkg/metrics" 52 "github.com/kubewharf/katalyst-core/pkg/util/cgroup/common" 53 cgroupcm "github.com/kubewharf/katalyst-core/pkg/util/cgroup/common" 54 cgroupcmutils "github.com/kubewharf/katalyst-core/pkg/util/cgroup/manager" 55 "github.com/kubewharf/katalyst-core/pkg/util/machine" 56 ) 57 58 const ( 59 podDebugAnnoKey = "qrm.katalyst.kubewharf.io/debug_pod" 60 ) 61 62 func getTestDynamicPolicyWithInitialization(topology *machine.CPUTopology, stateFileDirectory string) (*DynamicPolicy, error) { 63 dynamicPolicy, err := getTestDynamicPolicyWithoutInitialization(topology, stateFileDirectory) 64 if err != nil { 65 return nil, err 66 } 67 68 err = dynamicPolicy.initReclaimPool() 69 if err != nil { 70 return nil, err 71 } 72 73 dynamicPolicy.stopCh = make(chan struct{}) 74 return dynamicPolicy, nil 75 } 76 77 func getTestDynamicPolicyWithoutInitialization(topology *machine.CPUTopology, stateFileDirectory string) (*DynamicPolicy, error) { 78 stateImpl, err := state.NewCheckpointState(stateFileDirectory, cpuPluginStateFileName, cpuconsts.CPUResourcePluginPolicyNameDynamic, topology, false) 79 if err != nil { 80 return nil, err 81 } 82 83 machineInfo := &machine.KatalystMachineInfo{ 84 CPUTopology: topology, 85 } 86 87 reservedCPUs, _, err := calculator.TakeHTByNUMABalance(machineInfo, machineInfo.CPUDetails.CPUs().Clone(), 2) 88 if err != nil { 89 return nil, err 90 } 91 92 qosConfig := generic.NewQoSConfiguration() 93 dynamicConfig := dynamic.NewDynamicAgentConfiguration() 94 95 policyImplement := &DynamicPolicy{ 96 machineInfo: machineInfo, 97 qosConfig: qosConfig, 98 dynamicConfig: dynamicConfig, 99 state: stateImpl, 100 advisorValidator: validator.NewCPUAdvisorValidator(stateImpl, machineInfo), 101 reservedCPUs: reservedCPUs, 102 emitter: metrics.DummyMetrics{}, 103 podDebugAnnoKeys: []string{podDebugAnnoKey}, 104 } 105 106 state.SetContainerRequestedCores(policyImplement.getContainerRequestedCores) 107 108 // register allocation behaviors for pods with different QoS level 109 policyImplement.allocationHandlers = map[string]util.AllocationHandler{ 110 consts.PodAnnotationQoSLevelSharedCores: policyImplement.sharedCoresAllocationHandler, 111 consts.PodAnnotationQoSLevelDedicatedCores: policyImplement.dedicatedCoresAllocationHandler, 112 consts.PodAnnotationQoSLevelReclaimedCores: policyImplement.reclaimedCoresAllocationHandler, 113 } 114 115 // register hint providers for pods with different QoS level 116 policyImplement.hintHandlers = map[string]util.HintHandler{ 117 consts.PodAnnotationQoSLevelSharedCores: policyImplement.sharedCoresHintHandler, 118 consts.PodAnnotationQoSLevelDedicatedCores: policyImplement.dedicatedCoresHintHandler, 119 consts.PodAnnotationQoSLevelReclaimedCores: policyImplement.reclaimedCoresHintHandler, 120 } 121 122 policyImplement.metaServer = &metaserver.MetaServer{ 123 MetaAgent: &agent.MetaAgent{ 124 PodFetcher: &pod.PodFetcherStub{}, 125 }, 126 } 127 128 return policyImplement, nil 129 } 130 131 func TestInitPoolAndCalculator(t *testing.T) { 132 t.Parallel() 133 134 as := require.New(t) 135 136 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 137 as.Nil(err) 138 139 tmpDir, err := ioutil.TempDir("", "checkpoint-TestInitPoolAndCalculator") 140 as.Nil(err) 141 defer func() { _ = os.RemoveAll(tmpDir) }() 142 143 policyImpl, err := getTestDynamicPolicyWithoutInitialization(cpuTopology, tmpDir) 144 as.Nil(err) 145 146 err = policyImpl.initReclaimPool() 147 as.Nil(err) 148 149 reclaimPoolAllocationInfo := policyImpl.state.GetAllocationInfo(state.PoolNameReclaim, "") 150 151 as.NotNil(reclaimPoolAllocationInfo) 152 153 as.Equal(reclaimPoolAllocationInfo.AllocationResult.Size(), reservedReclaimedCPUsSize) 154 } 155 156 func TestRemovePod(t *testing.T) { 157 t.Parallel() 158 159 as := require.New(t) 160 161 tmpDir, err := ioutil.TempDir("", "checkpoint-TestRemovePod") 162 as.Nil(err) 163 defer func() { _ = os.RemoveAll(tmpDir) }() 164 165 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 166 as.Nil(err) 167 168 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) 169 as.Nil(err) 170 171 testName := "test" 172 173 // test for gt 174 req := &pluginapi.ResourceRequest{ 175 PodUid: string(uuid.NewUUID()), 176 PodNamespace: testName, 177 PodName: testName, 178 ContainerName: testName, 179 ContainerType: pluginapi.ContainerType_MAIN, 180 ContainerIndex: 0, 181 ResourceName: string(v1.ResourceCPU), 182 ResourceRequests: map[string]float64{ 183 string(v1.ResourceCPU): 2, 184 }, 185 Labels: map[string]string{}, 186 Annotations: map[string]string{}, 187 } 188 189 _, err = dynamicPolicy.Allocate(context.Background(), req) 190 as.Nil(err) 191 192 resp, err := dynamicPolicy.GetTopologyAwareResources(context.Background(), &pluginapi.GetTopologyAwareResourcesRequest{ 193 PodUid: req.PodUid, 194 ContainerName: testName, 195 }) 196 as.Nil(err) 197 198 as.Equal(&pluginapi.GetTopologyAwareResourcesResponse{ 199 PodUid: req.PodUid, 200 PodNamespace: testName, 201 PodName: testName, 202 ContainerTopologyAwareResources: &pluginapi.ContainerTopologyAwareResources{ 203 ContainerName: testName, 204 AllocatedResources: map[string]*pluginapi.TopologyAwareResource{ 205 string(v1.ResourceCPU): { 206 IsNodeResource: false, 207 IsScalarResource: true, 208 AggregatedQuantity: 14, 209 OriginalAggregatedQuantity: 14, 210 TopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 211 {ResourceValue: 3, Node: 0}, 212 {ResourceValue: 3, Node: 1}, 213 {ResourceValue: 4, Node: 2}, 214 {ResourceValue: 4, Node: 3}, 215 }, 216 OriginalTopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 217 {ResourceValue: 3, Node: 0}, 218 {ResourceValue: 3, Node: 1}, 219 {ResourceValue: 4, Node: 2}, 220 {ResourceValue: 4, Node: 3}, 221 }, 222 }, 223 }, 224 }, 225 }, resp) 226 227 _, _ = dynamicPolicy.RemovePod(context.Background(), &pluginapi.RemovePodRequest{ 228 PodUid: req.PodUid, 229 }) 230 231 _, err = dynamicPolicy.GetTopologyAwareResources(context.Background(), &pluginapi.GetTopologyAwareResourcesRequest{ 232 PodUid: req.PodUid, 233 ContainerName: testName, 234 }) 235 as.NotNil(err) 236 as.True(strings.Contains(err.Error(), "is not show up in cpu plugin state")) 237 } 238 239 func TestAllocate(t *testing.T) { 240 t.Parallel() 241 242 as := require.New(t) 243 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 244 as.Nil(err) 245 246 testName := "test" 247 248 testCases := []struct { 249 description string 250 req *pluginapi.ResourceRequest 251 expectedResp *pluginapi.ResourceAllocationResponse 252 cpuTopology *machine.CPUTopology 253 enhancementDefaultValues map[string]string 254 }{ 255 { 256 description: "req for init container", 257 req: &pluginapi.ResourceRequest{ 258 PodUid: string(uuid.NewUUID()), 259 PodNamespace: testName, 260 PodName: testName, 261 ContainerName: testName, 262 ContainerType: pluginapi.ContainerType_INIT, 263 ContainerIndex: 0, 264 ResourceName: string(v1.ResourceCPU), 265 ResourceRequests: map[string]float64{ 266 string(v1.ResourceCPU): 1, 267 }, 268 }, 269 expectedResp: &pluginapi.ResourceAllocationResponse{ 270 PodNamespace: testName, 271 PodName: testName, 272 ContainerName: testName, 273 ContainerType: pluginapi.ContainerType_INIT, 274 ContainerIndex: 0, 275 ResourceName: string(v1.ResourceCPU), 276 AllocationResult: nil, 277 Labels: map[string]string{ 278 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 279 }, 280 Annotations: map[string]string{ 281 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 282 }, 283 }, 284 cpuTopology: cpuTopology, 285 }, 286 { 287 description: "req for container of debug pod", 288 req: &pluginapi.ResourceRequest{ 289 PodUid: string(uuid.NewUUID()), 290 PodNamespace: testName, 291 PodName: testName, 292 ContainerName: testName, 293 ContainerType: pluginapi.ContainerType_MAIN, 294 ContainerIndex: 0, 295 ResourceName: string(v1.ResourceCPU), 296 ResourceRequests: map[string]float64{ 297 string(v1.ResourceCPU): 1, 298 }, 299 Annotations: map[string]string{ 300 podDebugAnnoKey: "", 301 }, 302 }, 303 expectedResp: &pluginapi.ResourceAllocationResponse{ 304 PodNamespace: testName, 305 PodName: testName, 306 ContainerName: testName, 307 ContainerType: pluginapi.ContainerType_MAIN, 308 ContainerIndex: 0, 309 ResourceName: string(v1.ResourceCPU), 310 AllocationResult: &pluginapi.ResourceAllocation{ 311 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 312 string(v1.ResourceCPU): { 313 IsNodeResource: false, 314 IsScalarResource: true, 315 }, 316 }, 317 }, 318 Labels: map[string]string{ 319 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 320 }, 321 Annotations: map[string]string{ 322 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 323 }, 324 }, 325 cpuTopology: cpuTopology, 326 }, 327 { 328 description: "req for shared_cores main container", 329 req: &pluginapi.ResourceRequest{ 330 PodUid: string(uuid.NewUUID()), 331 PodNamespace: testName, 332 PodName: testName, 333 ContainerName: testName, 334 ContainerType: pluginapi.ContainerType_MAIN, 335 ContainerIndex: 0, 336 ResourceName: string(v1.ResourceCPU), 337 ResourceRequests: map[string]float64{ 338 string(v1.ResourceCPU): 2, 339 }, 340 }, 341 expectedResp: &pluginapi.ResourceAllocationResponse{ 342 PodNamespace: testName, 343 PodName: testName, 344 ContainerName: testName, 345 ContainerType: pluginapi.ContainerType_MAIN, 346 ContainerIndex: 0, 347 ResourceName: string(v1.ResourceCPU), 348 AllocationResult: &pluginapi.ResourceAllocation{ 349 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 350 string(v1.ResourceCPU): { 351 OciPropertyName: util.OCIPropertyNameCPUSetCPUs, 352 IsNodeResource: false, 353 IsScalarResource: true, 354 AllocatedQuantity: 14, // ramp up 355 AllocationResult: machine.NewCPUSet(1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15).String(), 356 ResourceHints: &pluginapi.ListOfTopologyHints{ 357 Hints: []*pluginapi.TopologyHint{nil}, 358 }, 359 }, 360 }, 361 }, 362 Labels: map[string]string{ 363 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 364 }, 365 Annotations: map[string]string{ 366 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 367 }, 368 }, 369 cpuTopology: cpuTopology, 370 }, 371 { 372 description: "req for reclaimed_cores main container", 373 req: &pluginapi.ResourceRequest{ 374 PodUid: string(uuid.NewUUID()), 375 PodNamespace: testName, 376 PodName: testName, 377 ContainerName: testName, 378 ContainerType: pluginapi.ContainerType_MAIN, 379 ContainerIndex: 0, 380 ResourceName: string(v1.ResourceCPU), 381 ResourceRequests: map[string]float64{ 382 string(v1.ResourceCPU): 2, 383 }, 384 Annotations: map[string]string{ 385 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 386 }, 387 Labels: map[string]string{ 388 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 389 }, 390 }, 391 expectedResp: &pluginapi.ResourceAllocationResponse{ 392 PodNamespace: testName, 393 PodName: testName, 394 ContainerName: testName, 395 ContainerType: pluginapi.ContainerType_MAIN, 396 ContainerIndex: 0, 397 ResourceName: string(v1.ResourceCPU), 398 AllocationResult: &pluginapi.ResourceAllocation{ 399 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 400 string(v1.ResourceCPU): { 401 OciPropertyName: util.OCIPropertyNameCPUSetCPUs, 402 IsNodeResource: false, 403 IsScalarResource: true, 404 AllocatedQuantity: 4, 405 AllocationResult: machine.NewCPUSet(1, 3, 9, 11).String(), 406 ResourceHints: &pluginapi.ListOfTopologyHints{ 407 Hints: []*pluginapi.TopologyHint{nil}, 408 }, 409 }, 410 }, 411 }, 412 Labels: map[string]string{ 413 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 414 }, 415 Annotations: map[string]string{ 416 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 417 }, 418 }, 419 cpuTopology: cpuTopology, 420 }, 421 { 422 description: "req for dedicated_cores with numa_binding & numa_exclusive main container", 423 req: &pluginapi.ResourceRequest{ 424 PodUid: string(uuid.NewUUID()), 425 PodNamespace: testName, 426 PodName: testName, 427 ContainerName: testName, 428 ContainerType: pluginapi.ContainerType_MAIN, 429 ContainerIndex: 0, 430 ResourceName: string(v1.ResourceCPU), 431 Hint: &pluginapi.TopologyHint{ 432 Nodes: []uint64{0}, 433 Preferred: true, 434 }, 435 ResourceRequests: map[string]float64{ 436 string(v1.ResourceCPU): 2, 437 }, 438 Annotations: map[string]string{ 439 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 440 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "true"}`, 441 }, 442 Labels: map[string]string{ 443 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 444 }, 445 }, 446 expectedResp: &pluginapi.ResourceAllocationResponse{ 447 PodNamespace: testName, 448 PodName: testName, 449 ContainerName: testName, 450 ContainerType: pluginapi.ContainerType_MAIN, 451 ContainerIndex: 0, 452 ResourceName: string(v1.ResourceCPU), 453 AllocationResult: &pluginapi.ResourceAllocation{ 454 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 455 string(v1.ResourceCPU): { 456 OciPropertyName: util.OCIPropertyNameCPUSetCPUs, 457 IsNodeResource: false, 458 IsScalarResource: true, 459 AllocatedQuantity: 3, 460 AllocationResult: machine.NewCPUSet(1, 8, 9).String(), 461 ResourceHints: &pluginapi.ListOfTopologyHints{ 462 Hints: []*pluginapi.TopologyHint{ 463 { 464 Nodes: []uint64{0}, 465 Preferred: true, 466 }, 467 }, 468 }, 469 }, 470 }, 471 }, 472 Labels: map[string]string{ 473 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 474 }, 475 Annotations: map[string]string{ 476 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 477 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 478 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 479 }, 480 }, 481 cpuTopology: cpuTopology, 482 }, 483 { 484 description: "req for dedicated_cores with numa_binding & not numa_exclusive main container", 485 req: &pluginapi.ResourceRequest{ 486 PodUid: string(uuid.NewUUID()), 487 PodNamespace: testName, 488 PodName: testName, 489 ContainerName: testName, 490 ContainerType: pluginapi.ContainerType_MAIN, 491 ContainerIndex: 0, 492 ResourceName: string(v1.ResourceCPU), 493 Hint: &pluginapi.TopologyHint{ 494 Nodes: []uint64{0}, 495 Preferred: true, 496 }, 497 ResourceRequests: map[string]float64{ 498 string(v1.ResourceCPU): 2, 499 }, 500 Annotations: map[string]string{ 501 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 502 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, 503 }, 504 Labels: map[string]string{ 505 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 506 }, 507 }, 508 expectedResp: &pluginapi.ResourceAllocationResponse{ 509 PodNamespace: testName, 510 PodName: testName, 511 ContainerName: testName, 512 ContainerType: pluginapi.ContainerType_MAIN, 513 ContainerIndex: 0, 514 ResourceName: string(v1.ResourceCPU), 515 AllocationResult: &pluginapi.ResourceAllocation{ 516 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 517 string(v1.ResourceCPU): { 518 OciPropertyName: util.OCIPropertyNameCPUSetCPUs, 519 IsNodeResource: false, 520 IsScalarResource: true, 521 AllocatedQuantity: 2, 522 AllocationResult: machine.NewCPUSet(1, 9).String(), 523 ResourceHints: &pluginapi.ListOfTopologyHints{ 524 Hints: []*pluginapi.TopologyHint{ 525 { 526 Nodes: []uint64{0}, 527 Preferred: true, 528 }, 529 }, 530 }, 531 }, 532 }, 533 }, 534 Labels: map[string]string{ 535 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 536 }, 537 Annotations: map[string]string{ 538 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 539 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 540 consts.PodAnnotationMemoryEnhancementNumaExclusive: "false", 541 }, 542 }, 543 cpuTopology: cpuTopology, 544 }, 545 { 546 description: "req for dedicated_cores with numa_binding & default numa_exclusive true main container", 547 req: &pluginapi.ResourceRequest{ 548 PodUid: string(uuid.NewUUID()), 549 PodNamespace: testName, 550 PodName: testName, 551 ContainerName: testName, 552 ContainerType: pluginapi.ContainerType_MAIN, 553 ContainerIndex: 0, 554 ResourceName: string(v1.ResourceCPU), 555 Hint: &pluginapi.TopologyHint{ 556 Nodes: []uint64{0}, 557 Preferred: true, 558 }, 559 ResourceRequests: map[string]float64{ 560 string(v1.ResourceCPU): 2, 561 }, 562 Annotations: map[string]string{ 563 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 564 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, 565 }, 566 Labels: map[string]string{ 567 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 568 }, 569 }, 570 expectedResp: &pluginapi.ResourceAllocationResponse{ 571 PodNamespace: testName, 572 PodName: testName, 573 ContainerName: testName, 574 ContainerType: pluginapi.ContainerType_MAIN, 575 ContainerIndex: 0, 576 ResourceName: string(v1.ResourceCPU), 577 AllocationResult: &pluginapi.ResourceAllocation{ 578 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 579 string(v1.ResourceCPU): { 580 OciPropertyName: util.OCIPropertyNameCPUSetCPUs, 581 IsNodeResource: false, 582 IsScalarResource: true, 583 AllocatedQuantity: 3, 584 AllocationResult: machine.NewCPUSet(1, 8, 9).String(), 585 ResourceHints: &pluginapi.ListOfTopologyHints{ 586 Hints: []*pluginapi.TopologyHint{ 587 { 588 Nodes: []uint64{0}, 589 Preferred: true, 590 }, 591 }, 592 }, 593 }, 594 }, 595 }, 596 Labels: map[string]string{ 597 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 598 }, 599 Annotations: map[string]string{ 600 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 601 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 602 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 603 }, 604 }, 605 cpuTopology: cpuTopology, 606 enhancementDefaultValues: map[string]string{ 607 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 608 }, 609 }, 610 { 611 description: "req for dedicated_cores with numa_binding without default numa_exclusive main container", 612 req: &pluginapi.ResourceRequest{ 613 PodUid: string(uuid.NewUUID()), 614 PodNamespace: testName, 615 PodName: testName, 616 ContainerName: testName, 617 ContainerType: pluginapi.ContainerType_MAIN, 618 ContainerIndex: 0, 619 ResourceName: string(v1.ResourceCPU), 620 Hint: &pluginapi.TopologyHint{ 621 Nodes: []uint64{0}, 622 Preferred: true, 623 }, 624 ResourceRequests: map[string]float64{ 625 string(v1.ResourceCPU): 2, 626 }, 627 Annotations: map[string]string{ 628 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 629 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, 630 }, 631 Labels: map[string]string{ 632 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 633 }, 634 }, 635 expectedResp: &pluginapi.ResourceAllocationResponse{ 636 PodNamespace: testName, 637 PodName: testName, 638 ContainerName: testName, 639 ContainerType: pluginapi.ContainerType_MAIN, 640 ContainerIndex: 0, 641 ResourceName: string(v1.ResourceCPU), 642 AllocationResult: &pluginapi.ResourceAllocation{ 643 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 644 string(v1.ResourceCPU): { 645 OciPropertyName: util.OCIPropertyNameCPUSetCPUs, 646 IsNodeResource: false, 647 IsScalarResource: true, 648 AllocatedQuantity: 2, 649 AllocationResult: machine.NewCPUSet(1, 9).String(), 650 ResourceHints: &pluginapi.ListOfTopologyHints{ 651 Hints: []*pluginapi.TopologyHint{ 652 { 653 Nodes: []uint64{0}, 654 Preferred: true, 655 }, 656 }, 657 }, 658 }, 659 }, 660 }, 661 Labels: map[string]string{ 662 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 663 }, 664 Annotations: map[string]string{ 665 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 666 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 667 }, 668 }, 669 cpuTopology: cpuTopology, 670 }, 671 } 672 673 for _, tc := range testCases { 674 tmpDir, err := ioutil.TempDir("", "checkpoint-TestAllocate") 675 as.Nil(err) 676 677 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(tc.cpuTopology, tmpDir) 678 as.Nil(err) 679 680 if tc.enhancementDefaultValues != nil { 681 dynamicPolicy.qosConfig.QoSEnhancementDefaultValues = tc.enhancementDefaultValues 682 } 683 684 resp, err := dynamicPolicy.Allocate(context.Background(), tc.req) 685 as.Nil(err) 686 687 tc.expectedResp.PodUid = tc.req.PodUid 688 as.Equalf(tc.expectedResp, resp, "failed in test case: %s", tc.description) 689 690 _ = os.RemoveAll(tmpDir) 691 } 692 } 693 694 func TestGetTopologyHints(t *testing.T) { 695 t.Parallel() 696 697 as := require.New(t) 698 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 699 as.Nil(err) 700 701 testName := "test" 702 703 testCases := []struct { 704 description string 705 req *pluginapi.ResourceRequest 706 expectedResp *pluginapi.ResourceHintsResponse 707 cpuTopology *machine.CPUTopology 708 enhancementDefaultValues map[string]string 709 }{ 710 { 711 description: "req for container of debug pod", 712 req: &pluginapi.ResourceRequest{ 713 PodUid: string(uuid.NewUUID()), 714 PodNamespace: testName, 715 PodName: testName, 716 ContainerName: testName, 717 ContainerType: pluginapi.ContainerType_MAIN, 718 ContainerIndex: 0, 719 ResourceName: string(v1.ResourceCPU), 720 ResourceRequests: map[string]float64{ 721 string(v1.ResourceCPU): 2, 722 }, 723 Annotations: map[string]string{ 724 podDebugAnnoKey: "", 725 }, 726 }, 727 expectedResp: &pluginapi.ResourceHintsResponse{ 728 PodNamespace: testName, 729 PodName: testName, 730 ContainerName: testName, 731 ContainerType: pluginapi.ContainerType_MAIN, 732 ContainerIndex: 0, 733 ResourceName: string(v1.ResourceCPU), 734 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 735 string(v1.ResourceCPU): nil, 736 }, 737 Labels: map[string]string{ 738 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 739 }, 740 Annotations: map[string]string{ 741 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 742 }, 743 }, 744 cpuTopology: cpuTopology, 745 }, 746 { 747 description: "req for reclaimed_cores main container", 748 req: &pluginapi.ResourceRequest{ 749 PodUid: string(uuid.NewUUID()), 750 PodNamespace: testName, 751 PodName: testName, 752 ContainerName: testName, 753 ContainerType: pluginapi.ContainerType_MAIN, 754 ContainerIndex: 0, 755 ResourceName: string(v1.ResourceCPU), 756 ResourceRequests: map[string]float64{ 757 string(v1.ResourceCPU): 2, 758 }, 759 Labels: map[string]string{ 760 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 761 }, 762 Annotations: map[string]string{ 763 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 764 }, 765 }, 766 expectedResp: &pluginapi.ResourceHintsResponse{ 767 PodNamespace: testName, 768 PodName: testName, 769 ContainerName: testName, 770 ContainerType: pluginapi.ContainerType_MAIN, 771 ContainerIndex: 0, 772 ResourceName: string(v1.ResourceCPU), 773 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 774 string(v1.ResourceCPU): nil, 775 }, 776 Labels: map[string]string{ 777 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 778 }, 779 Annotations: map[string]string{ 780 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 781 }, 782 }, 783 cpuTopology: cpuTopology, 784 }, 785 { 786 description: "req for dedicated_cores with numa_binding & numa_exclusive main container", 787 req: &pluginapi.ResourceRequest{ 788 PodUid: string(uuid.NewUUID()), 789 PodNamespace: testName, 790 PodName: testName, 791 ContainerName: testName, 792 ContainerType: pluginapi.ContainerType_MAIN, 793 ContainerIndex: 0, 794 ResourceName: string(v1.ResourceCPU), 795 ResourceRequests: map[string]float64{ 796 string(v1.ResourceCPU): 2, 797 }, 798 Annotations: map[string]string{ 799 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 800 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "true"}`, 801 }, 802 Labels: map[string]string{ 803 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 804 }, 805 }, 806 expectedResp: &pluginapi.ResourceHintsResponse{ 807 PodNamespace: testName, 808 PodName: testName, 809 ContainerName: testName, 810 ContainerType: pluginapi.ContainerType_MAIN, 811 ContainerIndex: 0, 812 ResourceName: string(v1.ResourceCPU), 813 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 814 string(v1.ResourceCPU): { 815 Hints: []*pluginapi.TopologyHint{ 816 { 817 Nodes: []uint64{0}, 818 Preferred: true, 819 }, 820 { 821 Nodes: []uint64{1}, 822 Preferred: true, 823 }, 824 { 825 Nodes: []uint64{2}, 826 Preferred: true, 827 }, 828 { 829 Nodes: []uint64{3}, 830 Preferred: true, 831 }, 832 { 833 Nodes: []uint64{0, 1}, 834 Preferred: false, 835 }, 836 { 837 Nodes: []uint64{2, 3}, 838 Preferred: false, 839 }, 840 { 841 Nodes: []uint64{0, 1, 2}, 842 Preferred: false, 843 }, 844 { 845 Nodes: []uint64{0, 1, 3}, 846 Preferred: false, 847 }, 848 { 849 Nodes: []uint64{0, 2, 3}, 850 Preferred: false, 851 }, 852 { 853 Nodes: []uint64{1, 2, 3}, 854 Preferred: false, 855 }, 856 { 857 Nodes: []uint64{0, 1, 2, 3}, 858 Preferred: false, 859 }, 860 }, 861 }, 862 }, 863 Labels: map[string]string{ 864 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 865 }, 866 Annotations: map[string]string{ 867 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 868 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 869 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 870 }, 871 }, 872 cpuTopology: cpuTopology, 873 }, 874 { 875 description: "req for dedicated_cores with numa_binding & not numa_exclusive main container", 876 req: &pluginapi.ResourceRequest{ 877 PodUid: string(uuid.NewUUID()), 878 PodNamespace: testName, 879 PodName: testName, 880 ContainerName: testName, 881 ContainerType: pluginapi.ContainerType_MAIN, 882 ContainerIndex: 0, 883 ResourceName: string(v1.ResourceCPU), 884 ResourceRequests: map[string]float64{ 885 string(v1.ResourceCPU): 2, 886 }, 887 Annotations: map[string]string{ 888 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 889 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, 890 }, 891 Labels: map[string]string{ 892 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 893 }, 894 }, 895 expectedResp: &pluginapi.ResourceHintsResponse{ 896 PodNamespace: testName, 897 PodName: testName, 898 ContainerName: testName, 899 ContainerType: pluginapi.ContainerType_MAIN, 900 ContainerIndex: 0, 901 ResourceName: string(v1.ResourceCPU), 902 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 903 string(v1.ResourceCPU): { 904 Hints: []*pluginapi.TopologyHint{ 905 { 906 Nodes: []uint64{0}, 907 Preferred: true, 908 }, 909 { 910 Nodes: []uint64{1}, 911 Preferred: true, 912 }, 913 { 914 Nodes: []uint64{2}, 915 Preferred: true, 916 }, 917 { 918 Nodes: []uint64{3}, 919 Preferred: true, 920 }, 921 }, 922 }, 923 }, 924 Labels: map[string]string{ 925 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 926 }, 927 Annotations: map[string]string{ 928 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 929 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 930 consts.PodAnnotationMemoryEnhancementNumaExclusive: "false", 931 }, 932 }, 933 cpuTopology: cpuTopology, 934 }, 935 { 936 description: "req for dedicated_cores with numa_binding & default numa_exclusive true main container", 937 req: &pluginapi.ResourceRequest{ 938 PodUid: string(uuid.NewUUID()), 939 PodNamespace: testName, 940 PodName: testName, 941 ContainerName: testName, 942 ContainerType: pluginapi.ContainerType_MAIN, 943 ContainerIndex: 0, 944 ResourceName: string(v1.ResourceCPU), 945 ResourceRequests: map[string]float64{ 946 string(v1.ResourceCPU): 2, 947 }, 948 Annotations: map[string]string{ 949 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 950 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, 951 }, 952 Labels: map[string]string{ 953 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 954 }, 955 }, 956 expectedResp: &pluginapi.ResourceHintsResponse{ 957 PodNamespace: testName, 958 PodName: testName, 959 ContainerName: testName, 960 ContainerType: pluginapi.ContainerType_MAIN, 961 ContainerIndex: 0, 962 ResourceName: string(v1.ResourceCPU), 963 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 964 string(v1.ResourceCPU): { 965 Hints: []*pluginapi.TopologyHint{ 966 { 967 Nodes: []uint64{0}, 968 Preferred: true, 969 }, 970 { 971 Nodes: []uint64{1}, 972 Preferred: true, 973 }, 974 { 975 Nodes: []uint64{2}, 976 Preferred: true, 977 }, 978 { 979 Nodes: []uint64{3}, 980 Preferred: true, 981 }, 982 { 983 Nodes: []uint64{0, 1}, 984 Preferred: false, 985 }, 986 { 987 Nodes: []uint64{2, 3}, 988 Preferred: false, 989 }, 990 { 991 Nodes: []uint64{0, 1, 2}, 992 Preferred: false, 993 }, 994 { 995 Nodes: []uint64{0, 1, 3}, 996 Preferred: false, 997 }, 998 { 999 Nodes: []uint64{0, 2, 3}, 1000 Preferred: false, 1001 }, 1002 { 1003 Nodes: []uint64{1, 2, 3}, 1004 Preferred: false, 1005 }, 1006 { 1007 Nodes: []uint64{0, 1, 2, 3}, 1008 Preferred: false, 1009 }, 1010 }, 1011 }, 1012 }, 1013 Labels: map[string]string{ 1014 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1015 }, 1016 Annotations: map[string]string{ 1017 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1018 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 1019 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1020 }, 1021 }, 1022 cpuTopology: cpuTopology, 1023 enhancementDefaultValues: map[string]string{ 1024 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1025 }, 1026 }, 1027 { 1028 description: "req for dedicated_cores with numa_binding & without default numa_exclusive main container", 1029 req: &pluginapi.ResourceRequest{ 1030 PodUid: string(uuid.NewUUID()), 1031 PodNamespace: testName, 1032 PodName: testName, 1033 ContainerName: testName, 1034 ContainerType: pluginapi.ContainerType_MAIN, 1035 ContainerIndex: 0, 1036 ResourceName: string(v1.ResourceCPU), 1037 ResourceRequests: map[string]float64{ 1038 string(v1.ResourceCPU): 2, 1039 }, 1040 Annotations: map[string]string{ 1041 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1042 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, 1043 }, 1044 Labels: map[string]string{ 1045 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1046 }, 1047 }, 1048 expectedResp: &pluginapi.ResourceHintsResponse{ 1049 PodNamespace: testName, 1050 PodName: testName, 1051 ContainerName: testName, 1052 ContainerType: pluginapi.ContainerType_MAIN, 1053 ContainerIndex: 0, 1054 ResourceName: string(v1.ResourceCPU), 1055 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 1056 string(v1.ResourceCPU): { 1057 Hints: []*pluginapi.TopologyHint{ 1058 { 1059 Nodes: []uint64{0}, 1060 Preferred: true, 1061 }, 1062 { 1063 Nodes: []uint64{1}, 1064 Preferred: true, 1065 }, 1066 { 1067 Nodes: []uint64{2}, 1068 Preferred: true, 1069 }, 1070 { 1071 Nodes: []uint64{3}, 1072 Preferred: true, 1073 }, 1074 }, 1075 }, 1076 }, 1077 Labels: map[string]string{ 1078 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1079 }, 1080 Annotations: map[string]string{ 1081 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1082 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 1083 }, 1084 }, 1085 cpuTopology: cpuTopology, 1086 }, 1087 } 1088 1089 for _, tc := range testCases { 1090 tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetTopologyHints") 1091 as.Nil(err) 1092 1093 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(tc.cpuTopology, tmpDir) 1094 as.Nil(err) 1095 1096 if tc.enhancementDefaultValues != nil { 1097 dynamicPolicy.qosConfig.QoSEnhancementDefaultValues = tc.enhancementDefaultValues 1098 } 1099 1100 resp, err := dynamicPolicy.GetTopologyHints(context.Background(), tc.req) 1101 as.Nil(err) 1102 1103 tc.expectedResp.PodUid = tc.req.PodUid 1104 as.Equalf(tc.expectedResp, resp, "failed in test case: %s", tc.description) 1105 1106 _ = os.RemoveAll(tmpDir) 1107 } 1108 } 1109 1110 func TestGetTopologyAwareAllocatableResources(t *testing.T) { 1111 t.Parallel() 1112 1113 as := require.New(t) 1114 1115 tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetTopologyAwareAllocatableResources") 1116 as.Nil(err) 1117 defer func() { _ = os.RemoveAll(tmpDir) }() 1118 1119 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 1120 as.Nil(err) 1121 1122 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) 1123 as.Nil(err) 1124 1125 resp, err := dynamicPolicy.GetTopologyAwareAllocatableResources(context.Background(), &pluginapi.GetTopologyAwareAllocatableResourcesRequest{}) 1126 as.Nil(err) 1127 1128 as.Equal(&pluginapi.GetTopologyAwareAllocatableResourcesResponse{ 1129 AllocatableResources: map[string]*pluginapi.AllocatableTopologyAwareResource{ 1130 string(v1.ResourceCPU): { 1131 IsNodeResource: false, 1132 IsScalarResource: true, 1133 TopologyAwareAllocatableQuantityList: []*pluginapi.TopologyAwareQuantity{ 1134 {ResourceValue: 3, Node: 0}, 1135 {ResourceValue: 3, Node: 1}, 1136 {ResourceValue: 4, Node: 2}, 1137 {ResourceValue: 4, Node: 3}, 1138 }, 1139 TopologyAwareCapacityQuantityList: []*pluginapi.TopologyAwareQuantity{ 1140 {ResourceValue: 4, Node: 0}, 1141 {ResourceValue: 4, Node: 1}, 1142 {ResourceValue: 4, Node: 2}, 1143 {ResourceValue: 4, Node: 3}, 1144 }, 1145 AggregatedAllocatableQuantity: 14, 1146 AggregatedCapacityQuantity: 16, 1147 }, 1148 }, 1149 }, resp) 1150 } 1151 1152 func TestGetTopologyAwareResources(t *testing.T) { 1153 t.Parallel() 1154 1155 as := require.New(t) 1156 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 1157 as.Nil(err) 1158 1159 testName := "test" 1160 1161 testCases := []struct { 1162 description string 1163 req *pluginapi.ResourceRequest 1164 expectedResp *pluginapi.GetTopologyAwareResourcesResponse 1165 cpuTopology *machine.CPUTopology 1166 err error 1167 }{ 1168 { 1169 description: "req for init container", 1170 req: &pluginapi.ResourceRequest{ 1171 PodUid: string(uuid.NewUUID()), 1172 PodNamespace: testName, 1173 PodName: testName, 1174 ContainerName: testName, 1175 ContainerType: pluginapi.ContainerType_INIT, 1176 ContainerIndex: 0, 1177 ResourceName: string(v1.ResourceCPU), 1178 ResourceRequests: map[string]float64{ 1179 string(v1.ResourceCPU): 2, 1180 }, 1181 }, 1182 err: fmt.Errorf("error occurred"), 1183 cpuTopology: cpuTopology, 1184 }, 1185 { 1186 description: "req for shared_cores main container", 1187 req: &pluginapi.ResourceRequest{ 1188 PodUid: string(uuid.NewUUID()), 1189 PodNamespace: testName, 1190 PodName: testName, 1191 ContainerName: testName, 1192 ContainerType: pluginapi.ContainerType_MAIN, 1193 ContainerIndex: 0, 1194 ResourceName: string(v1.ResourceCPU), 1195 ResourceRequests: map[string]float64{ 1196 string(v1.ResourceCPU): 2, 1197 }, 1198 Labels: map[string]string{ 1199 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1200 }, 1201 Annotations: map[string]string{ 1202 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1203 }, 1204 }, 1205 expectedResp: &pluginapi.GetTopologyAwareResourcesResponse{ 1206 PodNamespace: testName, 1207 PodName: testName, 1208 ContainerTopologyAwareResources: &pluginapi.ContainerTopologyAwareResources{ 1209 ContainerName: testName, 1210 AllocatedResources: map[string]*pluginapi.TopologyAwareResource{ 1211 string(v1.ResourceCPU): { 1212 IsNodeResource: false, 1213 IsScalarResource: true, 1214 AggregatedQuantity: 14, 1215 OriginalAggregatedQuantity: 14, 1216 TopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 1217 {ResourceValue: 3, Node: 0}, 1218 {ResourceValue: 3, Node: 1}, 1219 {ResourceValue: 4, Node: 2}, 1220 {ResourceValue: 4, Node: 3}, 1221 }, 1222 OriginalTopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 1223 {ResourceValue: 3, Node: 0}, 1224 {ResourceValue: 3, Node: 1}, 1225 {ResourceValue: 4, Node: 2}, 1226 {ResourceValue: 4, Node: 3}, 1227 }, 1228 }, 1229 }, 1230 }, 1231 }, 1232 cpuTopology: cpuTopology, 1233 }, 1234 { 1235 description: "req for reclaimed_cores main container", 1236 req: &pluginapi.ResourceRequest{ 1237 PodUid: string(uuid.NewUUID()), 1238 PodNamespace: testName, 1239 PodName: testName, 1240 ContainerName: testName, 1241 ContainerType: pluginapi.ContainerType_MAIN, 1242 ContainerIndex: 0, 1243 ResourceName: string(v1.ResourceCPU), 1244 ResourceRequests: map[string]float64{ 1245 string(v1.ResourceCPU): 2, 1246 }, 1247 Labels: map[string]string{ 1248 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1249 }, 1250 Annotations: map[string]string{ 1251 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1252 }, 1253 }, 1254 expectedResp: &pluginapi.GetTopologyAwareResourcesResponse{ 1255 PodNamespace: testName, 1256 PodName: testName, 1257 ContainerTopologyAwareResources: &pluginapi.ContainerTopologyAwareResources{ 1258 ContainerName: testName, 1259 AllocatedResources: map[string]*pluginapi.TopologyAwareResource{ 1260 string(v1.ResourceCPU): { 1261 IsNodeResource: false, 1262 IsScalarResource: true, 1263 AggregatedQuantity: 4, 1264 OriginalAggregatedQuantity: 4, 1265 TopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 1266 {ResourceValue: 2, Node: 0}, 1267 {ResourceValue: 2, Node: 1}, 1268 }, 1269 OriginalTopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 1270 {ResourceValue: 2, Node: 0}, 1271 {ResourceValue: 2, Node: 1}, 1272 }, 1273 }, 1274 }, 1275 }, 1276 }, 1277 cpuTopology: cpuTopology, 1278 }, 1279 } 1280 1281 for _, tc := range testCases { 1282 tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetTopologyAwareResources") 1283 as.Nil(err) 1284 1285 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(tc.cpuTopology, tmpDir) 1286 as.Nil(err) 1287 1288 _, err = dynamicPolicy.Allocate(context.Background(), tc.req) 1289 as.Nil(err) 1290 1291 resp, err := dynamicPolicy.GetTopologyAwareResources(context.Background(), &pluginapi.GetTopologyAwareResourcesRequest{ 1292 PodUid: tc.req.PodUid, 1293 ContainerName: testName, 1294 }) 1295 1296 if tc.err != nil { 1297 as.NotNil(err) 1298 continue 1299 } else { 1300 as.Nil(err) 1301 tc.expectedResp.PodUid = tc.req.PodUid 1302 } 1303 1304 as.Equalf(tc.expectedResp, resp, "failed in test case: %s", tc.description) 1305 1306 if tc.req.Annotations[consts.PodAnnotationQoSLevelKey] == consts.PodAnnotationQoSLevelSharedCores { 1307 originalTransitionPeriod := dynamicPolicy.transitionPeriod 1308 dynamicPolicy.transitionPeriod = time.Millisecond * 10 1309 time.Sleep(20 * time.Millisecond) 1310 _, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) 1311 dynamicPolicy.transitionPeriod = originalTransitionPeriod 1312 as.Nil(err) 1313 allocationInfo := dynamicPolicy.state.GetAllocationInfo(tc.req.PodUid, testName) 1314 as.NotNil(allocationInfo) 1315 as.Equal(false, allocationInfo.RampUp) 1316 1317 resp, err = dynamicPolicy.GetTopologyAwareResources(context.Background(), &pluginapi.GetTopologyAwareResourcesRequest{ 1318 PodUid: tc.req.PodUid, 1319 ContainerName: testName, 1320 }) 1321 1322 as.Nil(err) 1323 as.NotNil(resp) 1324 1325 tc.expectedResp.ContainerTopologyAwareResources.AllocatedResources = map[string]*pluginapi.TopologyAwareResource{ 1326 string(v1.ResourceCPU): { 1327 IsNodeResource: false, 1328 IsScalarResource: true, 1329 AggregatedQuantity: 10, 1330 OriginalAggregatedQuantity: 10, 1331 TopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 1332 {ResourceValue: 2, Node: 0}, 1333 {ResourceValue: 2, Node: 1}, 1334 {ResourceValue: 4, Node: 2}, 1335 {ResourceValue: 2, Node: 3}, 1336 }, 1337 OriginalTopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 1338 {ResourceValue: 2, Node: 0}, 1339 {ResourceValue: 2, Node: 1}, 1340 {ResourceValue: 4, Node: 2}, 1341 {ResourceValue: 2, Node: 3}, 1342 }, 1343 }, 1344 } 1345 1346 as.Equalf(tc.expectedResp, resp, "failed in test case: %s", tc.description) 1347 } 1348 1349 os.Remove(tmpDir) 1350 } 1351 } 1352 1353 func TestGetResourcesAllocation(t *testing.T) { 1354 t.Parallel() 1355 1356 as := require.New(t) 1357 1358 tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetResourcesAllocation") 1359 as.Nil(err) 1360 defer func() { _ = os.RemoveAll(tmpDir) }() 1361 1362 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 1363 as.Nil(err) 1364 1365 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) 1366 as.Nil(err) 1367 1368 testName := "test" 1369 1370 // test for shared_cores 1371 req := &pluginapi.ResourceRequest{ 1372 PodUid: string(uuid.NewUUID()), 1373 PodNamespace: testName, 1374 PodName: testName, 1375 ContainerName: testName, 1376 ContainerType: pluginapi.ContainerType_MAIN, 1377 ContainerIndex: 0, 1378 ResourceName: string(v1.ResourceCPU), 1379 ResourceRequests: map[string]float64{ 1380 string(v1.ResourceCPU): 2, 1381 }, 1382 Labels: map[string]string{ 1383 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1384 }, 1385 Annotations: map[string]string{ 1386 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1387 }, 1388 } 1389 1390 _, err = dynamicPolicy.Allocate(context.Background(), req) 1391 as.Nil(err) 1392 1393 resp1, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) 1394 as.Nil(err) 1395 1396 as.NotNil(resp1.PodResources[req.PodUid]) 1397 as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName]) 1398 as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) 1399 as.Equal(resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)], &pluginapi.ResourceAllocationInfo{ 1400 OciPropertyName: util.OCIPropertyNameCPUSetCPUs, 1401 IsNodeResource: false, 1402 IsScalarResource: true, 1403 AllocatedQuantity: 14, 1404 AllocationResult: cpuTopology.CPUDetails.CPUs().Difference(dynamicPolicy.reservedCPUs).String(), 1405 }) 1406 1407 // test after ramping up 1408 originalTransitionPeriod := dynamicPolicy.transitionPeriod 1409 dynamicPolicy.transitionPeriod = time.Millisecond * 10 1410 time.Sleep(20 * time.Millisecond) 1411 _, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) 1412 as.Nil(err) 1413 dynamicPolicy.transitionPeriod = originalTransitionPeriod 1414 allocationInfo := dynamicPolicy.state.GetAllocationInfo(req.PodUid, testName) 1415 as.NotNil(allocationInfo) 1416 as.Equal(allocationInfo.RampUp, false) 1417 1418 resp2, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) 1419 as.Nil(err) 1420 as.NotNil(resp2.PodResources[req.PodUid]) 1421 as.NotNil(resp2.PodResources[req.PodUid].ContainerResources[testName]) 1422 as.NotNil(resp2.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) 1423 as.Equal(resp2.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)], &pluginapi.ResourceAllocationInfo{ 1424 OciPropertyName: util.OCIPropertyNameCPUSetCPUs, 1425 IsNodeResource: false, 1426 IsScalarResource: true, 1427 AllocatedQuantity: 10, 1428 AllocationResult: machine.NewCPUSet(1, 3, 4, 5, 6, 9, 11, 12, 13, 14).String(), 1429 }) 1430 1431 // test for reclaimed_cores 1432 req = &pluginapi.ResourceRequest{ 1433 PodUid: string(uuid.NewUUID()), 1434 PodNamespace: testName, 1435 PodName: testName, 1436 ContainerName: testName, 1437 ContainerType: pluginapi.ContainerType_MAIN, 1438 ContainerIndex: 0, 1439 ResourceName: string(v1.ResourceCPU), 1440 ResourceRequests: map[string]float64{ 1441 string(v1.ResourceCPU): 2, 1442 }, 1443 Labels: map[string]string{ 1444 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1445 }, 1446 Annotations: map[string]string{ 1447 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1448 }, 1449 } 1450 1451 _, err = dynamicPolicy.Allocate(context.Background(), req) 1452 as.Nil(err) 1453 1454 resp2, err = dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) 1455 as.Nil(err) 1456 1457 as.NotNil(resp2.PodResources[req.PodUid]) 1458 as.NotNil(resp2.PodResources[req.PodUid].ContainerResources[testName]) 1459 as.NotNil(resp2.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)]) 1460 as.Equal(resp2.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceCPU)], &pluginapi.ResourceAllocationInfo{ 1461 OciPropertyName: util.OCIPropertyNameCPUSetCPUs, 1462 IsNodeResource: false, 1463 IsScalarResource: true, 1464 AllocatedQuantity: 4, 1465 AllocationResult: machine.NewCPUSet(7, 8, 10, 15).String(), 1466 }) 1467 } 1468 1469 func TestAllocateByQoSAwareServerListAndWatchResp(t *testing.T) { 1470 t.Parallel() 1471 1472 as := require.New(t) 1473 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 1474 as.Nil(err) 1475 1476 pod1UID := string(uuid.NewUUID()) 1477 pod2UID := string(uuid.NewUUID()) 1478 pod3UID := string(uuid.NewUUID()) 1479 pod4UID := string(uuid.NewUUID()) 1480 testName := "test" 1481 1482 fmt.Println("1", pod1UID) 1483 fmt.Println("2", pod2UID) 1484 fmt.Println("3", pod3UID) 1485 fmt.Println("4", pod4UID) 1486 1487 testCases := []struct { 1488 description string 1489 podEntries state.PodEntries 1490 expectedPodEntries state.PodEntries 1491 expectedMachineState state.NUMANodeMap 1492 lwResp *advisorapi.ListAndWatchResponse 1493 cpuTopology *machine.CPUTopology 1494 }{ 1495 { 1496 description: "two shared_cores containers and one reclaimed_cores container", 1497 podEntries: state.PodEntries{ 1498 pod1UID: state.ContainerEntries{ 1499 testName: &state.AllocationInfo{ 1500 PodUid: pod1UID, 1501 PodNamespace: testName, 1502 PodName: testName, 1503 ContainerName: testName, 1504 ContainerType: pluginapi.ContainerType_MAIN.String(), 1505 ContainerIndex: 0, 1506 RampUp: false, 1507 OwnerPoolName: state.PoolNameShare, 1508 AllocationResult: machine.MustParse("1,3-6,9,11-14"), 1509 OriginalAllocationResult: machine.MustParse("1,3-6,9,11-14"), 1510 TopologyAwareAssignments: map[int]machine.CPUSet{ 1511 0: machine.NewCPUSet(1, 9), 1512 1: machine.NewCPUSet(3, 11), 1513 2: machine.NewCPUSet(4, 5, 11, 12), 1514 3: machine.NewCPUSet(6, 14), 1515 }, 1516 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1517 0: machine.NewCPUSet(1, 9), 1518 1: machine.NewCPUSet(3, 11), 1519 2: machine.NewCPUSet(4, 5, 11, 12), 1520 3: machine.NewCPUSet(6, 14), 1521 }, 1522 Labels: map[string]string{ 1523 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1524 }, 1525 Annotations: map[string]string{ 1526 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1527 }, 1528 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1529 RequestQuantity: 2, 1530 }, 1531 }, 1532 pod2UID: state.ContainerEntries{ 1533 testName: &state.AllocationInfo{ 1534 PodUid: pod2UID, 1535 PodNamespace: testName, 1536 PodName: testName, 1537 ContainerName: testName, 1538 ContainerType: pluginapi.ContainerType_MAIN.String(), 1539 ContainerIndex: 0, 1540 RampUp: false, 1541 OwnerPoolName: state.PoolNameShare, 1542 AllocationResult: machine.MustParse("1,3-6,9,11-14"), 1543 OriginalAllocationResult: machine.MustParse("1,3-6,9,11-14"), 1544 TopologyAwareAssignments: map[int]machine.CPUSet{ 1545 0: machine.NewCPUSet(1, 9), 1546 1: machine.NewCPUSet(3, 11), 1547 2: machine.NewCPUSet(4, 5, 11, 12), 1548 3: machine.NewCPUSet(6, 14), 1549 }, 1550 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1551 0: machine.NewCPUSet(1, 9), 1552 1: machine.NewCPUSet(3, 11), 1553 2: machine.NewCPUSet(4, 5, 11, 12), 1554 3: machine.NewCPUSet(6, 14), 1555 }, 1556 Labels: map[string]string{ 1557 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1558 }, 1559 Annotations: map[string]string{ 1560 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1561 }, 1562 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1563 RequestQuantity: 2, 1564 }, 1565 }, 1566 pod3UID: state.ContainerEntries{ 1567 testName: &state.AllocationInfo{ 1568 PodUid: pod3UID, 1569 PodNamespace: testName, 1570 PodName: testName, 1571 ContainerName: testName, 1572 ContainerType: pluginapi.ContainerType_MAIN.String(), 1573 ContainerIndex: 0, 1574 RampUp: false, 1575 OwnerPoolName: state.PoolNameReclaim, 1576 AllocationResult: machine.NewCPUSet(7, 8, 10, 15), 1577 OriginalAllocationResult: machine.NewCPUSet(7, 8, 10, 15), 1578 TopologyAwareAssignments: map[int]machine.CPUSet{ 1579 0: machine.NewCPUSet(8), 1580 1: machine.NewCPUSet(10), 1581 3: machine.NewCPUSet(7, 15), 1582 }, 1583 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1584 0: machine.NewCPUSet(8), 1585 1: machine.NewCPUSet(10), 1586 3: machine.NewCPUSet(7, 15), 1587 }, 1588 Labels: map[string]string{ 1589 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1590 }, 1591 Annotations: map[string]string{ 1592 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1593 }, 1594 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 1595 RequestQuantity: 2, 1596 }, 1597 }, 1598 }, 1599 lwResp: &advisorapi.ListAndWatchResponse{ 1600 Entries: map[string]*advisorapi.CalculationEntries{ 1601 state.PoolNameShare: { 1602 Entries: map[string]*advisorapi.CalculationInfo{ 1603 "": { 1604 OwnerPoolName: state.PoolNameShare, 1605 CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ 1606 -1: { 1607 Blocks: []*advisorapi.Block{ 1608 { 1609 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f66", 1610 Result: 6, 1611 }, 1612 }, 1613 }, 1614 }, 1615 }, 1616 }, 1617 }, 1618 state.PoolNameReserve: { 1619 Entries: map[string]*advisorapi.CalculationInfo{ 1620 "": { 1621 OwnerPoolName: state.PoolNameReserve, 1622 CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ 1623 -1: { 1624 Blocks: []*advisorapi.Block{ 1625 { 1626 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f65", 1627 Result: 2, 1628 }, 1629 }, 1630 }, 1631 }, 1632 }, 1633 }, 1634 }, 1635 }, 1636 }, 1637 expectedPodEntries: state.PodEntries{ 1638 pod1UID: state.ContainerEntries{ 1639 testName: &state.AllocationInfo{ 1640 PodUid: pod1UID, 1641 PodNamespace: testName, 1642 PodName: testName, 1643 ContainerName: testName, 1644 ContainerType: pluginapi.ContainerType_MAIN.String(), 1645 ContainerIndex: 0, 1646 RampUp: false, 1647 OwnerPoolName: state.PoolNameShare, 1648 AllocationResult: machine.MustParse("1,3-4,9,11-12"), 1649 OriginalAllocationResult: machine.MustParse("1,3-4,9,11-12"), 1650 TopologyAwareAssignments: map[int]machine.CPUSet{ 1651 0: machine.NewCPUSet(1, 9), 1652 1: machine.NewCPUSet(3, 11), 1653 2: machine.NewCPUSet(4, 12), 1654 }, 1655 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1656 0: machine.NewCPUSet(1, 9), 1657 1: machine.NewCPUSet(3, 11), 1658 2: machine.NewCPUSet(4, 12), 1659 }, 1660 Labels: map[string]string{ 1661 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1662 }, 1663 Annotations: map[string]string{ 1664 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1665 }, 1666 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1667 RequestQuantity: 2, 1668 }, 1669 }, 1670 pod2UID: state.ContainerEntries{ 1671 testName: &state.AllocationInfo{ 1672 PodUid: pod2UID, 1673 PodNamespace: testName, 1674 PodName: testName, 1675 ContainerName: testName, 1676 ContainerType: pluginapi.ContainerType_MAIN.String(), 1677 ContainerIndex: 0, 1678 RampUp: false, 1679 OwnerPoolName: state.PoolNameShare, 1680 AllocationResult: machine.MustParse("1,3-4,9,11-12"), 1681 OriginalAllocationResult: machine.MustParse("1,3-4,9,11-12"), 1682 TopologyAwareAssignments: map[int]machine.CPUSet{ 1683 0: machine.NewCPUSet(1, 9), 1684 1: machine.NewCPUSet(3, 11), 1685 2: machine.NewCPUSet(4, 12), 1686 }, 1687 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1688 0: machine.NewCPUSet(1, 9), 1689 1: machine.NewCPUSet(3, 11), 1690 2: machine.NewCPUSet(4, 12), 1691 }, 1692 Labels: map[string]string{ 1693 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1694 }, 1695 Annotations: map[string]string{ 1696 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1697 }, 1698 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1699 RequestQuantity: 2, 1700 }, 1701 }, 1702 pod3UID: state.ContainerEntries{ 1703 testName: &state.AllocationInfo{ 1704 PodUid: pod3UID, 1705 PodNamespace: testName, 1706 PodName: testName, 1707 ContainerName: testName, 1708 ContainerType: pluginapi.ContainerType_MAIN.String(), 1709 ContainerIndex: 0, 1710 RampUp: false, 1711 OwnerPoolName: state.PoolNameReclaim, 1712 AllocationResult: machine.MustParse("5-8,10,13-15"), 1713 OriginalAllocationResult: machine.MustParse("5-8,10,13-15"), 1714 TopologyAwareAssignments: map[int]machine.CPUSet{ 1715 0: machine.NewCPUSet(8), 1716 1: machine.NewCPUSet(10), 1717 2: machine.NewCPUSet(5, 13), 1718 3: machine.NewCPUSet(6, 7, 14, 15), 1719 }, 1720 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1721 0: machine.NewCPUSet(8), 1722 1: machine.NewCPUSet(10), 1723 2: machine.NewCPUSet(5, 13), 1724 3: machine.NewCPUSet(6, 7, 14, 15), 1725 }, 1726 Labels: map[string]string{ 1727 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1728 }, 1729 Annotations: map[string]string{ 1730 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1731 }, 1732 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 1733 RequestQuantity: 2, 1734 }, 1735 }, 1736 state.PoolNameReclaim: state.ContainerEntries{ 1737 "": &state.AllocationInfo{ 1738 PodUid: state.PoolNameReclaim, 1739 OwnerPoolName: state.PoolNameReclaim, 1740 AllocationResult: machine.MustParse("5-8,10,13-15"), 1741 OriginalAllocationResult: machine.MustParse("5-8,10,13-15"), 1742 TopologyAwareAssignments: map[int]machine.CPUSet{ 1743 0: machine.NewCPUSet(8), 1744 1: machine.NewCPUSet(10), 1745 2: machine.NewCPUSet(5, 13), 1746 3: machine.NewCPUSet(6, 7, 14, 15), 1747 }, 1748 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1749 0: machine.NewCPUSet(8), 1750 1: machine.NewCPUSet(10), 1751 2: machine.NewCPUSet(5, 13), 1752 3: machine.NewCPUSet(6, 7, 14, 15), 1753 }, 1754 }, 1755 }, 1756 state.PoolNameShare: state.ContainerEntries{ 1757 "": &state.AllocationInfo{ 1758 PodUid: state.PoolNameShare, 1759 OwnerPoolName: state.PoolNameShare, 1760 AllocationResult: machine.MustParse("1,3-4,9,11-12"), 1761 OriginalAllocationResult: machine.MustParse("1,3-4,9,11-12"), 1762 TopologyAwareAssignments: map[int]machine.CPUSet{ 1763 0: machine.NewCPUSet(1, 9), 1764 1: machine.NewCPUSet(3, 11), 1765 2: machine.NewCPUSet(4, 12), 1766 }, 1767 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1768 0: machine.NewCPUSet(1, 9), 1769 1: machine.NewCPUSet(3, 11), 1770 2: machine.NewCPUSet(4, 12), 1771 }, 1772 }, 1773 }, 1774 state.PoolNameReserve: state.ContainerEntries{ 1775 "": &state.AllocationInfo{ 1776 PodUid: state.PoolNameReserve, 1777 OwnerPoolName: state.PoolNameReserve, 1778 AllocationResult: machine.MustParse("0,2"), 1779 OriginalAllocationResult: machine.MustParse("0,2"), 1780 TopologyAwareAssignments: map[int]machine.CPUSet{ 1781 0: machine.NewCPUSet(0), 1782 1: machine.NewCPUSet(2), 1783 }, 1784 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1785 0: machine.NewCPUSet(0), 1786 1: machine.NewCPUSet(2), 1787 }, 1788 }, 1789 }, 1790 }, 1791 expectedMachineState: state.NUMANodeMap{ 1792 0: &state.NUMANodeState{ 1793 DefaultCPUSet: cpuTopology.CPUDetails.CPUsInNUMANodes(0).Clone(), 1794 AllocatedCPUSet: machine.NewCPUSet(), 1795 PodEntries: state.PodEntries{ 1796 pod1UID: state.ContainerEntries{ 1797 testName: &state.AllocationInfo{ 1798 PodUid: pod1UID, 1799 PodNamespace: testName, 1800 PodName: testName, 1801 ContainerName: testName, 1802 ContainerType: pluginapi.ContainerType_MAIN.String(), 1803 ContainerIndex: 0, 1804 RampUp: false, 1805 OwnerPoolName: state.PoolNameShare, 1806 AllocationResult: machine.NewCPUSet(1, 9), 1807 OriginalAllocationResult: machine.NewCPUSet(1, 9), 1808 TopologyAwareAssignments: map[int]machine.CPUSet{ 1809 0: machine.NewCPUSet(1, 9), 1810 }, 1811 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1812 0: machine.NewCPUSet(1, 9), 1813 }, 1814 Labels: map[string]string{ 1815 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1816 }, 1817 Annotations: map[string]string{ 1818 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1819 }, 1820 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1821 RequestQuantity: 2, 1822 }, 1823 }, 1824 pod2UID: state.ContainerEntries{ 1825 testName: &state.AllocationInfo{ 1826 PodUid: pod2UID, 1827 PodNamespace: testName, 1828 PodName: testName, 1829 ContainerName: testName, 1830 ContainerType: pluginapi.ContainerType_MAIN.String(), 1831 ContainerIndex: 0, 1832 RampUp: false, 1833 OwnerPoolName: state.PoolNameShare, 1834 AllocationResult: machine.NewCPUSet(1, 9), 1835 OriginalAllocationResult: machine.NewCPUSet(1, 9), 1836 TopologyAwareAssignments: map[int]machine.CPUSet{ 1837 0: machine.NewCPUSet(1, 9), 1838 }, 1839 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1840 0: machine.NewCPUSet(1, 9), 1841 }, 1842 Labels: map[string]string{ 1843 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1844 }, 1845 Annotations: map[string]string{ 1846 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1847 }, 1848 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1849 RequestQuantity: 2, 1850 }, 1851 }, 1852 pod3UID: state.ContainerEntries{ 1853 testName: &state.AllocationInfo{ 1854 PodUid: pod3UID, 1855 PodNamespace: testName, 1856 PodName: testName, 1857 ContainerName: testName, 1858 ContainerType: pluginapi.ContainerType_MAIN.String(), 1859 ContainerIndex: 0, 1860 RampUp: false, 1861 OwnerPoolName: state.PoolNameReclaim, 1862 AllocationResult: machine.MustParse("8"), 1863 OriginalAllocationResult: machine.MustParse("8"), 1864 TopologyAwareAssignments: map[int]machine.CPUSet{ 1865 0: machine.NewCPUSet(8), 1866 }, 1867 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1868 0: machine.NewCPUSet(8), 1869 }, 1870 Labels: map[string]string{ 1871 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1872 }, 1873 Annotations: map[string]string{ 1874 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1875 }, 1876 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 1877 RequestQuantity: 2, 1878 }, 1879 }, 1880 }, 1881 }, 1882 1: &state.NUMANodeState{ 1883 DefaultCPUSet: cpuTopology.CPUDetails.CPUsInNUMANodes(1).Clone(), 1884 AllocatedCPUSet: machine.NewCPUSet(), 1885 PodEntries: state.PodEntries{ 1886 pod1UID: state.ContainerEntries{ 1887 testName: &state.AllocationInfo{ 1888 PodUid: pod1UID, 1889 PodNamespace: testName, 1890 PodName: testName, 1891 ContainerName: testName, 1892 ContainerType: pluginapi.ContainerType_MAIN.String(), 1893 ContainerIndex: 0, 1894 RampUp: false, 1895 OwnerPoolName: state.PoolNameShare, 1896 AllocationResult: machine.MustParse("3,11"), 1897 OriginalAllocationResult: machine.MustParse("3,11"), 1898 TopologyAwareAssignments: map[int]machine.CPUSet{ 1899 1: machine.NewCPUSet(3, 11), 1900 }, 1901 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1902 1: machine.NewCPUSet(3, 11), 1903 }, 1904 Labels: map[string]string{ 1905 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1906 }, 1907 Annotations: map[string]string{ 1908 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1909 }, 1910 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1911 RequestQuantity: 2, 1912 }, 1913 }, 1914 pod2UID: state.ContainerEntries{ 1915 testName: &state.AllocationInfo{ 1916 PodUid: pod2UID, 1917 PodNamespace: testName, 1918 PodName: testName, 1919 ContainerName: testName, 1920 ContainerType: pluginapi.ContainerType_MAIN.String(), 1921 ContainerIndex: 0, 1922 RampUp: false, 1923 OwnerPoolName: state.PoolNameShare, 1924 AllocationResult: machine.MustParse("3,11"), 1925 OriginalAllocationResult: machine.MustParse("3,11"), 1926 TopologyAwareAssignments: map[int]machine.CPUSet{ 1927 1: machine.NewCPUSet(3, 11), 1928 }, 1929 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1930 1: machine.NewCPUSet(3, 11), 1931 }, 1932 Labels: map[string]string{ 1933 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1934 }, 1935 Annotations: map[string]string{ 1936 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1937 }, 1938 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1939 RequestQuantity: 2, 1940 }, 1941 }, 1942 pod3UID: state.ContainerEntries{ 1943 testName: &state.AllocationInfo{ 1944 PodUid: pod3UID, 1945 PodNamespace: testName, 1946 PodName: testName, 1947 ContainerName: testName, 1948 ContainerType: pluginapi.ContainerType_MAIN.String(), 1949 ContainerIndex: 0, 1950 RampUp: false, 1951 OwnerPoolName: state.PoolNameReclaim, 1952 AllocationResult: machine.MustParse("10"), 1953 OriginalAllocationResult: machine.MustParse("10"), 1954 TopologyAwareAssignments: map[int]machine.CPUSet{ 1955 1: machine.NewCPUSet(10), 1956 }, 1957 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1958 1: machine.NewCPUSet(10), 1959 }, 1960 Labels: map[string]string{ 1961 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1962 }, 1963 Annotations: map[string]string{ 1964 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1965 }, 1966 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 1967 RequestQuantity: 2, 1968 }, 1969 }, 1970 }, 1971 }, 1972 2: &state.NUMANodeState{ 1973 DefaultCPUSet: cpuTopology.CPUDetails.CPUsInNUMANodes(2).Clone(), 1974 AllocatedCPUSet: machine.NewCPUSet(), 1975 PodEntries: state.PodEntries{ 1976 pod1UID: state.ContainerEntries{ 1977 testName: &state.AllocationInfo{ 1978 PodUid: pod1UID, 1979 PodNamespace: testName, 1980 PodName: testName, 1981 ContainerName: testName, 1982 ContainerType: pluginapi.ContainerType_MAIN.String(), 1983 ContainerIndex: 0, 1984 RampUp: false, 1985 OwnerPoolName: state.PoolNameShare, 1986 AllocationResult: machine.MustParse("4,12"), 1987 OriginalAllocationResult: machine.MustParse("4,12"), 1988 TopologyAwareAssignments: map[int]machine.CPUSet{ 1989 2: machine.NewCPUSet(4, 12), 1990 }, 1991 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 1992 2: machine.NewCPUSet(4, 12), 1993 }, 1994 Labels: map[string]string{ 1995 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1996 }, 1997 Annotations: map[string]string{ 1998 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1999 }, 2000 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2001 RequestQuantity: 2, 2002 }, 2003 }, 2004 pod2UID: state.ContainerEntries{ 2005 testName: &state.AllocationInfo{ 2006 PodUid: pod2UID, 2007 PodNamespace: testName, 2008 PodName: testName, 2009 ContainerName: testName, 2010 ContainerType: pluginapi.ContainerType_MAIN.String(), 2011 ContainerIndex: 0, 2012 RampUp: false, 2013 OwnerPoolName: state.PoolNameShare, 2014 AllocationResult: machine.MustParse("4,12"), 2015 OriginalAllocationResult: machine.MustParse("4,12"), 2016 TopologyAwareAssignments: map[int]machine.CPUSet{ 2017 2: machine.NewCPUSet(4, 12), 2018 }, 2019 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2020 2: machine.NewCPUSet(4, 12), 2021 }, 2022 Labels: map[string]string{ 2023 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2024 }, 2025 Annotations: map[string]string{ 2026 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2027 }, 2028 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2029 RequestQuantity: 2, 2030 }, 2031 }, 2032 pod3UID: state.ContainerEntries{ 2033 testName: &state.AllocationInfo{ 2034 PodUid: pod3UID, 2035 PodNamespace: testName, 2036 PodName: testName, 2037 ContainerName: testName, 2038 ContainerType: pluginapi.ContainerType_MAIN.String(), 2039 ContainerIndex: 0, 2040 RampUp: false, 2041 OwnerPoolName: state.PoolNameReclaim, 2042 AllocationResult: machine.MustParse("5,13"), 2043 OriginalAllocationResult: machine.MustParse("5,13"), 2044 TopologyAwareAssignments: map[int]machine.CPUSet{ 2045 2: machine.NewCPUSet(5, 13), 2046 }, 2047 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2048 2: machine.NewCPUSet(5, 13), 2049 }, 2050 Labels: map[string]string{ 2051 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2052 }, 2053 Annotations: map[string]string{ 2054 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2055 }, 2056 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2057 RequestQuantity: 2, 2058 }, 2059 }, 2060 }, 2061 }, 2062 3: &state.NUMANodeState{ 2063 DefaultCPUSet: cpuTopology.CPUDetails.CPUsInNUMANodes(3).Clone(), 2064 AllocatedCPUSet: machine.NewCPUSet(), 2065 PodEntries: state.PodEntries{ 2066 pod3UID: state.ContainerEntries{ 2067 testName: &state.AllocationInfo{ 2068 PodUid: pod3UID, 2069 PodNamespace: testName, 2070 PodName: testName, 2071 ContainerName: testName, 2072 ContainerType: pluginapi.ContainerType_MAIN.String(), 2073 ContainerIndex: 0, 2074 RampUp: false, 2075 OwnerPoolName: state.PoolNameReclaim, 2076 AllocationResult: machine.MustParse("6,7,14,15"), 2077 OriginalAllocationResult: machine.MustParse("6,7,14,15"), 2078 TopologyAwareAssignments: map[int]machine.CPUSet{ 2079 3: machine.NewCPUSet(6, 7, 14, 15), 2080 }, 2081 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2082 3: machine.NewCPUSet(6, 7, 14, 15), 2083 }, 2084 Labels: map[string]string{ 2085 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2086 }, 2087 Annotations: map[string]string{ 2088 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2089 }, 2090 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2091 RequestQuantity: 2, 2092 }, 2093 }, 2094 }, 2095 }, 2096 }, 2097 cpuTopology: cpuTopology, 2098 }, 2099 { 2100 description: "two shared_cores containers and one reclaimed_cores container and one dedicated_cores", 2101 podEntries: state.PodEntries{ 2102 pod1UID: state.ContainerEntries{ 2103 testName: &state.AllocationInfo{ 2104 PodUid: pod1UID, 2105 PodNamespace: testName, 2106 PodName: testName, 2107 ContainerName: testName, 2108 ContainerType: pluginapi.ContainerType_MAIN.String(), 2109 ContainerIndex: 0, 2110 RampUp: false, 2111 OwnerPoolName: state.PoolNameShare, 2112 AllocationResult: machine.MustParse("4-5,6-7"), 2113 OriginalAllocationResult: machine.MustParse("4-5,6-7"), 2114 TopologyAwareAssignments: map[int]machine.CPUSet{ 2115 2: machine.NewCPUSet(4, 5), 2116 3: machine.NewCPUSet(6, 7), 2117 }, 2118 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2119 2: machine.NewCPUSet(4, 5), 2120 3: machine.NewCPUSet(6, 7), 2121 }, 2122 Labels: map[string]string{ 2123 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2124 }, 2125 Annotations: map[string]string{ 2126 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2127 }, 2128 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2129 RequestQuantity: 2, 2130 }, 2131 }, 2132 pod2UID: state.ContainerEntries{ 2133 testName: &state.AllocationInfo{ 2134 PodUid: pod2UID, 2135 PodNamespace: testName, 2136 PodName: testName, 2137 ContainerName: testName, 2138 ContainerType: pluginapi.ContainerType_MAIN.String(), 2139 ContainerIndex: 0, 2140 RampUp: false, 2141 OwnerPoolName: state.PoolNameShare, 2142 AllocationResult: machine.MustParse("4-5,6-7"), 2143 OriginalAllocationResult: machine.MustParse("4-5,6-7"), 2144 TopologyAwareAssignments: map[int]machine.CPUSet{ 2145 2: machine.NewCPUSet(4, 5), 2146 3: machine.NewCPUSet(6, 7), 2147 }, 2148 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2149 2: machine.NewCPUSet(4, 5), 2150 3: machine.NewCPUSet(6, 7), 2151 }, 2152 Labels: map[string]string{ 2153 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2154 }, 2155 Annotations: map[string]string{ 2156 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2157 }, 2158 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2159 RequestQuantity: 2, 2160 }, 2161 }, 2162 pod3UID: state.ContainerEntries{ 2163 testName: &state.AllocationInfo{ 2164 PodUid: pod3UID, 2165 PodNamespace: testName, 2166 PodName: testName, 2167 ContainerName: testName, 2168 ContainerType: pluginapi.ContainerType_MAIN.String(), 2169 ContainerIndex: 0, 2170 RampUp: false, 2171 OwnerPoolName: state.PoolNameReclaim, 2172 AllocationResult: machine.NewCPUSet(12, 13, 14, 15), 2173 OriginalAllocationResult: machine.NewCPUSet(12, 13, 14, 15), 2174 TopologyAwareAssignments: map[int]machine.CPUSet{ 2175 2: machine.NewCPUSet(12, 13), 2176 3: machine.NewCPUSet(14, 15), 2177 }, 2178 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2179 2: machine.NewCPUSet(12, 13), 2180 3: machine.NewCPUSet(14, 15), 2181 }, 2182 Labels: map[string]string{ 2183 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2184 }, 2185 Annotations: map[string]string{ 2186 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2187 }, 2188 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2189 RequestQuantity: 2, 2190 }, 2191 }, 2192 pod4UID: state.ContainerEntries{ 2193 testName: &state.AllocationInfo{ 2194 PodUid: pod4UID, 2195 PodNamespace: testName, 2196 PodName: testName, 2197 ContainerName: testName, 2198 ContainerType: pluginapi.ContainerType_MAIN.String(), 2199 ContainerIndex: 0, 2200 RampUp: false, 2201 OwnerPoolName: state.PoolNameDedicated, 2202 AllocationResult: machine.NewCPUSet(1, 3, 8, 9, 10, 11), 2203 OriginalAllocationResult: machine.NewCPUSet(1, 3, 8, 9, 10, 11), 2204 TopologyAwareAssignments: map[int]machine.CPUSet{ 2205 0: machine.NewCPUSet(1, 8, 9), 2206 1: machine.NewCPUSet(3, 10, 11), 2207 }, 2208 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2209 0: machine.NewCPUSet(1, 8, 9), 2210 1: machine.NewCPUSet(3, 10, 11), 2211 }, 2212 Labels: map[string]string{ 2213 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2214 }, 2215 Annotations: map[string]string{ 2216 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2217 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2218 }, 2219 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2220 RequestQuantity: 2, 2221 }, 2222 }, 2223 }, 2224 lwResp: &advisorapi.ListAndWatchResponse{ 2225 Entries: map[string]*advisorapi.CalculationEntries{ 2226 state.PoolNameShare: { 2227 Entries: map[string]*advisorapi.CalculationInfo{ 2228 "": { 2229 OwnerPoolName: state.PoolNameShare, 2230 CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ 2231 -1: { 2232 Blocks: []*advisorapi.Block{ 2233 { 2234 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f66", 2235 Result: 6, 2236 }, 2237 }, 2238 }, 2239 }, 2240 }, 2241 }, 2242 }, 2243 state.PoolNameReserve: { 2244 Entries: map[string]*advisorapi.CalculationInfo{ 2245 "": { 2246 OwnerPoolName: state.PoolNameReserve, 2247 CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ 2248 -1: { 2249 Blocks: []*advisorapi.Block{ 2250 { 2251 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f65", 2252 Result: 2, 2253 }, 2254 }, 2255 }, 2256 }, 2257 }, 2258 }, 2259 }, 2260 state.PoolNameReclaim: { 2261 Entries: map[string]*advisorapi.CalculationInfo{ 2262 "": { 2263 OwnerPoolName: state.PoolNameReclaim, 2264 CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ 2265 0: { 2266 Blocks: []*advisorapi.Block{ 2267 { 2268 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f30", 2269 Result: 1, 2270 }, 2271 }, 2272 }, 2273 1: { 2274 Blocks: []*advisorapi.Block{ 2275 { 2276 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f31", 2277 Result: 1, 2278 }, 2279 }, 2280 }, 2281 -1: { 2282 Blocks: []*advisorapi.Block{ 2283 { 2284 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f32", 2285 Result: 2, 2286 }, 2287 }, 2288 }, 2289 }, 2290 }, 2291 }, 2292 }, 2293 pod4UID: { 2294 Entries: map[string]*advisorapi.CalculationInfo{ 2295 testName: { 2296 OwnerPoolName: state.PoolNameDedicated, 2297 CalculationResultsByNumas: map[int64]*advisorapi.NumaCalculationResult{ 2298 0: { 2299 Blocks: []*advisorapi.Block{ 2300 { 2301 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd1f20", 2302 Result: 2, 2303 }, 2304 { 2305 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f30", 2306 Result: 1, 2307 }, 2308 }, 2309 }, 2310 1: { 2311 Blocks: []*advisorapi.Block{ 2312 { 2313 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd1f21", 2314 Result: 2, 2315 }, 2316 { 2317 BlockId: "1b7f4ee0-65af-41a4-bb38-5f1268fd2f31", 2318 Result: 1, 2319 }, 2320 }, 2321 }, 2322 }, 2323 }, 2324 }, 2325 }, 2326 }, 2327 }, 2328 expectedPodEntries: state.PodEntries{ 2329 pod1UID: state.ContainerEntries{ 2330 testName: &state.AllocationInfo{ 2331 PodUid: pod1UID, 2332 PodNamespace: testName, 2333 PodName: testName, 2334 ContainerName: testName, 2335 ContainerType: pluginapi.ContainerType_MAIN.String(), 2336 ContainerIndex: 0, 2337 RampUp: false, 2338 OwnerPoolName: state.PoolNameShare, 2339 AllocationResult: machine.MustParse("4,5,6,7,12,13"), 2340 OriginalAllocationResult: machine.MustParse("4,5,6,7,12,13"), 2341 TopologyAwareAssignments: map[int]machine.CPUSet{ 2342 2: machine.NewCPUSet(4, 5, 12, 13), 2343 3: machine.NewCPUSet(6, 7), 2344 }, 2345 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2346 2: machine.NewCPUSet(4, 5, 12, 13), 2347 3: machine.NewCPUSet(6, 7), 2348 }, 2349 Labels: map[string]string{ 2350 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2351 }, 2352 Annotations: map[string]string{ 2353 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2354 }, 2355 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2356 RequestQuantity: 2, 2357 }, 2358 }, 2359 pod2UID: state.ContainerEntries{ 2360 testName: &state.AllocationInfo{ 2361 PodUid: pod2UID, 2362 PodNamespace: testName, 2363 PodName: testName, 2364 ContainerName: testName, 2365 ContainerType: pluginapi.ContainerType_MAIN.String(), 2366 ContainerIndex: 0, 2367 RampUp: false, 2368 OwnerPoolName: state.PoolNameShare, 2369 AllocationResult: machine.MustParse("4,5,6,7,12,13"), 2370 OriginalAllocationResult: machine.MustParse("4,5,6,7,12,13"), 2371 TopologyAwareAssignments: map[int]machine.CPUSet{ 2372 2: machine.NewCPUSet(4, 5, 12, 13), 2373 3: machine.NewCPUSet(6, 7), 2374 }, 2375 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2376 2: machine.NewCPUSet(4, 5, 12, 13), 2377 3: machine.NewCPUSet(6, 7), 2378 }, 2379 Labels: map[string]string{ 2380 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2381 }, 2382 Annotations: map[string]string{ 2383 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2384 }, 2385 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2386 RequestQuantity: 2, 2387 }, 2388 }, 2389 pod3UID: state.ContainerEntries{ 2390 testName: &state.AllocationInfo{ 2391 PodUid: pod3UID, 2392 PodNamespace: testName, 2393 PodName: testName, 2394 ContainerName: testName, 2395 ContainerType: pluginapi.ContainerType_MAIN.String(), 2396 ContainerIndex: 0, 2397 RampUp: false, 2398 OwnerPoolName: state.PoolNameReclaim, 2399 AllocationResult: machine.MustParse("1,3,14-15"), 2400 OriginalAllocationResult: machine.MustParse("1,3,14-15"), 2401 TopologyAwareAssignments: map[int]machine.CPUSet{ 2402 0: machine.NewCPUSet(1), 2403 1: machine.NewCPUSet(3), 2404 3: machine.NewCPUSet(14, 15), 2405 }, 2406 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2407 0: machine.NewCPUSet(1), 2408 1: machine.NewCPUSet(3), 2409 3: machine.NewCPUSet(14, 15), 2410 }, 2411 Labels: map[string]string{ 2412 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2413 }, 2414 Annotations: map[string]string{ 2415 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2416 }, 2417 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2418 RequestQuantity: 2, 2419 }, 2420 }, 2421 pod4UID: state.ContainerEntries{ 2422 testName: &state.AllocationInfo{ 2423 PodUid: pod4UID, 2424 PodNamespace: testName, 2425 PodName: testName, 2426 ContainerName: testName, 2427 ContainerType: pluginapi.ContainerType_MAIN.String(), 2428 ContainerIndex: 0, 2429 RampUp: false, 2430 OwnerPoolName: state.PoolNameDedicated, 2431 AllocationResult: machine.NewCPUSet(1, 3, 8, 9, 10, 11), 2432 OriginalAllocationResult: machine.NewCPUSet(1, 3, 8, 9, 10, 11), 2433 TopologyAwareAssignments: map[int]machine.CPUSet{ 2434 0: machine.NewCPUSet(1, 8, 9), 2435 1: machine.NewCPUSet(3, 10, 11), 2436 }, 2437 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2438 0: machine.NewCPUSet(1, 8, 9), 2439 1: machine.NewCPUSet(3, 10, 11), 2440 }, 2441 Labels: map[string]string{ 2442 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2443 }, 2444 Annotations: map[string]string{ 2445 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2446 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, 2447 }, 2448 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2449 RequestQuantity: 2, 2450 }, 2451 }, 2452 state.PoolNameReclaim: state.ContainerEntries{ 2453 "": &state.AllocationInfo{ 2454 PodUid: state.PoolNameReclaim, 2455 OwnerPoolName: state.PoolNameReclaim, 2456 AllocationResult: machine.MustParse("1,3,14-15"), 2457 OriginalAllocationResult: machine.MustParse("1,3,14-15"), 2458 TopologyAwareAssignments: map[int]machine.CPUSet{ 2459 0: machine.NewCPUSet(1), 2460 1: machine.NewCPUSet(3), 2461 3: machine.NewCPUSet(14, 15), 2462 }, 2463 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2464 0: machine.NewCPUSet(1), 2465 1: machine.NewCPUSet(3), 2466 3: machine.NewCPUSet(14, 15), 2467 }, 2468 }, 2469 }, 2470 state.PoolNameShare: state.ContainerEntries{ 2471 "": &state.AllocationInfo{ 2472 PodUid: state.PoolNameShare, 2473 OwnerPoolName: state.PoolNameShare, 2474 AllocationResult: machine.MustParse("4,5,6,7,12,13"), 2475 OriginalAllocationResult: machine.MustParse("4,5,6,7,12,13"), 2476 TopologyAwareAssignments: map[int]machine.CPUSet{ 2477 2: machine.NewCPUSet(4, 5, 12, 13), 2478 3: machine.NewCPUSet(6, 7), 2479 }, 2480 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2481 2: machine.NewCPUSet(4, 5, 12, 13), 2482 3: machine.NewCPUSet(6, 7), 2483 }, 2484 }, 2485 }, 2486 state.PoolNameReserve: state.ContainerEntries{ 2487 "": &state.AllocationInfo{ 2488 PodUid: state.PoolNameReserve, 2489 OwnerPoolName: state.PoolNameReserve, 2490 AllocationResult: machine.MustParse("0,2"), 2491 OriginalAllocationResult: machine.MustParse("0,2"), 2492 TopologyAwareAssignments: map[int]machine.CPUSet{ 2493 0: machine.NewCPUSet(0), 2494 1: machine.NewCPUSet(2), 2495 }, 2496 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2497 0: machine.NewCPUSet(0), 2498 1: machine.NewCPUSet(2), 2499 }, 2500 }, 2501 }, 2502 }, 2503 expectedMachineState: state.NUMANodeMap{ 2504 0: &state.NUMANodeState{ 2505 DefaultCPUSet: machine.NewCPUSet(0), 2506 AllocatedCPUSet: machine.NewCPUSet(1, 8, 9), 2507 PodEntries: state.PodEntries{ 2508 pod3UID: state.ContainerEntries{ 2509 testName: &state.AllocationInfo{ 2510 PodUid: pod3UID, 2511 PodNamespace: testName, 2512 PodName: testName, 2513 ContainerName: testName, 2514 ContainerType: pluginapi.ContainerType_MAIN.String(), 2515 ContainerIndex: 0, 2516 RampUp: false, 2517 OwnerPoolName: state.PoolNameReclaim, 2518 AllocationResult: machine.MustParse("1"), 2519 OriginalAllocationResult: machine.MustParse("1"), 2520 TopologyAwareAssignments: map[int]machine.CPUSet{ 2521 0: machine.NewCPUSet(1), 2522 }, 2523 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2524 0: machine.NewCPUSet(1), 2525 }, 2526 Labels: map[string]string{ 2527 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2528 }, 2529 Annotations: map[string]string{ 2530 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2531 }, 2532 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2533 RequestQuantity: 2, 2534 }, 2535 }, 2536 pod4UID: state.ContainerEntries{ 2537 testName: &state.AllocationInfo{ 2538 PodUid: pod4UID, 2539 PodNamespace: testName, 2540 PodName: testName, 2541 ContainerName: testName, 2542 ContainerType: pluginapi.ContainerType_MAIN.String(), 2543 ContainerIndex: 0, 2544 RampUp: false, 2545 OwnerPoolName: state.PoolNameDedicated, 2546 AllocationResult: machine.NewCPUSet(1, 8, 9), 2547 OriginalAllocationResult: machine.NewCPUSet(1, 8, 9), 2548 TopologyAwareAssignments: map[int]machine.CPUSet{ 2549 0: machine.NewCPUSet(1, 8, 9), 2550 }, 2551 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2552 0: machine.NewCPUSet(1, 8, 9), 2553 }, 2554 Labels: map[string]string{ 2555 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2556 }, 2557 Annotations: map[string]string{ 2558 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2559 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2560 }, 2561 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2562 RequestQuantity: 2, 2563 }, 2564 }, 2565 }, 2566 }, 2567 1: &state.NUMANodeState{ 2568 DefaultCPUSet: machine.NewCPUSet(2), 2569 AllocatedCPUSet: machine.NewCPUSet(3, 10, 11), 2570 PodEntries: state.PodEntries{ 2571 pod3UID: state.ContainerEntries{ 2572 testName: &state.AllocationInfo{ 2573 PodUid: pod3UID, 2574 PodNamespace: testName, 2575 PodName: testName, 2576 ContainerName: testName, 2577 ContainerType: pluginapi.ContainerType_MAIN.String(), 2578 ContainerIndex: 0, 2579 RampUp: false, 2580 OwnerPoolName: state.PoolNameReclaim, 2581 AllocationResult: machine.MustParse("3"), 2582 OriginalAllocationResult: machine.MustParse("3"), 2583 TopologyAwareAssignments: map[int]machine.CPUSet{ 2584 0: machine.NewCPUSet(3), 2585 }, 2586 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2587 0: machine.NewCPUSet(3), 2588 }, 2589 Labels: map[string]string{ 2590 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2591 }, 2592 Annotations: map[string]string{ 2593 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2594 }, 2595 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2596 RequestQuantity: 2, 2597 }, 2598 }, 2599 pod4UID: state.ContainerEntries{ 2600 testName: &state.AllocationInfo{ 2601 PodUid: pod4UID, 2602 PodNamespace: testName, 2603 PodName: testName, 2604 ContainerName: testName, 2605 ContainerType: pluginapi.ContainerType_MAIN.String(), 2606 ContainerIndex: 0, 2607 RampUp: false, 2608 OwnerPoolName: state.PoolNameDedicated, 2609 AllocationResult: machine.NewCPUSet(3, 10, 11), 2610 OriginalAllocationResult: machine.NewCPUSet(3, 10, 11), 2611 TopologyAwareAssignments: map[int]machine.CPUSet{ 2612 1: machine.NewCPUSet(3, 10, 11), 2613 }, 2614 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2615 1: machine.NewCPUSet(3, 10, 11), 2616 }, 2617 Labels: map[string]string{ 2618 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2619 }, 2620 Annotations: map[string]string{ 2621 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2622 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2623 }, 2624 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2625 RequestQuantity: 2, 2626 }, 2627 }, 2628 }, 2629 }, 2630 2: &state.NUMANodeState{ 2631 DefaultCPUSet: cpuTopology.CPUDetails.CPUsInNUMANodes(2).Clone(), 2632 AllocatedCPUSet: machine.NewCPUSet(), 2633 PodEntries: state.PodEntries{ 2634 pod1UID: state.ContainerEntries{ 2635 testName: &state.AllocationInfo{ 2636 PodUid: pod1UID, 2637 PodNamespace: testName, 2638 PodName: testName, 2639 ContainerName: testName, 2640 ContainerType: pluginapi.ContainerType_MAIN.String(), 2641 ContainerIndex: 0, 2642 RampUp: false, 2643 OwnerPoolName: state.PoolNameShare, 2644 AllocationResult: machine.MustParse("4-5,12-13"), 2645 OriginalAllocationResult: machine.MustParse("4-5,12-13"), 2646 TopologyAwareAssignments: map[int]machine.CPUSet{ 2647 2: machine.NewCPUSet(4, 5, 12, 13), 2648 }, 2649 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2650 2: machine.NewCPUSet(4, 5, 12, 13), 2651 }, 2652 Labels: map[string]string{ 2653 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2654 }, 2655 Annotations: map[string]string{ 2656 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2657 }, 2658 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2659 RequestQuantity: 2, 2660 }, 2661 }, 2662 pod2UID: state.ContainerEntries{ 2663 testName: &state.AllocationInfo{ 2664 PodUid: pod2UID, 2665 PodNamespace: testName, 2666 PodName: testName, 2667 ContainerName: testName, 2668 ContainerType: pluginapi.ContainerType_MAIN.String(), 2669 ContainerIndex: 0, 2670 RampUp: false, 2671 OwnerPoolName: state.PoolNameShare, 2672 AllocationResult: machine.MustParse("4-5,12-13"), 2673 OriginalAllocationResult: machine.MustParse("4-5,12-13"), 2674 TopologyAwareAssignments: map[int]machine.CPUSet{ 2675 2: machine.NewCPUSet(4, 5, 12, 13), 2676 }, 2677 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2678 2: machine.NewCPUSet(4, 5, 12, 13), 2679 }, 2680 Labels: map[string]string{ 2681 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2682 }, 2683 Annotations: map[string]string{ 2684 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2685 }, 2686 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2687 RequestQuantity: 2, 2688 }, 2689 }, 2690 //pod3UID: state.ContainerEntries{ 2691 // testName: &state.AllocationInfo{ 2692 // PodUid: pod3UID, 2693 // PodNamespace: testName, 2694 // PodName: testName, 2695 // ContainerName: testName, 2696 // ContainerType: pluginapi.ContainerType_MAIN.String(), 2697 // ContainerIndex: 0, 2698 // RampUp: false, 2699 // OwnerPoolName: state.PoolNameReclaim, 2700 // AllocationResult: machine.MustParse("5,13"), 2701 // OriginalAllocationResult: machine.MustParse("5,13"), 2702 // TopologyAwareAssignments: map[int]machine.CPUSet{ 2703 // 2: machine.NewCPUSet(5, 13), 2704 // }, 2705 // OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2706 // 2: machine.NewCPUSet(5, 13), 2707 // }, 2708 // Labels: map[string]string{ 2709 // consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2710 // }, 2711 // Annotations: map[string]string{ 2712 // consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2713 // }, 2714 // QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2715 // RequestQuantity: 2, 2716 // }, 2717 //}, 2718 }, 2719 }, 2720 3: &state.NUMANodeState{ 2721 DefaultCPUSet: cpuTopology.CPUDetails.CPUsInNUMANodes(3).Clone(), 2722 AllocatedCPUSet: machine.NewCPUSet(), 2723 PodEntries: state.PodEntries{ 2724 pod1UID: state.ContainerEntries{ 2725 testName: &state.AllocationInfo{ 2726 PodUid: pod1UID, 2727 PodNamespace: testName, 2728 PodName: testName, 2729 ContainerName: testName, 2730 ContainerType: pluginapi.ContainerType_MAIN.String(), 2731 ContainerIndex: 0, 2732 RampUp: false, 2733 OwnerPoolName: state.PoolNameShare, 2734 AllocationResult: machine.MustParse("6,7"), 2735 OriginalAllocationResult: machine.MustParse("6,7"), 2736 TopologyAwareAssignments: map[int]machine.CPUSet{ 2737 3: machine.NewCPUSet(6, 7), 2738 }, 2739 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2740 3: machine.NewCPUSet(6, 7), 2741 }, 2742 Labels: map[string]string{ 2743 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2744 }, 2745 Annotations: map[string]string{ 2746 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2747 }, 2748 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2749 RequestQuantity: 2, 2750 }, 2751 }, 2752 pod2UID: state.ContainerEntries{ 2753 testName: &state.AllocationInfo{ 2754 PodUid: pod2UID, 2755 PodNamespace: testName, 2756 PodName: testName, 2757 ContainerName: testName, 2758 ContainerType: pluginapi.ContainerType_MAIN.String(), 2759 ContainerIndex: 0, 2760 RampUp: false, 2761 OwnerPoolName: state.PoolNameShare, 2762 AllocationResult: machine.MustParse("6,7"), 2763 OriginalAllocationResult: machine.MustParse("6,7"), 2764 TopologyAwareAssignments: map[int]machine.CPUSet{ 2765 3: machine.NewCPUSet(6, 7), 2766 }, 2767 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2768 3: machine.NewCPUSet(6, 7), 2769 }, 2770 Labels: map[string]string{ 2771 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2772 }, 2773 Annotations: map[string]string{ 2774 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2775 }, 2776 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2777 RequestQuantity: 2, 2778 }, 2779 }, 2780 pod3UID: state.ContainerEntries{ 2781 testName: &state.AllocationInfo{ 2782 PodUid: pod3UID, 2783 PodNamespace: testName, 2784 PodName: testName, 2785 ContainerName: testName, 2786 ContainerType: pluginapi.ContainerType_MAIN.String(), 2787 ContainerIndex: 0, 2788 RampUp: false, 2789 OwnerPoolName: state.PoolNameReclaim, 2790 AllocationResult: machine.MustParse("14,15"), 2791 OriginalAllocationResult: machine.MustParse("14,15"), 2792 TopologyAwareAssignments: map[int]machine.CPUSet{ 2793 3: machine.NewCPUSet(14, 15), 2794 }, 2795 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 2796 3: machine.NewCPUSet(14, 15), 2797 }, 2798 Labels: map[string]string{ 2799 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2800 }, 2801 Annotations: map[string]string{ 2802 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2803 }, 2804 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2805 RequestQuantity: 2, 2806 }, 2807 }, 2808 }, 2809 }, 2810 }, 2811 cpuTopology: cpuTopology, 2812 }, 2813 } 2814 2815 for i, tc := range testCases { 2816 tmpDir, err := ioutil.TempDir("", fmt.Sprintf("checkpoint-TestAllocateByQoSAwareServerListAndWatchResp-%v", i)) 2817 as.Nil(err) 2818 2819 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(tc.cpuTopology, tmpDir) 2820 as.Nil(err) 2821 2822 dynamicPolicy.dynamicConfig.GetDynamicConfiguration().EnableReclaim = true 2823 2824 machineState, err := generateMachineStateFromPodEntries(tc.cpuTopology, tc.podEntries) 2825 as.Nil(err) 2826 2827 dynamicPolicy.state.SetPodEntries(tc.podEntries) 2828 dynamicPolicy.state.SetMachineState(machineState) 2829 dynamicPolicy.initReservePool() 2830 2831 err = dynamicPolicy.allocateByCPUAdvisor(tc.lwResp) 2832 as.Nilf(err, "dynamicPolicy.allocateByCPUAdvisorServerListAndWatchResp got err: %v, case: %s", err, tc.description) 2833 2834 match, err := entriesMatch(tc.expectedPodEntries, dynamicPolicy.state.GetPodEntries()) 2835 as.Nilf(err, "failed in test case: %s", tc.description) 2836 as.Equalf(true, match, "failed in test case: %s", tc.description) 2837 2838 match, err = machineStateMatch(tc.expectedMachineState, dynamicPolicy.state.GetMachineState()) 2839 as.Nilf(err, "failed in test case: %s", tc.description) 2840 as.Equalf(true, match, "failed in test case: %s", tc.description) 2841 2842 os.RemoveAll(tmpDir) 2843 } 2844 } 2845 2846 func machineStateMatch(state1, state2 state.NUMANodeMap) (bool, error) { 2847 if len(state1) != len(state2) { 2848 return false, nil 2849 } 2850 2851 for numaId, numaState := range state1 { 2852 if numaState == nil { 2853 return false, fmt.Errorf("empty allocationInfo") 2854 } 2855 2856 if state2[numaId] == nil { 2857 fmt.Printf("NUMA: %d isn't found in state2\n", numaId) 2858 return false, nil 2859 } 2860 2861 if numaState.DefaultCPUSet.Size() != state2[numaId].DefaultCPUSet.Size() { 2862 fmt.Printf("NUMA: %d DefaultCPUSet: (state1: %d, state2: %d) not match\n", 2863 numaId, numaState.DefaultCPUSet.Size(), state2[numaId].DefaultCPUSet.Size()) 2864 return false, nil 2865 } else if numaState.AllocatedCPUSet.Size() != state2[numaId].AllocatedCPUSet.Size() { 2866 fmt.Printf("NUMA: %d AllocatedCPUSet: (state1: %d, state2: %d) not match\n", 2867 numaId, numaState.AllocatedCPUSet.Size(), state2[numaId].AllocatedCPUSet.Size()) 2868 return false, nil 2869 } 2870 } 2871 2872 return true, nil 2873 } 2874 2875 func entriesMatch(entries1, entries2 state.PodEntries) (bool, error) { 2876 if len(entries1) != len(entries2) { 2877 return false, nil 2878 } 2879 2880 for entryName, subEntries := range entries1 { 2881 for subEntryName, allocationInfo := range subEntries { 2882 if allocationInfo == nil { 2883 return false, fmt.Errorf("empty allocationInfo") 2884 } 2885 2886 if entries2[entryName][subEntryName] == nil { 2887 fmt.Printf("%s:%s isn't found in entries2\n", entryName, subEntryName) 2888 return false, nil 2889 } 2890 2891 if allocationInfo.AllocationResult.Size() != entries2[entryName][subEntryName].AllocationResult.Size() { 2892 fmt.Printf("%s:%s allocationResult: (entries1: %d, entries2: %d) isn't match\n", 2893 entryName, subEntryName, allocationInfo.AllocationResult.Size(), 2894 entries2[entryName][subEntryName].AllocationResult.Size()) 2895 return false, nil 2896 } 2897 2898 if allocationInfo.QoSLevel == consts.PodAnnotationQoSLevelDedicatedCores && 2899 allocationInfo.Annotations[consts.PodAnnotationMemoryEnhancementNumaBinding] == consts.PodAnnotationMemoryEnhancementNumaBindingEnable { 2900 for numaId, cset := range allocationInfo.TopologyAwareAssignments { 2901 if cset.Size() != entries2[entryName][subEntryName].TopologyAwareAssignments[numaId].Size() { 2902 fmt.Printf("%s:%s NUMA: %d allocationResult: (entries1: %d, entries2: %d) not match\n", 2903 entryName, subEntryName, numaId, cset.Size(), 2904 entries2[entryName][subEntryName].TopologyAwareAssignments[numaId].Size()) 2905 return false, nil 2906 } 2907 } 2908 } 2909 } 2910 } 2911 2912 return true, nil 2913 } 2914 2915 func TestGetReadonlyState(t *testing.T) { 2916 t.Parallel() 2917 2918 as := require.New(t) 2919 readonlyState, err := GetReadonlyState() 2920 as.NotNil(err) 2921 as.Nil(readonlyState) 2922 } 2923 2924 func TestClearResidualState(t *testing.T) { 2925 t.Parallel() 2926 2927 as := require.New(t) 2928 2929 tmpDir, err := ioutil.TempDir("", "checkpoint_TestClearResidualState") 2930 as.Nil(err) 2931 defer os.RemoveAll(tmpDir) 2932 2933 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2934 as.Nil(err) 2935 2936 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) 2937 as.Nil(err) 2938 2939 dynamicPolicy.clearResidualState(nil, nil, nil, nil, nil) 2940 } 2941 2942 func TestStart(t *testing.T) { 2943 t.Parallel() 2944 2945 as := require.New(t) 2946 2947 tmpDir, err := ioutil.TempDir("", "checkpoint_TestStart") 2948 as.Nil(err) 2949 defer os.RemoveAll(tmpDir) 2950 2951 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2952 as.Nil(err) 2953 2954 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) 2955 as.Nil(err) 2956 2957 err = dynamicPolicy.Start() 2958 as.Nil(err) 2959 } 2960 2961 func TestStop(t *testing.T) { 2962 t.Parallel() 2963 2964 as := require.New(t) 2965 2966 tmpDir, err := ioutil.TempDir("", "checkpoint_TestStop") 2967 as.Nil(err) 2968 defer os.RemoveAll(tmpDir) 2969 2970 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2971 as.Nil(err) 2972 2973 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) 2974 as.Nil(err) 2975 2976 err = dynamicPolicy.Stop() 2977 as.Nil(err) 2978 } 2979 2980 func TestCheckCPUSet(t *testing.T) { 2981 t.Parallel() 2982 2983 as := require.New(t) 2984 2985 tmpDir, err := ioutil.TempDir("", "checkpoint_TestCheckCPUSet") 2986 as.Nil(err) 2987 defer os.RemoveAll(tmpDir) 2988 2989 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2990 as.Nil(err) 2991 2992 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) 2993 as.Nil(err) 2994 2995 dynamicPolicy.checkCPUSet(nil, nil, nil, nil, nil) 2996 } 2997 2998 func TestSchedIdle(t *testing.T) { 2999 t.Parallel() 3000 3001 as := require.New(t) 3002 3003 _, err1 := os.Stat("/sys/fs/cgroup/cpu/kubepods/cpu.idle") 3004 _, err2 := os.Stat("/sys/fs/cgroup/kubepods/cpu.idle") 3005 3006 support := err1 == nil && err2 == nil 3007 3008 as.Equalf(support, cgroupcm.IsCPUIdleSupported(), "sched idle support status isn't correct") 3009 3010 if cgroupcm.IsCPUIdleSupported() { 3011 absCgroupPath := common.GetAbsCgroupPath("cpu", "test") 3012 3013 fs := &utilfs.DefaultFs{} 3014 err := fs.MkdirAll(absCgroupPath, 0o755) 3015 3016 as.Nil(err) 3017 3018 var enableCPUIdle bool 3019 _ = cgroupcmutils.ApplyCPUWithRelativePath("test", &cgroupcm.CPUData{CpuIdlePtr: &enableCPUIdle}) 3020 3021 contents, err := ioutil.ReadFile(filepath.Join(absCgroupPath, "cpu.idle")) //nolint:gosec 3022 as.Nil(err) 3023 3024 as.Equal("1", contents) 3025 } 3026 } 3027 3028 func TestRemoveContainer(t *testing.T) { 3029 t.Parallel() 3030 3031 as := require.New(t) 3032 3033 tmpDir, err := ioutil.TempDir("", "checkpoint-TestRemoveContainer") 3034 as.Nil(err) 3035 defer os.RemoveAll(tmpDir) 3036 3037 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 3038 as.Nil(err) 3039 3040 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) 3041 as.Nil(err) 3042 3043 podUID := string(uuid.NewUUID()) 3044 containerName := "testName" 3045 testName := "testName" 3046 3047 podEntries := state.PodEntries{ 3048 podUID: state.ContainerEntries{ 3049 containerName: &state.AllocationInfo{ 3050 PodUid: podUID, 3051 PodNamespace: testName, 3052 PodName: testName, 3053 ContainerName: containerName, 3054 ContainerType: pluginapi.ContainerType_MAIN.String(), 3055 ContainerIndex: 0, 3056 RampUp: false, 3057 OwnerPoolName: state.PoolNameShare, 3058 AllocationResult: machine.MustParse("1,3-6,9,11-14"), 3059 OriginalAllocationResult: machine.MustParse("1,3-6,9,11-14"), 3060 TopologyAwareAssignments: map[int]machine.CPUSet{ 3061 0: machine.NewCPUSet(1, 9), 3062 1: machine.NewCPUSet(3, 11), 3063 2: machine.NewCPUSet(4, 5, 11, 12), 3064 3: machine.NewCPUSet(6, 14), 3065 }, 3066 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 3067 0: machine.NewCPUSet(1, 9), 3068 1: machine.NewCPUSet(3, 11), 3069 2: machine.NewCPUSet(4, 5, 11, 12), 3070 3: machine.NewCPUSet(6, 14), 3071 }, 3072 Labels: map[string]string{ 3073 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 3074 }, 3075 Annotations: map[string]string{ 3076 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 3077 }, 3078 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 3079 RequestQuantity: 2, 3080 }, 3081 }, 3082 } 3083 3084 dynamicPolicy.state.SetPodEntries(podEntries) 3085 3086 allocationInfo := dynamicPolicy.state.GetAllocationInfo(podUID, containerName) 3087 as.NotNil(allocationInfo) 3088 3089 dynamicPolicy.removeContainer(podUID, containerName) 3090 3091 allocationInfo = dynamicPolicy.state.GetAllocationInfo(podUID, containerName) 3092 as.Nil(allocationInfo) 3093 3094 dynamicPolicy.removeContainer(podUID, containerName) 3095 3096 allocationInfo = dynamicPolicy.state.GetAllocationInfo(podUID, containerName) 3097 as.Nil(allocationInfo) 3098 } 3099 3100 func TestShoudSharedCoresRampUp(t *testing.T) { 3101 t.Parallel() 3102 3103 as := require.New(t) 3104 3105 tmpDir, err := ioutil.TempDir("", "checkpoint-TestShoudSharedCoresRampUp") 3106 as.Nil(err) 3107 defer func() { _ = os.RemoveAll(tmpDir) }() 3108 3109 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 3110 as.Nil(err) 3111 3112 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, tmpDir) 3113 as.Nil(err) 3114 3115 dynamicPolicy.state.SetAllocationInfo(state.PoolNameShare, "", &state.AllocationInfo{ 3116 PodUid: state.PoolNameShare, 3117 OwnerPoolName: state.PoolNameShare, 3118 AllocationResult: machine.MustParse("1,3-6,9,11-14"), 3119 OriginalAllocationResult: machine.MustParse("1,3-6,9,11-14"), 3120 TopologyAwareAssignments: map[int]machine.CPUSet{ 3121 0: machine.NewCPUSet(1, 9), 3122 1: machine.NewCPUSet(3, 11), 3123 2: machine.NewCPUSet(4, 5, 11, 12), 3124 3: machine.NewCPUSet(6, 14), 3125 }, 3126 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 3127 0: machine.NewCPUSet(1, 9), 3128 1: machine.NewCPUSet(3, 11), 3129 2: machine.NewCPUSet(4, 5, 11, 12), 3130 3: machine.NewCPUSet(6, 14), 3131 }, 3132 }) 3133 3134 existPodUID := uuid.NewUUID() 3135 existName := "exist" 3136 dynamicPolicy.state.SetAllocationInfo(string(existPodUID), existName, &state.AllocationInfo{ 3137 PodUid: string(existPodUID), 3138 PodNamespace: existName, 3139 PodName: existName, 3140 ContainerName: existName, 3141 ContainerType: pluginapi.ContainerType_MAIN.String(), 3142 ContainerIndex: 0, 3143 RampUp: false, 3144 OwnerPoolName: state.PoolNameShare, 3145 AllocationResult: machine.MustParse("1,3-6,9,11-14"), 3146 OriginalAllocationResult: machine.MustParse("1,3-6,9,11-14"), 3147 TopologyAwareAssignments: map[int]machine.CPUSet{ 3148 0: machine.NewCPUSet(1, 9), 3149 1: machine.NewCPUSet(3, 11), 3150 2: machine.NewCPUSet(4, 5, 11, 12), 3151 3: machine.NewCPUSet(6, 14), 3152 }, 3153 OriginalTopologyAwareAssignments: map[int]machine.CPUSet{ 3154 0: machine.NewCPUSet(1, 9), 3155 1: machine.NewCPUSet(3, 11), 3156 2: machine.NewCPUSet(4, 5, 11, 12), 3157 3: machine.NewCPUSet(6, 14), 3158 }, 3159 Labels: map[string]string{ 3160 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 3161 }, 3162 Annotations: map[string]string{ 3163 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 3164 }, 3165 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 3166 RequestQuantity: 2, 3167 }) 3168 3169 testName := "test" 3170 podUID := uuid.NewUUID() 3171 dynamicPolicy.metaServer = &metaserver.MetaServer{ 3172 MetaAgent: &agent.MetaAgent{ 3173 PodFetcher: &pod.PodFetcherStub{ 3174 PodList: []*v1.Pod{ 3175 { 3176 ObjectMeta: metav1.ObjectMeta{ 3177 Name: testName, 3178 Namespace: testName, 3179 UID: podUID, 3180 }, 3181 Status: v1.PodStatus{ 3182 Phase: v1.PodRunning, 3183 }, 3184 }, 3185 }, 3186 }, 3187 }, 3188 } 3189 3190 req := &pluginapi.ResourceRequest{ 3191 PodUid: string(podUID), 3192 PodNamespace: testName, 3193 PodName: testName, 3194 ContainerName: testName, 3195 ContainerType: pluginapi.ContainerType_MAIN, 3196 ContainerIndex: 0, 3197 ResourceName: string(v1.ResourceCPU), 3198 ResourceRequests: map[string]float64{ 3199 string(v1.ResourceCPU): 2, 3200 }, 3201 Labels: map[string]string{}, 3202 Annotations: map[string]string{}, 3203 } 3204 3205 _, err = dynamicPolicy.Allocate(context.Background(), req) 3206 as.Nil(err) 3207 3208 allocationInfo := dynamicPolicy.state.GetAllocationInfo(req.PodUid, testName) 3209 as.NotNil(allocationInfo) 3210 as.Equal(false, allocationInfo.RampUp) 3211 as.Equal(allocationInfo.OwnerPoolName, state.PoolNameShare) 3212 }