github.com/kubewharf/katalyst-core@v0.5.3/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go (about) 1 //go:build linux 2 // +build linux 3 4 /* 5 Copyright 2022 The Katalyst Authors. 6 7 Licensed under the Apache License, Version 2.0 (the "License"); 8 you may not use this file except in compliance with the License. 9 You may obtain a copy of the License at 10 11 http://www.apache.org/licenses/LICENSE-2.0 12 13 Unless required by applicable law or agreed to in writing, software 14 distributed under the License is distributed on an "AS IS" BASIS, 15 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 See the License for the specific language governing permissions and 17 limitations under the License. 18 */ 19 20 package dynamicpolicy 21 22 import ( 23 "context" 24 "fmt" 25 "io/ioutil" 26 "os" 27 "path" 28 "path/filepath" 29 "reflect" 30 "strconv" 31 "strings" 32 "sync" 33 "testing" 34 "time" 35 36 "github.com/cilium/ebpf" 37 "github.com/cilium/ebpf/rlimit" 38 info "github.com/google/cadvisor/info/v1" 39 "github.com/stretchr/testify/assert" 40 "github.com/stretchr/testify/require" 41 v1 "k8s.io/api/core/v1" 42 "k8s.io/apimachinery/pkg/api/resource" 43 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 44 "k8s.io/apimachinery/pkg/runtime" 45 "k8s.io/apimachinery/pkg/types" 46 "k8s.io/apimachinery/pkg/util/uuid" 47 pluginapi "k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1" 48 maputil "k8s.io/kubernetes/pkg/util/maps" 49 50 "github.com/kubewharf/katalyst-api/pkg/consts" 51 katalystbase "github.com/kubewharf/katalyst-core/cmd/base" 52 appagent "github.com/kubewharf/katalyst-core/cmd/katalyst-agent/app/agent" 53 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/advisorsvc" 54 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/commonstate" 55 memconsts "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/consts" 56 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/memoryadvisor" 57 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/oom" 58 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/state" 59 "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util" 60 "github.com/kubewharf/katalyst-core/pkg/config" 61 configagent "github.com/kubewharf/katalyst-core/pkg/config/agent" 62 "github.com/kubewharf/katalyst-core/pkg/config/agent/global" 63 qrmconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/qrm" 64 "github.com/kubewharf/katalyst-core/pkg/config/generic" 65 coreconsts "github.com/kubewharf/katalyst-core/pkg/consts" 66 "github.com/kubewharf/katalyst-core/pkg/metaserver" 67 "github.com/kubewharf/katalyst-core/pkg/metaserver/agent" 68 metaserveragent "github.com/kubewharf/katalyst-core/pkg/metaserver/agent" 69 "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod" 70 "github.com/kubewharf/katalyst-core/pkg/metaserver/external" 71 "github.com/kubewharf/katalyst-core/pkg/metrics" 72 "github.com/kubewharf/katalyst-core/pkg/util/asyncworker" 73 "github.com/kubewharf/katalyst-core/pkg/util/machine" 74 "github.com/kubewharf/katalyst-core/pkg/util/qos" 75 ) 76 77 const ( 78 podDebugAnnoKey = "qrm.katalyst.kubewharf.io/debug_pod" 79 ) 80 81 var fakeConf = &config.Configuration{ 82 AgentConfiguration: &configagent.AgentConfiguration{ 83 GenericAgentConfiguration: &configagent.GenericAgentConfiguration{ 84 GenericQRMPluginConfiguration: &qrmconfig.GenericQRMPluginConfiguration{ 85 UseKubeletReservedConfig: false, 86 }, 87 }, 88 StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ 89 QRMPluginsConfiguration: &qrmconfig.QRMPluginsConfiguration{ 90 MemoryQRMPluginConfig: &qrmconfig.MemoryQRMPluginConfig{ 91 ReservedMemoryGB: 4, 92 }, 93 }, 94 }, 95 }, 96 } 97 98 func getTestDynamicPolicyWithInitialization(topology *machine.CPUTopology, machineInfo *info.MachineInfo, stateFileDirectory string) (*DynamicPolicy, error) { 99 reservedMemory, err := getReservedMemory(fakeConf, &metaserver.MetaServer{}, machineInfo) 100 if err != nil { 101 return nil, err 102 } 103 104 resourcesReservedMemory := map[v1.ResourceName]map[int]uint64{ 105 v1.ResourceMemory: reservedMemory, 106 } 107 108 qosConfig := generic.NewQoSConfiguration() 109 qosConfig.SetExpandQoSLevelSelector(consts.PodAnnotationQoSLevelSharedCores, map[string]string{ 110 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 111 }) 112 qosConfig.SetExpandQoSLevelSelector(consts.PodAnnotationQoSLevelDedicatedCores, map[string]string{ 113 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 114 }) 115 qosConfig.SetExpandQoSLevelSelector(consts.PodAnnotationQoSLevelReclaimedCores, map[string]string{ 116 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 117 }) 118 119 stateImpl, err := state.NewCheckpointState(stateFileDirectory, memoryPluginStateFileName, 120 memconsts.MemoryResourcePluginPolicyNameDynamic, topology, machineInfo, resourcesReservedMemory, false) 121 if err != nil { 122 return nil, fmt.Errorf("NewCheckpointState failed with error: %v", err) 123 } 124 125 policyImplement := &DynamicPolicy{ 126 topology: topology, 127 qosConfig: qosConfig, 128 state: stateImpl, 129 emitter: metrics.DummyMetrics{}, 130 migratingMemory: make(map[string]map[string]bool), 131 stopCh: make(chan struct{}), 132 podDebugAnnoKeys: []string{podDebugAnnoKey}, 133 } 134 135 policyImplement.allocationHandlers = map[string]util.AllocationHandler{ 136 consts.PodAnnotationQoSLevelSharedCores: policyImplement.sharedCoresAllocationHandler, 137 consts.PodAnnotationQoSLevelDedicatedCores: policyImplement.dedicatedCoresAllocationHandler, 138 consts.PodAnnotationQoSLevelReclaimedCores: policyImplement.reclaimedCoresAllocationHandler, 139 } 140 141 policyImplement.hintHandlers = map[string]util.HintHandler{ 142 consts.PodAnnotationQoSLevelSharedCores: policyImplement.sharedCoresHintHandler, 143 consts.PodAnnotationQoSLevelDedicatedCores: policyImplement.dedicatedCoresHintHandler, 144 consts.PodAnnotationQoSLevelReclaimedCores: policyImplement.reclaimedCoresHintHandler, 145 } 146 147 policyImplement.asyncWorkers = asyncworker.NewAsyncWorkers(memoryPluginAsyncWorkersName, policyImplement.emitter) 148 149 return policyImplement, nil 150 } 151 152 func TestCheckMemorySet(t *testing.T) { 153 t.Parallel() 154 155 as := require.New(t) 156 157 tmpDir, err := ioutil.TempDir("", "checkpoint-TestCheckMemorySet") 158 as.Nil(err) 159 defer os.RemoveAll(tmpDir) 160 161 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 162 as.Nil(err) 163 164 machineInfo := &info.MachineInfo{} 165 166 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 167 as.Nil(err) 168 169 dynamicPolicy.state.SetPodResourceEntries(state.PodResourceEntries{ 170 v1.ResourceMemory: state.PodEntries{ 171 "podUID": state.ContainerEntries{ 172 "testName": &state.AllocationInfo{ 173 PodUid: "podUID", 174 PodNamespace: "testName", 175 PodName: "testName", 176 ContainerName: "testName", 177 ContainerType: pluginapi.ContainerType_MAIN.String(), 178 ContainerIndex: 0, 179 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 180 RampUp: false, 181 AggregatedQuantity: 9663676416, 182 NumaAllocationResult: machine.NewCPUSet(0), 183 TopologyAwareAllocations: map[int]uint64{ 184 0: 9663676416, 185 }, 186 Annotations: map[string]string{ 187 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 188 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 189 }, 190 Labels: map[string]string{ 191 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 192 }, 193 }, 194 }, 195 }, 196 }) 197 198 dynamicPolicy.metaServer = &metaserver.MetaServer{ 199 MetaAgent: &agent.MetaAgent{ 200 PodFetcher: &pod.PodFetcherStub{}, 201 }, 202 } 203 dynamicPolicy.checkMemorySet(nil, nil, nil, nil, nil) 204 } 205 206 func TestClearResidualState(t *testing.T) { 207 t.Parallel() 208 209 as := require.New(t) 210 211 tmpDir, err := ioutil.TempDir("", "checkpoint-TestClearResidualState") 212 as.Nil(err) 213 defer os.RemoveAll(tmpDir) 214 215 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 216 as.Nil(err) 217 218 machineInfo := &info.MachineInfo{} 219 220 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 221 as.Nil(err) 222 223 dynamicPolicy.metaServer = &metaserver.MetaServer{ 224 MetaAgent: &agent.MetaAgent{ 225 PodFetcher: &pod.PodFetcherStub{}, 226 }, 227 } 228 dynamicPolicy.clearResidualState(nil, nil, nil, nil, nil) 229 } 230 231 func TestSetMemoryMigrate(t *testing.T) { 232 t.Parallel() 233 234 as := require.New(t) 235 236 tmpDir, err := ioutil.TempDir("", "checkpoint-TestSetMemoryMigrate") 237 as.Nil(err) 238 defer os.RemoveAll(tmpDir) 239 240 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 241 as.Nil(err) 242 243 machineInfo := &info.MachineInfo{} 244 245 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 246 as.Nil(err) 247 248 dynamicPolicy.state.SetPodResourceEntries(state.PodResourceEntries{ 249 v1.ResourceMemory: state.PodEntries{ 250 "podUID": state.ContainerEntries{ 251 "testName": &state.AllocationInfo{ 252 PodUid: "podUID", 253 PodNamespace: "testName", 254 PodName: "testName", 255 ContainerName: "testName", 256 ContainerType: pluginapi.ContainerType_MAIN.String(), 257 ContainerIndex: 0, 258 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 259 RampUp: false, 260 AggregatedQuantity: 9663676416, 261 NumaAllocationResult: machine.NewCPUSet(0), 262 TopologyAwareAllocations: map[int]uint64{ 263 0: 9663676416, 264 }, 265 Annotations: map[string]string{ 266 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 267 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 268 }, 269 Labels: map[string]string{ 270 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 271 }, 272 }, 273 }, 274 "podUID-1": state.ContainerEntries{ 275 "testName-1": &state.AllocationInfo{ 276 PodUid: "podUID-1", 277 PodNamespace: "testName-1", 278 PodName: "testName-1", 279 ContainerName: "testName-1", 280 ContainerType: pluginapi.ContainerType_MAIN.String(), 281 ContainerIndex: 0, 282 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 283 RampUp: false, 284 AggregatedQuantity: 9663676416, 285 NumaAllocationResult: machine.NewCPUSet(0), 286 TopologyAwareAllocations: map[int]uint64{ 287 0: 9663676416, 288 }, 289 }, 290 }, 291 }, 292 }) 293 294 dynamicPolicy.metaServer = &metaserver.MetaServer{ 295 MetaAgent: &agent.MetaAgent{ 296 PodFetcher: &pod.PodFetcherStub{}, 297 }, 298 } 299 dynamicPolicy.setMemoryMigrate() 300 } 301 302 func TestRemovePod(t *testing.T) { 303 t.Parallel() 304 305 as := require.New(t) 306 307 tmpDir, err := ioutil.TempDir("", "checkpoint-TestRemovePod") 308 as.Nil(err) 309 defer os.RemoveAll(tmpDir) 310 311 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 312 as.Nil(err) 313 314 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 315 as.Nil(err) 316 317 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 318 as.Nil(err) 319 320 testName := "test" 321 322 // test for gt 323 req := &pluginapi.ResourceRequest{ 324 PodUid: string(uuid.NewUUID()), 325 PodNamespace: testName, 326 PodName: testName, 327 ContainerName: testName, 328 ContainerType: pluginapi.ContainerType_MAIN, 329 ContainerIndex: 0, 330 ResourceName: string(v1.ResourceMemory), 331 Hint: &pluginapi.TopologyHint{ 332 Nodes: []uint64{0}, 333 Preferred: true, 334 }, 335 ResourceRequests: map[string]float64{ 336 string(v1.ResourceMemory): 2147483648, 337 }, 338 Annotations: map[string]string{ 339 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 340 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "true"}`, 341 }, 342 Labels: map[string]string{ 343 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 344 }, 345 } 346 347 _, err = dynamicPolicy.Allocate(context.Background(), req) 348 as.Nil(err) 349 350 resp, err := dynamicPolicy.GetTopologyAwareResources(context.Background(), &pluginapi.GetTopologyAwareResourcesRequest{ 351 PodUid: req.PodUid, 352 ContainerName: testName, 353 }) 354 as.Nil(err) 355 356 as.Equal(&pluginapi.GetTopologyAwareResourcesResponse{ 357 PodUid: req.PodUid, 358 PodNamespace: testName, 359 PodName: testName, 360 ContainerTopologyAwareResources: &pluginapi.ContainerTopologyAwareResources{ 361 ContainerName: testName, 362 AllocatedResources: map[string]*pluginapi.TopologyAwareResource{ 363 string(v1.ResourceMemory): { 364 IsNodeResource: false, 365 IsScalarResource: true, 366 AggregatedQuantity: 7516192768, 367 OriginalAggregatedQuantity: 7516192768, 368 TopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 369 {ResourceValue: 7516192768, Node: 0}, 370 }, 371 OriginalTopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 372 {ResourceValue: 7516192768, Node: 0}, 373 }, 374 }, 375 }, 376 }, 377 }, resp) 378 379 dynamicPolicy.RemovePod(context.Background(), &pluginapi.RemovePodRequest{ 380 PodUid: req.PodUid, 381 }) 382 383 _, err = dynamicPolicy.GetTopologyAwareResources(context.Background(), &pluginapi.GetTopologyAwareResourcesRequest{ 384 PodUid: req.PodUid, 385 ContainerName: testName, 386 }) 387 as.NotNil(err) 388 as.True(strings.Contains(err.Error(), "is not show up in memory plugin state")) 389 } 390 391 func TestAllocate(t *testing.T) { 392 t.Parallel() 393 394 as := require.New(t) 395 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 396 as.Nil(err) 397 398 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 399 as.Nil(err) 400 401 testName := "test" 402 403 testCases := []struct { 404 description string 405 req *pluginapi.ResourceRequest 406 expectedResp *pluginapi.ResourceAllocationResponse 407 enhancementDefaultValues map[string]string 408 }{ 409 { 410 description: "req for init container", 411 req: &pluginapi.ResourceRequest{ 412 PodUid: string(uuid.NewUUID()), 413 PodNamespace: testName, 414 PodName: testName, 415 ContainerName: testName, 416 ContainerType: pluginapi.ContainerType_INIT, 417 ContainerIndex: 0, 418 ResourceName: string(v1.ResourceMemory), 419 ResourceRequests: map[string]float64{ 420 string(v1.ResourceMemory): 1048576, 421 }, 422 }, 423 expectedResp: &pluginapi.ResourceAllocationResponse{ 424 PodNamespace: testName, 425 PodName: testName, 426 ContainerName: testName, 427 ContainerType: pluginapi.ContainerType_INIT, 428 ContainerIndex: 0, 429 ResourceName: string(v1.ResourceMemory), 430 AllocationResult: nil, 431 Labels: map[string]string{ 432 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 433 }, 434 Annotations: map[string]string{ 435 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 436 }, 437 }, 438 }, 439 { 440 description: "req for container of debug pod", 441 req: &pluginapi.ResourceRequest{ 442 PodUid: string(uuid.NewUUID()), 443 PodNamespace: testName, 444 PodName: testName, 445 ContainerName: testName, 446 ContainerType: pluginapi.ContainerType_MAIN, 447 ContainerIndex: 0, 448 ResourceName: string(v1.ResourceMemory), 449 ResourceRequests: map[string]float64{ 450 string(v1.ResourceMemory): 1073741824, 451 }, 452 Annotations: map[string]string{ 453 podDebugAnnoKey: "", 454 }, 455 }, 456 expectedResp: &pluginapi.ResourceAllocationResponse{ 457 PodNamespace: testName, 458 PodName: testName, 459 ContainerName: testName, 460 ContainerType: pluginapi.ContainerType_MAIN, 461 ContainerIndex: 0, 462 ResourceName: string(v1.ResourceMemory), 463 AllocationResult: &pluginapi.ResourceAllocation{ 464 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 465 string(v1.ResourceMemory): { 466 IsNodeResource: false, 467 IsScalarResource: true, 468 }, 469 }, 470 }, 471 Labels: map[string]string{ 472 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 473 }, 474 Annotations: map[string]string{ 475 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 476 }, 477 }, 478 }, 479 { 480 description: "req for shared_cores main container", 481 req: &pluginapi.ResourceRequest{ 482 PodUid: string(uuid.NewUUID()), 483 PodNamespace: testName, 484 PodName: testName, 485 ContainerName: testName, 486 ContainerType: pluginapi.ContainerType_MAIN, 487 ContainerIndex: 0, 488 ResourceName: string(v1.ResourceMemory), 489 ResourceRequests: map[string]float64{ 490 string(v1.ResourceMemory): 1073741824, 491 }, 492 }, 493 expectedResp: &pluginapi.ResourceAllocationResponse{ 494 PodNamespace: testName, 495 PodName: testName, 496 ContainerName: testName, 497 ContainerType: pluginapi.ContainerType_MAIN, 498 ContainerIndex: 0, 499 ResourceName: string(v1.ResourceMemory), 500 AllocationResult: &pluginapi.ResourceAllocation{ 501 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 502 string(v1.ResourceMemory): { 503 OciPropertyName: util.OCIPropertyNameCPUSetMems, 504 IsNodeResource: false, 505 IsScalarResource: true, 506 AllocatedQuantity: 0, 507 AllocationResult: machine.NewCPUSet(0, 1, 2, 3).String(), 508 ResourceHints: &pluginapi.ListOfTopologyHints{ 509 Hints: []*pluginapi.TopologyHint{nil}, 510 }, 511 }, 512 }, 513 }, 514 Labels: map[string]string{ 515 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 516 }, 517 Annotations: map[string]string{ 518 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 519 }, 520 }, 521 }, 522 { 523 description: "req for reclaimed_cores main container", 524 req: &pluginapi.ResourceRequest{ 525 PodUid: string(uuid.NewUUID()), 526 PodNamespace: testName, 527 PodName: testName, 528 ContainerName: testName, 529 ContainerType: pluginapi.ContainerType_MAIN, 530 ContainerIndex: 0, 531 ResourceName: string(v1.ResourceMemory), 532 ResourceRequests: map[string]float64{ 533 string(v1.ResourceMemory): 1073741824, 534 }, 535 Annotations: map[string]string{ 536 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 537 }, 538 Labels: map[string]string{ 539 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 540 }, 541 }, 542 expectedResp: &pluginapi.ResourceAllocationResponse{ 543 PodNamespace: testName, 544 PodName: testName, 545 ContainerName: testName, 546 ContainerType: pluginapi.ContainerType_MAIN, 547 ContainerIndex: 0, 548 ResourceName: string(v1.ResourceMemory), 549 AllocationResult: &pluginapi.ResourceAllocation{ 550 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 551 string(v1.ResourceMemory): { 552 OciPropertyName: util.OCIPropertyNameCPUSetMems, 553 IsNodeResource: false, 554 IsScalarResource: true, 555 AllocatedQuantity: 0, 556 AllocationResult: machine.NewCPUSet(0, 1, 2, 3).String(), 557 ResourceHints: &pluginapi.ListOfTopologyHints{ 558 Hints: []*pluginapi.TopologyHint{nil}, 559 }, 560 }, 561 }, 562 }, 563 Labels: map[string]string{ 564 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 565 }, 566 Annotations: map[string]string{ 567 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 568 }, 569 }, 570 }, 571 { 572 description: "req for dedicated_cores with numa_binding & numa_exclusive main container", 573 req: &pluginapi.ResourceRequest{ 574 PodUid: string(uuid.NewUUID()), 575 PodNamespace: testName, 576 PodName: testName, 577 ContainerName: testName, 578 ContainerType: pluginapi.ContainerType_MAIN, 579 ContainerIndex: 0, 580 ResourceName: string(v1.ResourceMemory), 581 Hint: &pluginapi.TopologyHint{ 582 Nodes: []uint64{0}, 583 Preferred: true, 584 }, 585 ResourceRequests: map[string]float64{ 586 string(v1.ResourceMemory): 2147483648, 587 }, 588 Annotations: map[string]string{ 589 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 590 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "true"}`, 591 }, 592 Labels: map[string]string{ 593 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 594 }, 595 }, 596 expectedResp: &pluginapi.ResourceAllocationResponse{ 597 PodNamespace: testName, 598 PodName: testName, 599 ContainerName: testName, 600 ContainerType: pluginapi.ContainerType_MAIN, 601 ContainerIndex: 0, 602 ResourceName: string(v1.ResourceMemory), 603 AllocationResult: &pluginapi.ResourceAllocation{ 604 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 605 string(v1.ResourceMemory): { 606 OciPropertyName: util.OCIPropertyNameCPUSetMems, 607 IsNodeResource: false, 608 IsScalarResource: true, 609 AllocatedQuantity: 7516192768, 610 AllocationResult: machine.NewCPUSet(0).String(), 611 ResourceHints: &pluginapi.ListOfTopologyHints{ 612 Hints: []*pluginapi.TopologyHint{ 613 { 614 Nodes: []uint64{0}, 615 Preferred: true, 616 }, 617 }, 618 }, 619 }, 620 }, 621 }, 622 Labels: map[string]string{ 623 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 624 }, 625 Annotations: map[string]string{ 626 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 627 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 628 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 629 }, 630 }, 631 }, 632 { 633 description: "req for dedicated_cores with numa_binding & not numa_exclusive main container", 634 req: &pluginapi.ResourceRequest{ 635 PodUid: string(uuid.NewUUID()), 636 PodNamespace: testName, 637 PodName: testName, 638 ContainerName: testName, 639 ContainerType: pluginapi.ContainerType_MAIN, 640 ContainerIndex: 0, 641 ResourceName: string(v1.ResourceMemory), 642 Hint: &pluginapi.TopologyHint{ 643 Nodes: []uint64{0}, 644 Preferred: true, 645 }, 646 ResourceRequests: map[string]float64{ 647 string(v1.ResourceMemory): 2147483648, 648 }, 649 Annotations: map[string]string{ 650 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 651 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, 652 }, 653 Labels: map[string]string{ 654 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 655 }, 656 }, 657 expectedResp: &pluginapi.ResourceAllocationResponse{ 658 PodNamespace: testName, 659 PodName: testName, 660 ContainerName: testName, 661 ContainerType: pluginapi.ContainerType_MAIN, 662 ContainerIndex: 0, 663 ResourceName: string(v1.ResourceMemory), 664 AllocationResult: &pluginapi.ResourceAllocation{ 665 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 666 string(v1.ResourceMemory): { 667 OciPropertyName: util.OCIPropertyNameCPUSetMems, 668 IsNodeResource: false, 669 IsScalarResource: true, 670 AllocatedQuantity: 2147483648, 671 AllocationResult: machine.NewCPUSet(0).String(), 672 ResourceHints: &pluginapi.ListOfTopologyHints{ 673 Hints: []*pluginapi.TopologyHint{ 674 { 675 Nodes: []uint64{0}, 676 Preferred: true, 677 }, 678 }, 679 }, 680 }, 681 }, 682 }, 683 Labels: map[string]string{ 684 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 685 }, 686 Annotations: map[string]string{ 687 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 688 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 689 consts.PodAnnotationMemoryEnhancementNumaExclusive: "false", 690 }, 691 }, 692 }, 693 { 694 description: "req for dedicated_cores with numa_binding & default numa_exclusive true main container", 695 req: &pluginapi.ResourceRequest{ 696 PodUid: string(uuid.NewUUID()), 697 PodNamespace: testName, 698 PodName: testName, 699 ContainerName: testName, 700 ContainerType: pluginapi.ContainerType_MAIN, 701 ContainerIndex: 0, 702 ResourceName: string(v1.ResourceMemory), 703 Hint: &pluginapi.TopologyHint{ 704 Nodes: []uint64{0}, 705 Preferred: true, 706 }, 707 ResourceRequests: map[string]float64{ 708 string(v1.ResourceMemory): 2147483648, 709 }, 710 Annotations: map[string]string{ 711 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 712 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, 713 }, 714 Labels: map[string]string{ 715 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 716 }, 717 }, 718 expectedResp: &pluginapi.ResourceAllocationResponse{ 719 PodNamespace: testName, 720 PodName: testName, 721 ContainerName: testName, 722 ContainerType: pluginapi.ContainerType_MAIN, 723 ContainerIndex: 0, 724 ResourceName: string(v1.ResourceMemory), 725 AllocationResult: &pluginapi.ResourceAllocation{ 726 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 727 string(v1.ResourceMemory): { 728 OciPropertyName: util.OCIPropertyNameCPUSetMems, 729 IsNodeResource: false, 730 IsScalarResource: true, 731 AllocatedQuantity: 7516192768, 732 AllocationResult: machine.NewCPUSet(0).String(), 733 ResourceHints: &pluginapi.ListOfTopologyHints{ 734 Hints: []*pluginapi.TopologyHint{ 735 { 736 Nodes: []uint64{0}, 737 Preferred: true, 738 }, 739 }, 740 }, 741 }, 742 }, 743 }, 744 Labels: map[string]string{ 745 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 746 }, 747 Annotations: map[string]string{ 748 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 749 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 750 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 751 }, 752 }, 753 enhancementDefaultValues: map[string]string{ 754 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 755 }, 756 }, 757 { 758 description: "req for dedicated_cores with numa_binding & without default numa_exclusive main container", 759 req: &pluginapi.ResourceRequest{ 760 PodUid: string(uuid.NewUUID()), 761 PodNamespace: testName, 762 PodName: testName, 763 ContainerName: testName, 764 ContainerType: pluginapi.ContainerType_MAIN, 765 ContainerIndex: 0, 766 ResourceName: string(v1.ResourceMemory), 767 Hint: &pluginapi.TopologyHint{ 768 Nodes: []uint64{0}, 769 Preferred: true, 770 }, 771 ResourceRequests: map[string]float64{ 772 string(v1.ResourceMemory): 2147483648, 773 }, 774 Annotations: map[string]string{ 775 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 776 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, 777 }, 778 Labels: map[string]string{ 779 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 780 }, 781 }, 782 expectedResp: &pluginapi.ResourceAllocationResponse{ 783 PodNamespace: testName, 784 PodName: testName, 785 ContainerName: testName, 786 ContainerType: pluginapi.ContainerType_MAIN, 787 ContainerIndex: 0, 788 ResourceName: string(v1.ResourceMemory), 789 AllocationResult: &pluginapi.ResourceAllocation{ 790 ResourceAllocation: map[string]*pluginapi.ResourceAllocationInfo{ 791 string(v1.ResourceMemory): { 792 OciPropertyName: util.OCIPropertyNameCPUSetMems, 793 IsNodeResource: false, 794 IsScalarResource: true, 795 AllocatedQuantity: 2147483648, 796 AllocationResult: machine.NewCPUSet(0).String(), 797 ResourceHints: &pluginapi.ListOfTopologyHints{ 798 Hints: []*pluginapi.TopologyHint{ 799 { 800 Nodes: []uint64{0}, 801 Preferred: true, 802 }, 803 }, 804 }, 805 }, 806 }, 807 }, 808 Labels: map[string]string{ 809 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 810 }, 811 Annotations: map[string]string{ 812 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 813 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 814 }, 815 }, 816 }, 817 } 818 819 for _, tc := range testCases { 820 tmpDir, err := ioutil.TempDir("", "checkpoint-TestAllocate") 821 as.Nil(err) 822 823 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 824 as.Nil(err) 825 826 if tc.enhancementDefaultValues != nil { 827 dynamicPolicy.qosConfig.QoSEnhancementDefaultValues = tc.enhancementDefaultValues 828 } 829 830 dynamicPolicy.enableMemoryAdvisor = true 831 dynamicPolicy.advisorClient = advisorsvc.NewStubAdvisorServiceClient() 832 833 resp, err := dynamicPolicy.Allocate(context.Background(), tc.req) 834 as.Nil(err) 835 836 tc.expectedResp.PodUid = tc.req.PodUid 837 as.Equalf(tc.expectedResp, resp, "failed in test case: %s", tc.description) 838 839 os.RemoveAll(tmpDir) 840 } 841 } 842 843 func TestGetTopologyHints(t *testing.T) { 844 t.Parallel() 845 846 as := require.New(t) 847 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 848 as.Nil(err) 849 850 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 851 as.Nil(err) 852 853 testName := "test" 854 855 testCases := []struct { 856 description string 857 req *pluginapi.ResourceRequest 858 expectedResp *pluginapi.ResourceHintsResponse 859 enhancementDefaultValues map[string]string 860 }{ 861 { 862 description: "req for container of debug pod", 863 req: &pluginapi.ResourceRequest{ 864 PodUid: string(uuid.NewUUID()), 865 PodNamespace: testName, 866 PodName: testName, 867 ContainerName: testName, 868 ContainerType: pluginapi.ContainerType_MAIN, 869 ContainerIndex: 0, 870 ResourceName: string(v1.ResourceMemory), 871 ResourceRequests: map[string]float64{ 872 string(v1.ResourceMemory): 1073741824, 873 }, 874 Annotations: map[string]string{ 875 podDebugAnnoKey: "", 876 }, 877 }, 878 expectedResp: &pluginapi.ResourceHintsResponse{ 879 PodNamespace: testName, 880 PodName: testName, 881 ContainerName: testName, 882 ContainerType: pluginapi.ContainerType_MAIN, 883 ContainerIndex: 0, 884 ResourceName: string(v1.ResourceMemory), 885 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 886 string(v1.ResourceMemory): nil, 887 }, 888 Labels: map[string]string{ 889 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 890 }, 891 Annotations: map[string]string{ 892 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 893 }, 894 }, 895 }, 896 { 897 description: "req for shared_cores main container", 898 req: &pluginapi.ResourceRequest{ 899 PodUid: string(uuid.NewUUID()), 900 PodNamespace: testName, 901 PodName: testName, 902 ContainerName: testName, 903 ContainerType: pluginapi.ContainerType_MAIN, 904 ContainerIndex: 0, 905 ResourceName: string(v1.ResourceMemory), 906 ResourceRequests: map[string]float64{ 907 string(v1.ResourceMemory): 1073741824, 908 }, 909 }, 910 expectedResp: &pluginapi.ResourceHintsResponse{ 911 PodNamespace: testName, 912 PodName: testName, 913 ContainerName: testName, 914 ContainerType: pluginapi.ContainerType_MAIN, 915 ContainerIndex: 0, 916 ResourceName: string(v1.ResourceMemory), 917 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 918 string(v1.ResourceMemory): nil, 919 }, 920 Labels: map[string]string{ 921 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 922 }, 923 Annotations: map[string]string{ 924 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 925 }, 926 }, 927 }, 928 { 929 description: "req for reclaimed_cores main container", 930 req: &pluginapi.ResourceRequest{ 931 PodUid: string(uuid.NewUUID()), 932 PodNamespace: testName, 933 PodName: testName, 934 ContainerName: testName, 935 ContainerType: pluginapi.ContainerType_MAIN, 936 ContainerIndex: 0, 937 ResourceName: string(v1.ResourceMemory), 938 ResourceRequests: map[string]float64{ 939 string(v1.ResourceMemory): 1073741824, 940 }, 941 Labels: map[string]string{ 942 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 943 }, 944 Annotations: map[string]string{ 945 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 946 }, 947 }, 948 expectedResp: &pluginapi.ResourceHintsResponse{ 949 PodNamespace: testName, 950 PodName: testName, 951 ContainerName: testName, 952 ContainerType: pluginapi.ContainerType_MAIN, 953 ContainerIndex: 0, 954 ResourceName: string(v1.ResourceMemory), 955 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 956 string(v1.ResourceMemory): nil, 957 }, 958 Labels: map[string]string{ 959 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 960 }, 961 Annotations: map[string]string{ 962 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 963 }, 964 }, 965 }, 966 { 967 description: "req for dedicated_cores with numa_binding & numa_exclusive main container", 968 req: &pluginapi.ResourceRequest{ 969 PodUid: string(uuid.NewUUID()), 970 PodNamespace: testName, 971 PodName: testName, 972 ContainerName: testName, 973 ContainerType: pluginapi.ContainerType_MAIN, 974 ContainerIndex: 0, 975 ResourceName: string(v1.ResourceMemory), 976 ResourceRequests: map[string]float64{ 977 string(v1.ResourceMemory): 10737418240, 978 }, 979 Annotations: map[string]string{ 980 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 981 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "true"}`, 982 }, 983 Labels: map[string]string{ 984 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 985 }, 986 }, 987 expectedResp: &pluginapi.ResourceHintsResponse{ 988 PodNamespace: testName, 989 PodName: testName, 990 ContainerName: testName, 991 ContainerType: pluginapi.ContainerType_MAIN, 992 ContainerIndex: 0, 993 ResourceName: string(v1.ResourceMemory), 994 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 995 string(v1.ResourceMemory): { 996 Hints: []*pluginapi.TopologyHint{ 997 { 998 Nodes: []uint64{0, 1}, 999 Preferred: true, 1000 }, 1001 { 1002 Nodes: []uint64{2, 3}, 1003 Preferred: true, 1004 }, 1005 { 1006 Nodes: []uint64{0, 1, 2}, 1007 Preferred: false, 1008 }, 1009 { 1010 Nodes: []uint64{0, 1, 3}, 1011 Preferred: false, 1012 }, 1013 { 1014 Nodes: []uint64{0, 2, 3}, 1015 Preferred: false, 1016 }, 1017 { 1018 Nodes: []uint64{1, 2, 3}, 1019 Preferred: false, 1020 }, 1021 { 1022 Nodes: []uint64{0, 1, 2, 3}, 1023 Preferred: false, 1024 }, 1025 }, 1026 }, 1027 }, 1028 Labels: map[string]string{ 1029 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1030 }, 1031 Annotations: map[string]string{ 1032 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1033 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1034 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1035 }, 1036 }, 1037 }, 1038 { 1039 description: "req for dedicated_cores with numa_binding & not numa_exclusive main container", 1040 req: &pluginapi.ResourceRequest{ 1041 PodUid: string(uuid.NewUUID()), 1042 PodNamespace: testName, 1043 PodName: testName, 1044 ContainerName: testName, 1045 ContainerType: pluginapi.ContainerType_MAIN, 1046 ContainerIndex: 0, 1047 ResourceName: string(v1.ResourceMemory), 1048 ResourceRequests: map[string]float64{ 1049 string(v1.ResourceMemory): 1073741824, 1050 }, 1051 Annotations: map[string]string{ 1052 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1053 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "false"}`, 1054 }, 1055 Labels: map[string]string{ 1056 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1057 }, 1058 }, 1059 expectedResp: &pluginapi.ResourceHintsResponse{ 1060 PodNamespace: testName, 1061 PodName: testName, 1062 ContainerName: testName, 1063 ContainerType: pluginapi.ContainerType_MAIN, 1064 ContainerIndex: 0, 1065 ResourceName: string(v1.ResourceMemory), 1066 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 1067 string(v1.ResourceMemory): { 1068 Hints: []*pluginapi.TopologyHint{ 1069 { 1070 Nodes: []uint64{0}, 1071 Preferred: true, 1072 }, 1073 { 1074 Nodes: []uint64{1}, 1075 Preferred: true, 1076 }, 1077 { 1078 Nodes: []uint64{2}, 1079 Preferred: true, 1080 }, 1081 { 1082 Nodes: []uint64{3}, 1083 Preferred: true, 1084 }, 1085 }, 1086 }, 1087 }, 1088 Labels: map[string]string{ 1089 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1090 }, 1091 Annotations: map[string]string{ 1092 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1093 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1094 consts.PodAnnotationMemoryEnhancementNumaExclusive: "false", 1095 }, 1096 }, 1097 }, 1098 { 1099 description: "req for dedicated_cores with numa_binding & default numa_exclusive true main container", 1100 req: &pluginapi.ResourceRequest{ 1101 PodUid: string(uuid.NewUUID()), 1102 PodNamespace: testName, 1103 PodName: testName, 1104 ContainerName: testName, 1105 ContainerType: pluginapi.ContainerType_MAIN, 1106 ContainerIndex: 0, 1107 ResourceName: string(v1.ResourceMemory), 1108 ResourceRequests: map[string]float64{ 1109 string(v1.ResourceMemory): 10737418240, 1110 }, 1111 Annotations: map[string]string{ 1112 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1113 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, 1114 }, 1115 Labels: map[string]string{ 1116 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1117 }, 1118 }, 1119 expectedResp: &pluginapi.ResourceHintsResponse{ 1120 PodNamespace: testName, 1121 PodName: testName, 1122 ContainerName: testName, 1123 ContainerType: pluginapi.ContainerType_MAIN, 1124 ContainerIndex: 0, 1125 ResourceName: string(v1.ResourceMemory), 1126 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 1127 string(v1.ResourceMemory): { 1128 Hints: []*pluginapi.TopologyHint{ 1129 { 1130 Nodes: []uint64{0, 1}, 1131 Preferred: true, 1132 }, 1133 { 1134 Nodes: []uint64{2, 3}, 1135 Preferred: true, 1136 }, 1137 { 1138 Nodes: []uint64{0, 1, 2}, 1139 Preferred: false, 1140 }, 1141 { 1142 Nodes: []uint64{0, 1, 3}, 1143 Preferred: false, 1144 }, 1145 { 1146 Nodes: []uint64{0, 2, 3}, 1147 Preferred: false, 1148 }, 1149 { 1150 Nodes: []uint64{1, 2, 3}, 1151 Preferred: false, 1152 }, 1153 { 1154 Nodes: []uint64{0, 1, 2, 3}, 1155 Preferred: false, 1156 }, 1157 }, 1158 }, 1159 }, 1160 Labels: map[string]string{ 1161 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1162 }, 1163 Annotations: map[string]string{ 1164 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1165 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1166 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1167 }, 1168 }, 1169 enhancementDefaultValues: map[string]string{ 1170 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1171 }, 1172 }, 1173 { 1174 description: "req for dedicated_cores with numa_binding & without numa_exclusive main container", 1175 req: &pluginapi.ResourceRequest{ 1176 PodUid: string(uuid.NewUUID()), 1177 PodNamespace: testName, 1178 PodName: testName, 1179 ContainerName: testName, 1180 ContainerType: pluginapi.ContainerType_MAIN, 1181 ContainerIndex: 0, 1182 ResourceName: string(v1.ResourceMemory), 1183 ResourceRequests: map[string]float64{ 1184 string(v1.ResourceMemory): 1073741824, 1185 }, 1186 Annotations: map[string]string{ 1187 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1188 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true"}`, 1189 }, 1190 Labels: map[string]string{ 1191 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1192 }, 1193 }, 1194 expectedResp: &pluginapi.ResourceHintsResponse{ 1195 PodNamespace: testName, 1196 PodName: testName, 1197 ContainerName: testName, 1198 ContainerType: pluginapi.ContainerType_MAIN, 1199 ContainerIndex: 0, 1200 ResourceName: string(v1.ResourceMemory), 1201 ResourceHints: map[string]*pluginapi.ListOfTopologyHints{ 1202 string(v1.ResourceMemory): { 1203 Hints: []*pluginapi.TopologyHint{ 1204 { 1205 Nodes: []uint64{0}, 1206 Preferred: true, 1207 }, 1208 { 1209 Nodes: []uint64{1}, 1210 Preferred: true, 1211 }, 1212 { 1213 Nodes: []uint64{2}, 1214 Preferred: true, 1215 }, 1216 { 1217 Nodes: []uint64{3}, 1218 Preferred: true, 1219 }, 1220 }, 1221 }, 1222 }, 1223 Labels: map[string]string{ 1224 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1225 }, 1226 Annotations: map[string]string{ 1227 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1228 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1229 }, 1230 }, 1231 }, 1232 } 1233 1234 for _, tc := range testCases { 1235 tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetTopologyHints") 1236 as.Nil(err) 1237 1238 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 1239 as.Nil(err) 1240 1241 if tc.enhancementDefaultValues != nil { 1242 dynamicPolicy.qosConfig.QoSEnhancementDefaultValues = tc.enhancementDefaultValues 1243 } 1244 1245 resp, err := dynamicPolicy.GetTopologyHints(context.Background(), tc.req) 1246 as.Nil(err) 1247 1248 tc.expectedResp.PodUid = tc.req.PodUid 1249 as.Equalf(tc.expectedResp, resp, "failed in test case: %s", tc.description) 1250 1251 os.RemoveAll(tmpDir) 1252 } 1253 } 1254 1255 func TestGetTopologyAwareAllocatableResources(t *testing.T) { 1256 t.Parallel() 1257 1258 as := require.New(t) 1259 1260 tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetTopologyAwareAllocatableResources") 1261 as.Nil(err) 1262 defer os.RemoveAll(tmpDir) 1263 1264 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 1265 as.Nil(err) 1266 1267 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 1268 as.Nil(err) 1269 1270 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 1271 as.Nil(err) 1272 1273 resp, err := dynamicPolicy.GetTopologyAwareAllocatableResources(context.Background(), &pluginapi.GetTopologyAwareAllocatableResourcesRequest{}) 1274 as.Nil(err) 1275 1276 as.Equal(&pluginapi.GetTopologyAwareAllocatableResourcesResponse{ 1277 AllocatableResources: map[string]*pluginapi.AllocatableTopologyAwareResource{ 1278 string(v1.ResourceMemory): { 1279 IsNodeResource: false, 1280 IsScalarResource: true, 1281 TopologyAwareAllocatableQuantityList: []*pluginapi.TopologyAwareQuantity{ 1282 {ResourceValue: 7516192768, Node: 0}, 1283 {ResourceValue: 7516192768, Node: 1}, 1284 {ResourceValue: 7516192768, Node: 2}, 1285 {ResourceValue: 7516192768, Node: 3}, 1286 }, 1287 TopologyAwareCapacityQuantityList: []*pluginapi.TopologyAwareQuantity{ 1288 {ResourceValue: 8589934592, Node: 0}, 1289 {ResourceValue: 8589934592, Node: 1}, 1290 {ResourceValue: 8589934592, Node: 2}, 1291 {ResourceValue: 8589934592, Node: 3}, 1292 }, 1293 AggregatedAllocatableQuantity: 30064771072, 1294 AggregatedCapacityQuantity: 34359738368, 1295 }, 1296 }, 1297 }, resp) 1298 } 1299 1300 func TestGetTopologyAwareResources(t *testing.T) { 1301 t.Parallel() 1302 1303 as := require.New(t) 1304 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 1305 as.Nil(err) 1306 1307 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 1308 as.Nil(err) 1309 1310 testName := "test" 1311 1312 testCases := []struct { 1313 description string 1314 req *pluginapi.ResourceRequest 1315 expectedResp *pluginapi.GetTopologyAwareResourcesResponse 1316 err error 1317 }{ 1318 { 1319 description: "req for init container", 1320 req: &pluginapi.ResourceRequest{ 1321 PodUid: string(uuid.NewUUID()), 1322 PodNamespace: testName, 1323 PodName: testName, 1324 ContainerName: testName, 1325 ContainerType: pluginapi.ContainerType_INIT, 1326 ContainerIndex: 0, 1327 ResourceName: string(v1.ResourceMemory), 1328 ResourceRequests: map[string]float64{ 1329 string(v1.ResourceMemory): 2, 1330 }, 1331 }, 1332 err: fmt.Errorf("error occurred"), 1333 }, 1334 { 1335 description: "req for shared_cores main container", 1336 req: &pluginapi.ResourceRequest{ 1337 PodUid: string(uuid.NewUUID()), 1338 PodNamespace: testName, 1339 PodName: testName, 1340 ContainerName: testName, 1341 ContainerType: pluginapi.ContainerType_MAIN, 1342 ContainerIndex: 0, 1343 ResourceName: string(v1.ResourceMemory), 1344 ResourceRequests: map[string]float64{ 1345 string(v1.ResourceMemory): 1073741824, 1346 }, 1347 Labels: map[string]string{ 1348 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1349 }, 1350 Annotations: map[string]string{ 1351 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1352 }, 1353 }, 1354 expectedResp: &pluginapi.GetTopologyAwareResourcesResponse{ 1355 PodNamespace: testName, 1356 PodName: testName, 1357 ContainerTopologyAwareResources: &pluginapi.ContainerTopologyAwareResources{ 1358 ContainerName: testName, 1359 AllocatedResources: map[string]*pluginapi.TopologyAwareResource{ 1360 string(v1.ResourceMemory): { 1361 IsNodeResource: false, 1362 IsScalarResource: true, 1363 AggregatedQuantity: 0, 1364 OriginalAggregatedQuantity: 0, 1365 TopologyAwareQuantityList: nil, 1366 OriginalTopologyAwareQuantityList: nil, 1367 }, 1368 }, 1369 }, 1370 }, 1371 }, 1372 { 1373 description: "req for reclaimed_cores main container", 1374 req: &pluginapi.ResourceRequest{ 1375 PodUid: string(uuid.NewUUID()), 1376 PodNamespace: testName, 1377 PodName: testName, 1378 ContainerName: testName, 1379 ContainerType: pluginapi.ContainerType_MAIN, 1380 ContainerIndex: 0, 1381 ResourceName: string(v1.ResourceMemory), 1382 ResourceRequests: map[string]float64{ 1383 string(v1.ResourceMemory): 1073741824, 1384 }, 1385 Labels: map[string]string{ 1386 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1387 }, 1388 Annotations: map[string]string{ 1389 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1390 }, 1391 }, 1392 expectedResp: &pluginapi.GetTopologyAwareResourcesResponse{ 1393 PodNamespace: testName, 1394 PodName: testName, 1395 ContainerTopologyAwareResources: &pluginapi.ContainerTopologyAwareResources{ 1396 ContainerName: testName, 1397 AllocatedResources: map[string]*pluginapi.TopologyAwareResource{ 1398 string(v1.ResourceMemory): { 1399 IsNodeResource: false, 1400 IsScalarResource: true, 1401 AggregatedQuantity: 0, 1402 OriginalAggregatedQuantity: 0, 1403 TopologyAwareQuantityList: nil, 1404 OriginalTopologyAwareQuantityList: nil, 1405 }, 1406 }, 1407 }, 1408 }, 1409 }, 1410 { 1411 description: "req for dedicated_cores with numa_binding main container", 1412 req: &pluginapi.ResourceRequest{ 1413 PodUid: string(uuid.NewUUID()), 1414 PodNamespace: testName, 1415 PodName: testName, 1416 ContainerName: testName, 1417 ContainerType: pluginapi.ContainerType_MAIN, 1418 ContainerIndex: 0, 1419 ResourceName: string(v1.ResourceMemory), 1420 Hint: &pluginapi.TopologyHint{ 1421 Nodes: []uint64{0, 1}, 1422 Preferred: true, 1423 }, 1424 ResourceRequests: map[string]float64{ 1425 string(v1.ResourceMemory): 10737418240, 1426 }, 1427 Annotations: map[string]string{ 1428 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1429 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "true"}`, 1430 }, 1431 Labels: map[string]string{ 1432 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1433 }, 1434 }, 1435 expectedResp: &pluginapi.GetTopologyAwareResourcesResponse{ 1436 PodNamespace: testName, 1437 PodName: testName, 1438 ContainerTopologyAwareResources: &pluginapi.ContainerTopologyAwareResources{ 1439 ContainerName: testName, 1440 AllocatedResources: map[string]*pluginapi.TopologyAwareResource{ 1441 string(v1.ResourceMemory): { 1442 IsNodeResource: false, 1443 IsScalarResource: true, 1444 AggregatedQuantity: 15032385536, 1445 OriginalAggregatedQuantity: 15032385536, 1446 TopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 1447 {ResourceValue: 7516192768, Node: 0}, 1448 {ResourceValue: 7516192768, Node: 1}, 1449 }, 1450 OriginalTopologyAwareQuantityList: []*pluginapi.TopologyAwareQuantity{ 1451 {ResourceValue: 7516192768, Node: 0}, 1452 {ResourceValue: 7516192768, Node: 1}, 1453 }, 1454 }, 1455 }, 1456 }, 1457 }, 1458 }, 1459 } 1460 1461 for _, tc := range testCases { 1462 tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetTopologyAwareResources") 1463 as.Nil(err) 1464 1465 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 1466 as.Nil(err) 1467 1468 _, err = dynamicPolicy.Allocate(context.Background(), tc.req) 1469 as.Nil(err) 1470 1471 resp, err := dynamicPolicy.GetTopologyAwareResources(context.Background(), &pluginapi.GetTopologyAwareResourcesRequest{ 1472 PodUid: tc.req.PodUid, 1473 ContainerName: testName, 1474 }) 1475 1476 if tc.err != nil { 1477 as.NotNil(err) 1478 continue 1479 } else { 1480 as.Nil(err) 1481 tc.expectedResp.PodUid = tc.req.PodUid 1482 } 1483 1484 as.Equalf(tc.expectedResp, resp, "failed in test case: %s", tc.description) 1485 1486 os.Remove(tmpDir) 1487 } 1488 } 1489 1490 func TestGetResourcesAllocation(t *testing.T) { 1491 t.Parallel() 1492 1493 as := require.New(t) 1494 1495 tmpDir, err := ioutil.TempDir("", "checkpoint-TestGetResourcesAllocation") 1496 as.Nil(err) 1497 defer os.RemoveAll(tmpDir) 1498 1499 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 1500 as.Nil(err) 1501 1502 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 1503 as.Nil(err) 1504 1505 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 1506 as.Nil(err) 1507 1508 testName := "test" 1509 1510 // test for shared_cores 1511 req := &pluginapi.ResourceRequest{ 1512 PodUid: string(uuid.NewUUID()), 1513 PodNamespace: testName, 1514 PodName: testName, 1515 ContainerName: testName, 1516 ContainerType: pluginapi.ContainerType_MAIN, 1517 ContainerIndex: 0, 1518 ResourceName: string(v1.ResourceMemory), 1519 ResourceRequests: map[string]float64{ 1520 string(v1.ResourceMemory): 1073741824, 1521 }, 1522 Labels: map[string]string{ 1523 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1524 }, 1525 Annotations: map[string]string{ 1526 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1527 }, 1528 } 1529 1530 _, err = dynamicPolicy.Allocate(context.Background(), req) 1531 as.Nil(err) 1532 1533 resp1, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) 1534 as.Nil(err) 1535 1536 as.NotNil(resp1.PodResources[req.PodUid]) 1537 as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName]) 1538 as.NotNil(resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceMemory)]) 1539 as.Equal(resp1.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceMemory)], &pluginapi.ResourceAllocationInfo{ 1540 OciPropertyName: util.OCIPropertyNameCPUSetMems, 1541 IsNodeResource: false, 1542 IsScalarResource: true, 1543 AllocatedQuantity: 0, 1544 AllocationResult: machine.NewCPUSet(0, 1, 2, 3).String(), 1545 }) 1546 1547 // test for reclaimed_cores 1548 req = &pluginapi.ResourceRequest{ 1549 PodUid: string(uuid.NewUUID()), 1550 PodNamespace: testName, 1551 PodName: testName, 1552 ContainerName: testName, 1553 ContainerType: pluginapi.ContainerType_MAIN, 1554 ContainerIndex: 0, 1555 ResourceName: string(v1.ResourceMemory), 1556 ResourceRequests: map[string]float64{ 1557 string(v1.ResourceMemory): 1073741824, 1558 }, 1559 Labels: map[string]string{ 1560 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1561 }, 1562 Annotations: map[string]string{ 1563 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1564 }, 1565 } 1566 1567 _, err = dynamicPolicy.Allocate(context.Background(), req) 1568 as.Nil(err) 1569 1570 resp2, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) 1571 as.Nil(err) 1572 1573 as.NotNil(resp2.PodResources[req.PodUid]) 1574 as.NotNil(resp2.PodResources[req.PodUid].ContainerResources[testName]) 1575 as.NotNil(resp2.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceMemory)]) 1576 as.Equal(resp2.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceMemory)], &pluginapi.ResourceAllocationInfo{ 1577 OciPropertyName: util.OCIPropertyNameCPUSetMems, 1578 IsNodeResource: false, 1579 IsScalarResource: true, 1580 AllocatedQuantity: 0, 1581 AllocationResult: machine.NewCPUSet(0, 1, 2, 3).String(), 1582 }) 1583 1584 os.RemoveAll(tmpDir) 1585 dynamicPolicy, err = getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 1586 as.Nil(err) 1587 1588 // test for dedicated_cores with numa_binding 1589 req = &pluginapi.ResourceRequest{ 1590 PodUid: string(uuid.NewUUID()), 1591 PodNamespace: testName, 1592 PodName: testName, 1593 ContainerName: testName, 1594 ContainerType: pluginapi.ContainerType_MAIN, 1595 ContainerIndex: 0, 1596 ResourceName: string(v1.ResourceMemory), 1597 Hint: &pluginapi.TopologyHint{ 1598 Nodes: []uint64{0}, 1599 Preferred: true, 1600 }, 1601 ResourceRequests: map[string]float64{ 1602 string(v1.ResourceMemory): 2147483648, 1603 }, 1604 Annotations: map[string]string{ 1605 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1606 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding": "true", "numa_exclusive": "true"}`, 1607 }, 1608 Labels: map[string]string{ 1609 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1610 }, 1611 } 1612 1613 _, err = dynamicPolicy.Allocate(context.Background(), req) 1614 as.Nil(err) 1615 1616 resp3, err := dynamicPolicy.GetResourcesAllocation(context.Background(), &pluginapi.GetResourcesAllocationRequest{}) 1617 as.Nil(err) 1618 1619 as.NotNil(resp3.PodResources[req.PodUid]) 1620 as.NotNil(resp3.PodResources[req.PodUid].ContainerResources[testName]) 1621 as.NotNil(resp3.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceMemory)]) 1622 as.Equal(resp3.PodResources[req.PodUid].ContainerResources[testName].ResourceAllocation[string(v1.ResourceMemory)], &pluginapi.ResourceAllocationInfo{ 1623 OciPropertyName: util.OCIPropertyNameCPUSetMems, 1624 IsNodeResource: false, 1625 IsScalarResource: true, 1626 AllocatedQuantity: 7516192768, 1627 AllocationResult: machine.NewCPUSet(0).String(), 1628 }) 1629 } 1630 1631 func TestGetReadonlyState(t *testing.T) { 1632 t.Parallel() 1633 1634 as := require.New(t) 1635 readonlyState, err := GetReadonlyState() 1636 if readonlyState == nil { 1637 as.NotNil(err) 1638 } 1639 } 1640 1641 func TestGenerateResourcesMachineStateFromPodEntries(t *testing.T) { 1642 t.Parallel() 1643 1644 as := require.New(t) 1645 1646 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 1647 as.Nil(err) 1648 1649 reservedMemory, err := getReservedMemory(fakeConf, &metaserver.MetaServer{}, machineInfo) 1650 as.Nil(err) 1651 1652 podUID := string(uuid.NewUUID()) 1653 testName := "test" 1654 1655 podEntries := state.PodEntries{ 1656 podUID: state.ContainerEntries{ 1657 testName: &state.AllocationInfo{ 1658 PodUid: podUID, 1659 PodNamespace: testName, 1660 PodName: testName, 1661 ContainerName: testName, 1662 ContainerType: pluginapi.ContainerType_MAIN.String(), 1663 ContainerIndex: 0, 1664 RampUp: false, 1665 AggregatedQuantity: 9663676416, 1666 NumaAllocationResult: machine.NewCPUSet(0), 1667 TopologyAwareAllocations: map[int]uint64{ 1668 0: 9663676416, 1669 }, 1670 }, 1671 }, 1672 } 1673 1674 podResourceEntries := state.PodResourceEntries{ 1675 v1.ResourceMemory: podEntries, 1676 } 1677 1678 reserved := map[v1.ResourceName]map[int]uint64{ 1679 v1.ResourceMemory: reservedMemory, 1680 } 1681 1682 resourcesMachineState, err := state.GenerateMachineStateFromPodEntries(machineInfo, podResourceEntries, reserved) 1683 as.Nil(err) 1684 1685 as.NotNil(resourcesMachineState[v1.ResourceMemory][0]) 1686 as.Equal(uint64(9663676416), resourcesMachineState[v1.ResourceMemory][0].Allocatable) 1687 } 1688 1689 func TestHandleAdvisorResp(t *testing.T) { 1690 t.Parallel() 1691 1692 as := require.New(t) 1693 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 1694 as.Nil(err) 1695 1696 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 1697 as.Nil(err) 1698 1699 reservedMemory, err := getReservedMemory(fakeConf, &metaserver.MetaServer{}, machineInfo) 1700 as.Nil(err) 1701 1702 resourcesReservedMemory := map[v1.ResourceName]map[int]uint64{ 1703 v1.ResourceMemory: reservedMemory, 1704 } 1705 1706 pod1UID := string(uuid.NewUUID()) 1707 pod2UID := string(uuid.NewUUID()) 1708 pod3UID := string(uuid.NewUUID()) 1709 pod4UID := string(uuid.NewUUID()) 1710 testName := "test" 1711 1712 testCases := []struct { 1713 description string 1714 podResourceEntries state.PodResourceEntries 1715 expectedPodResourceEntries state.PodResourceEntries 1716 expectedMachineState state.NUMANodeResourcesMap 1717 lwResp *advisorsvc.ListAndWatchResponse 1718 }{ 1719 { 1720 description: "one shared_cores container, two reclaimed_cores container, one dedicated_cores container", 1721 podResourceEntries: state.PodResourceEntries{ 1722 v1.ResourceMemory: state.PodEntries{ 1723 pod1UID: state.ContainerEntries{ 1724 testName: &state.AllocationInfo{ 1725 PodUid: pod1UID, 1726 PodNamespace: testName, 1727 PodName: testName, 1728 ContainerName: testName, 1729 ContainerType: pluginapi.ContainerType_MAIN.String(), 1730 ContainerIndex: 0, 1731 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 1732 RampUp: false, 1733 AggregatedQuantity: 7516192768, 1734 NumaAllocationResult: machine.NewCPUSet(0), 1735 TopologyAwareAllocations: map[int]uint64{ 1736 0: 7516192768, 1737 }, 1738 Annotations: map[string]string{ 1739 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1740 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 1741 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1742 }, 1743 Labels: map[string]string{ 1744 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1745 }, 1746 }, 1747 }, 1748 pod2UID: state.ContainerEntries{ 1749 testName: &state.AllocationInfo{ 1750 PodUid: pod2UID, 1751 PodNamespace: testName, 1752 PodName: testName, 1753 ContainerName: testName, 1754 ContainerType: pluginapi.ContainerType_MAIN.String(), 1755 ContainerIndex: 0, 1756 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1757 RampUp: false, 1758 NumaAllocationResult: machine.NewCPUSet(1, 2, 3), 1759 Annotations: map[string]string{ 1760 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1761 }, 1762 Labels: map[string]string{ 1763 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1764 }, 1765 }, 1766 }, 1767 pod3UID: state.ContainerEntries{ 1768 testName: &state.AllocationInfo{ 1769 PodUid: pod3UID, 1770 PodNamespace: testName, 1771 PodName: testName, 1772 ContainerName: testName, 1773 ContainerType: pluginapi.ContainerType_MAIN.String(), 1774 ContainerIndex: 0, 1775 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 1776 RampUp: false, 1777 NumaAllocationResult: machine.NewCPUSet(0, 1, 2, 3), 1778 Annotations: map[string]string{ 1779 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1780 }, 1781 Labels: map[string]string{ 1782 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1783 }, 1784 }, 1785 }, 1786 }, 1787 }, 1788 lwResp: &advisorsvc.ListAndWatchResponse{ 1789 PodEntries: map[string]*advisorsvc.CalculationEntries{ 1790 pod1UID: { 1791 ContainerEntries: map[string]*advisorsvc.CalculationInfo{ 1792 testName: { 1793 CalculationResult: &advisorsvc.CalculationResult{ 1794 Values: map[string]string{ 1795 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): "5516192768", 1796 }, 1797 }, 1798 }, 1799 }, 1800 }, 1801 pod2UID: { 1802 ContainerEntries: map[string]*advisorsvc.CalculationInfo{ 1803 testName: { 1804 CalculationResult: &advisorsvc.CalculationResult{ 1805 Values: map[string]string{ 1806 string(memoryadvisor.ControlKnobKeyDropCache): "true", 1807 }, 1808 }, 1809 }, 1810 }, 1811 }, 1812 pod3UID: { 1813 ContainerEntries: map[string]*advisorsvc.CalculationInfo{ 1814 testName: { 1815 CalculationResult: &advisorsvc.CalculationResult{ 1816 Values: map[string]string{ 1817 string(memoryadvisor.ControlKnobKeyCPUSetMems): "2-3", 1818 }, 1819 }, 1820 }, 1821 }, 1822 }, 1823 pod4UID: { 1824 ContainerEntries: map[string]*advisorsvc.CalculationInfo{ 1825 testName: { 1826 CalculationResult: &advisorsvc.CalculationResult{ 1827 Values: map[string]string{ 1828 string(memoryadvisor.ControlKnobKeySwapMax): coreconsts.ControlKnobON, 1829 string(memoryadvisor.ControlKnowKeyMemoryOffloading): "40960", 1830 }, 1831 }, 1832 }, 1833 }, 1834 }, 1835 }, 1836 }, 1837 expectedPodResourceEntries: state.PodResourceEntries{ 1838 v1.ResourceMemory: state.PodEntries{ 1839 pod1UID: state.ContainerEntries{ 1840 testName: &state.AllocationInfo{ 1841 PodUid: pod1UID, 1842 PodNamespace: testName, 1843 PodName: testName, 1844 ContainerName: testName, 1845 ContainerType: pluginapi.ContainerType_MAIN.String(), 1846 ContainerIndex: 0, 1847 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 1848 RampUp: false, 1849 AggregatedQuantity: 7516192768, 1850 NumaAllocationResult: machine.NewCPUSet(0), 1851 TopologyAwareAllocations: map[int]uint64{ 1852 0: 7516192768, 1853 }, 1854 Annotations: map[string]string{ 1855 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1856 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 1857 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1858 }, 1859 Labels: map[string]string{ 1860 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1861 }, 1862 ExtraControlKnobInfo: map[string]commonstate.ControlKnobInfo{ 1863 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 1864 ControlKnobValue: "5516192768", 1865 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 1866 }, 1867 }, 1868 }, 1869 }, 1870 pod2UID: state.ContainerEntries{ 1871 testName: &state.AllocationInfo{ 1872 PodUid: pod2UID, 1873 PodNamespace: testName, 1874 PodName: testName, 1875 ContainerName: testName, 1876 ContainerType: pluginapi.ContainerType_MAIN.String(), 1877 ContainerIndex: 0, 1878 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1879 RampUp: false, 1880 NumaAllocationResult: machine.NewCPUSet(1, 2, 3), 1881 Annotations: map[string]string{ 1882 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1883 }, 1884 Labels: map[string]string{ 1885 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1886 }, 1887 ExtraControlKnobInfo: make(map[string]commonstate.ControlKnobInfo), 1888 }, 1889 }, 1890 pod3UID: state.ContainerEntries{ 1891 testName: &state.AllocationInfo{ 1892 PodUid: pod3UID, 1893 PodNamespace: testName, 1894 PodName: testName, 1895 ContainerName: testName, 1896 ContainerType: pluginapi.ContainerType_MAIN.String(), 1897 ContainerIndex: 0, 1898 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 1899 RampUp: false, 1900 NumaAllocationResult: machine.NewCPUSet(2, 3), 1901 Annotations: map[string]string{ 1902 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1903 }, 1904 Labels: map[string]string{ 1905 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 1906 }, 1907 ExtraControlKnobInfo: make(map[string]commonstate.ControlKnobInfo), 1908 }, 1909 }, 1910 }, 1911 }, 1912 expectedMachineState: state.NUMANodeResourcesMap{ 1913 v1.ResourceMemory: { 1914 0: &state.NUMANodeState{ 1915 TotalMemSize: 8589934592, 1916 SystemReserved: 1073741824, 1917 Allocatable: 7516192768, 1918 Allocated: 7516192768, 1919 Free: 0, 1920 PodEntries: state.PodEntries{ 1921 pod1UID: state.ContainerEntries{ 1922 testName: &state.AllocationInfo{ 1923 PodUid: pod1UID, 1924 PodNamespace: testName, 1925 PodName: testName, 1926 ContainerName: testName, 1927 ContainerType: pluginapi.ContainerType_MAIN.String(), 1928 ContainerIndex: 0, 1929 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 1930 RampUp: false, 1931 AggregatedQuantity: 7516192768, 1932 NumaAllocationResult: machine.NewCPUSet(0), 1933 TopologyAwareAllocations: map[int]uint64{ 1934 0: 7516192768, 1935 }, 1936 Annotations: map[string]string{ 1937 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1938 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 1939 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 1940 }, 1941 Labels: map[string]string{ 1942 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 1943 }, 1944 ExtraControlKnobInfo: map[string]commonstate.ControlKnobInfo{ 1945 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 1946 ControlKnobValue: "5516192768", 1947 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 1948 }, 1949 }, 1950 }, 1951 }, 1952 }, 1953 }, 1954 1: &state.NUMANodeState{ 1955 TotalMemSize: 8589934592, 1956 SystemReserved: 1073741824, 1957 Allocatable: 7516192768, 1958 Allocated: 0, 1959 Free: 7516192768, 1960 PodEntries: state.PodEntries{ 1961 pod2UID: state.ContainerEntries{ 1962 testName: &state.AllocationInfo{ 1963 PodUid: pod2UID, 1964 PodNamespace: testName, 1965 PodName: testName, 1966 ContainerName: testName, 1967 ContainerType: pluginapi.ContainerType_MAIN.String(), 1968 ContainerIndex: 0, 1969 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1970 RampUp: false, 1971 NumaAllocationResult: machine.NewCPUSet(1), 1972 Annotations: map[string]string{ 1973 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1974 }, 1975 Labels: map[string]string{ 1976 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 1977 }, 1978 ExtraControlKnobInfo: make(map[string]commonstate.ControlKnobInfo), 1979 }, 1980 }, 1981 }, 1982 }, 1983 2: &state.NUMANodeState{ 1984 TotalMemSize: 8589934592, 1985 SystemReserved: 1073741824, 1986 Allocatable: 7516192768, 1987 Allocated: 0, 1988 Free: 7516192768, 1989 PodEntries: state.PodEntries{ 1990 pod2UID: state.ContainerEntries{ 1991 testName: &state.AllocationInfo{ 1992 PodUid: pod2UID, 1993 PodNamespace: testName, 1994 PodName: testName, 1995 ContainerName: testName, 1996 ContainerType: pluginapi.ContainerType_MAIN.String(), 1997 ContainerIndex: 0, 1998 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 1999 RampUp: false, 2000 NumaAllocationResult: machine.NewCPUSet(2), 2001 Annotations: map[string]string{ 2002 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2003 }, 2004 Labels: map[string]string{ 2005 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2006 }, 2007 ExtraControlKnobInfo: make(map[string]commonstate.ControlKnobInfo), 2008 }, 2009 }, 2010 pod3UID: state.ContainerEntries{ 2011 testName: &state.AllocationInfo{ 2012 PodUid: pod3UID, 2013 PodNamespace: testName, 2014 PodName: testName, 2015 ContainerName: testName, 2016 ContainerType: pluginapi.ContainerType_MAIN.String(), 2017 ContainerIndex: 0, 2018 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2019 RampUp: false, 2020 NumaAllocationResult: machine.NewCPUSet(2), 2021 Annotations: map[string]string{ 2022 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2023 }, 2024 Labels: map[string]string{ 2025 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2026 }, 2027 ExtraControlKnobInfo: make(map[string]commonstate.ControlKnobInfo), 2028 }, 2029 }, 2030 }, 2031 }, 2032 3: &state.NUMANodeState{ 2033 TotalMemSize: 8589934592, 2034 SystemReserved: 1073741824, 2035 Allocatable: 7516192768, 2036 Allocated: 0, 2037 Free: 7516192768, 2038 PodEntries: state.PodEntries{ 2039 pod2UID: state.ContainerEntries{ 2040 testName: &state.AllocationInfo{ 2041 PodUid: pod2UID, 2042 PodNamespace: testName, 2043 PodName: testName, 2044 ContainerName: testName, 2045 ContainerType: pluginapi.ContainerType_MAIN.String(), 2046 ContainerIndex: 0, 2047 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 2048 RampUp: false, 2049 NumaAllocationResult: machine.NewCPUSet(3), 2050 Annotations: map[string]string{ 2051 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2052 }, 2053 Labels: map[string]string{ 2054 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 2055 }, 2056 ExtraControlKnobInfo: make(map[string]commonstate.ControlKnobInfo), 2057 }, 2058 }, 2059 pod3UID: state.ContainerEntries{ 2060 testName: &state.AllocationInfo{ 2061 PodUid: pod3UID, 2062 PodNamespace: testName, 2063 PodName: testName, 2064 ContainerName: testName, 2065 ContainerType: pluginapi.ContainerType_MAIN.String(), 2066 ContainerIndex: 0, 2067 QoSLevel: consts.PodAnnotationQoSLevelReclaimedCores, 2068 RampUp: false, 2069 NumaAllocationResult: machine.NewCPUSet(3), 2070 Annotations: map[string]string{ 2071 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2072 }, 2073 Labels: map[string]string{ 2074 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelReclaimedCores, 2075 }, 2076 ExtraControlKnobInfo: make(map[string]commonstate.ControlKnobInfo), 2077 }, 2078 }, 2079 }, 2080 }, 2081 }, 2082 }, 2083 }, 2084 { 2085 description: "apply memory limits in invalid high level cgroup relative path", 2086 podResourceEntries: nil, 2087 lwResp: &advisorsvc.ListAndWatchResponse{ 2088 ExtraEntries: []*advisorsvc.CalculationInfo{ 2089 { 2090 CgroupPath: "invalid", 2091 CalculationResult: &advisorsvc.CalculationResult{ 2092 Values: map[string]string{ 2093 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): "12345", 2094 }, 2095 }, 2096 }, 2097 }, 2098 }, 2099 expectedPodResourceEntries: nil, 2100 expectedMachineState: nil, 2101 }, 2102 } 2103 2104 for _, tc := range testCases { 2105 tmpDir, err := ioutil.TempDir("", "checkpoint-TestHandleAdvisorResp") 2106 as.Nil(err) 2107 2108 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 2109 as.Nil(err) 2110 2111 dynamicPolicy.metaServer = &metaserver.MetaServer{ 2112 MetaAgent: &agent.MetaAgent{ 2113 PodFetcher: &pod.PodFetcherStub{ 2114 PodList: []*v1.Pod{ 2115 { 2116 ObjectMeta: metav1.ObjectMeta{ 2117 UID: types.UID(pod2UID), 2118 }, 2119 Spec: v1.PodSpec{ 2120 Containers: []v1.Container{ 2121 {Name: testName}, 2122 }, 2123 }, 2124 }, 2125 }, 2126 }, 2127 }, 2128 } 2129 2130 memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyMemoryLimitInBytes, 2131 memoryadvisor.ControlKnobHandlerWithChecker(dynamicPolicy.handleAdvisorMemoryLimitInBytes)) 2132 memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyCPUSetMems, 2133 memoryadvisor.ControlKnobHandlerWithChecker(handleAdvisorCPUSetMems)) 2134 memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyDropCache, 2135 memoryadvisor.ControlKnobHandlerWithChecker(dynamicPolicy.handleAdvisorDropCache)) 2136 memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnowKeyMemoryOffloading, 2137 memoryadvisor.ControlKnobHandlerWithChecker(dynamicPolicy.handleAdvisorMemoryOffloading)) 2138 2139 machineState, err := state.GenerateMachineStateFromPodEntries(machineInfo, tc.podResourceEntries, resourcesReservedMemory) 2140 as.Nil(err) 2141 2142 if tc.podResourceEntries != nil { 2143 dynamicPolicy.state.SetPodResourceEntries(tc.podResourceEntries) 2144 dynamicPolicy.state.SetMachineState(machineState) 2145 } 2146 2147 err = dynamicPolicy.handleAdvisorResp(tc.lwResp) 2148 as.Nilf(err, "dynamicPolicy.handleAdvisorResp got err: %v, case: %s", err, tc.description) 2149 2150 if tc.expectedPodResourceEntries != nil { 2151 as.Equalf(tc.expectedPodResourceEntries, dynamicPolicy.state.GetPodResourceEntries(), 2152 "PodResourceEntries mismatches with expected one, failed in test case: %s", tc.description) 2153 } 2154 2155 if tc.expectedMachineState != nil { 2156 as.Equalf(tc.expectedMachineState, dynamicPolicy.state.GetMachineState(), 2157 "MachineState mismatches with expected one, failed in test case: %s", tc.description) 2158 } 2159 2160 os.RemoveAll(tmpDir) 2161 } 2162 } 2163 2164 func TestSetExtraControlKnobByConfigForAllocationInfo(t *testing.T) { 2165 t.Parallel() 2166 2167 as := require.New(t) 2168 2169 pod1UID := string(uuid.NewUUID()) 2170 testName := "test" 2171 2172 testMemLimitAnnoKey := "test_mem_limit_anno_key" 2173 2174 testCases := []struct { 2175 description string 2176 pod *v1.Pod 2177 inputAllocationInfo *state.AllocationInfo 2178 extraControlKnobConfigs commonstate.ExtraControlKnobConfigs 2179 outputAllocationInfo *state.AllocationInfo 2180 }{ 2181 { 2182 description: "input allocationInfo already has extra control knob entry corresponding to extraControlKnobConfigs", 2183 pod: &v1.Pod{ 2184 ObjectMeta: metav1.ObjectMeta{ 2185 Annotations: map[string]string{testMemLimitAnnoKey: "6516192768"}, 2186 }, 2187 }, 2188 inputAllocationInfo: &state.AllocationInfo{ 2189 PodUid: pod1UID, 2190 PodNamespace: testName, 2191 PodName: testName, 2192 ContainerName: testName, 2193 ContainerType: pluginapi.ContainerType_MAIN.String(), 2194 ContainerIndex: 0, 2195 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2196 RampUp: false, 2197 AggregatedQuantity: 7516192768, 2198 NumaAllocationResult: machine.NewCPUSet(0), 2199 TopologyAwareAllocations: map[int]uint64{ 2200 0: 7516192768, 2201 }, 2202 Annotations: map[string]string{ 2203 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2204 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2205 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 2206 }, 2207 Labels: map[string]string{ 2208 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2209 }, 2210 ExtraControlKnobInfo: map[string]commonstate.ControlKnobInfo{ 2211 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 2212 ControlKnobValue: "5516192768", 2213 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 2214 }, 2215 }, 2216 }, 2217 extraControlKnobConfigs: map[string]commonstate.ExtraControlKnobConfig{ 2218 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 2219 PodExplicitlyAnnotationKey: testMemLimitAnnoKey, 2220 QoSLevelToDefaultValue: map[string]string{ 2221 consts.PodAnnotationQoSLevelDedicatedCores: "1516192768", 2222 consts.PodAnnotationQoSLevelSharedCores: "2516192768", 2223 consts.PodAnnotationQoSLevelReclaimedCores: "3516192768", 2224 }, 2225 ControlKnobInfo: commonstate.ControlKnobInfo{ 2226 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 2227 }, 2228 }, 2229 }, 2230 outputAllocationInfo: &state.AllocationInfo{ 2231 PodUid: pod1UID, 2232 PodNamespace: testName, 2233 PodName: testName, 2234 ContainerName: testName, 2235 ContainerType: pluginapi.ContainerType_MAIN.String(), 2236 ContainerIndex: 0, 2237 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2238 RampUp: false, 2239 AggregatedQuantity: 7516192768, 2240 NumaAllocationResult: machine.NewCPUSet(0), 2241 TopologyAwareAllocations: map[int]uint64{ 2242 0: 7516192768, 2243 }, 2244 Annotations: map[string]string{ 2245 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2246 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2247 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 2248 }, 2249 Labels: map[string]string{ 2250 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2251 }, 2252 ExtraControlKnobInfo: map[string]commonstate.ControlKnobInfo{ 2253 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 2254 ControlKnobValue: "5516192768", 2255 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 2256 }, 2257 }, 2258 }, 2259 }, 2260 { 2261 description: "set allocationInfo default control knob value by annotation", 2262 pod: &v1.Pod{ 2263 ObjectMeta: metav1.ObjectMeta{ 2264 Annotations: map[string]string{testMemLimitAnnoKey: "6516192768"}, 2265 }, 2266 }, 2267 inputAllocationInfo: &state.AllocationInfo{ 2268 PodUid: pod1UID, 2269 PodNamespace: testName, 2270 PodName: testName, 2271 ContainerName: testName, 2272 ContainerType: pluginapi.ContainerType_MAIN.String(), 2273 ContainerIndex: 0, 2274 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2275 RampUp: false, 2276 AggregatedQuantity: 7516192768, 2277 NumaAllocationResult: machine.NewCPUSet(0), 2278 TopologyAwareAllocations: map[int]uint64{ 2279 0: 7516192768, 2280 }, 2281 Annotations: map[string]string{ 2282 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2283 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2284 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 2285 }, 2286 Labels: map[string]string{ 2287 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2288 }, 2289 }, 2290 extraControlKnobConfigs: map[string]commonstate.ExtraControlKnobConfig{ 2291 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 2292 PodExplicitlyAnnotationKey: testMemLimitAnnoKey, 2293 QoSLevelToDefaultValue: map[string]string{ 2294 consts.PodAnnotationQoSLevelDedicatedCores: "1516192768", 2295 consts.PodAnnotationQoSLevelSharedCores: "2516192768", 2296 consts.PodAnnotationQoSLevelReclaimedCores: "3516192768", 2297 }, 2298 ControlKnobInfo: commonstate.ControlKnobInfo{ 2299 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 2300 }, 2301 }, 2302 }, 2303 outputAllocationInfo: &state.AllocationInfo{ 2304 PodUid: pod1UID, 2305 PodNamespace: testName, 2306 PodName: testName, 2307 ContainerName: testName, 2308 ContainerType: pluginapi.ContainerType_MAIN.String(), 2309 ContainerIndex: 0, 2310 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2311 RampUp: false, 2312 AggregatedQuantity: 7516192768, 2313 NumaAllocationResult: machine.NewCPUSet(0), 2314 TopologyAwareAllocations: map[int]uint64{ 2315 0: 7516192768, 2316 }, 2317 Annotations: map[string]string{ 2318 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2319 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2320 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 2321 }, 2322 Labels: map[string]string{ 2323 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2324 }, 2325 ExtraControlKnobInfo: map[string]commonstate.ControlKnobInfo{ 2326 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 2327 ControlKnobValue: "6516192768", 2328 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 2329 }, 2330 }, 2331 }, 2332 }, 2333 { 2334 description: "set allocationInfo default control knob value by qos level", 2335 pod: &v1.Pod{}, 2336 inputAllocationInfo: &state.AllocationInfo{ 2337 PodUid: pod1UID, 2338 PodNamespace: testName, 2339 PodName: testName, 2340 ContainerName: testName, 2341 ContainerType: pluginapi.ContainerType_MAIN.String(), 2342 ContainerIndex: 0, 2343 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2344 RampUp: false, 2345 AggregatedQuantity: 7516192768, 2346 NumaAllocationResult: machine.NewCPUSet(0), 2347 TopologyAwareAllocations: map[int]uint64{ 2348 0: 7516192768, 2349 }, 2350 Annotations: map[string]string{ 2351 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2352 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2353 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 2354 }, 2355 Labels: map[string]string{ 2356 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2357 }, 2358 }, 2359 extraControlKnobConfigs: map[string]commonstate.ExtraControlKnobConfig{ 2360 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 2361 PodExplicitlyAnnotationKey: testMemLimitAnnoKey, 2362 QoSLevelToDefaultValue: map[string]string{ 2363 consts.PodAnnotationQoSLevelDedicatedCores: "1516192768", 2364 consts.PodAnnotationQoSLevelSharedCores: "2516192768", 2365 consts.PodAnnotationQoSLevelReclaimedCores: "3516192768", 2366 }, 2367 ControlKnobInfo: commonstate.ControlKnobInfo{ 2368 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 2369 }, 2370 }, 2371 }, 2372 outputAllocationInfo: &state.AllocationInfo{ 2373 PodUid: pod1UID, 2374 PodNamespace: testName, 2375 PodName: testName, 2376 ContainerName: testName, 2377 ContainerType: pluginapi.ContainerType_MAIN.String(), 2378 ContainerIndex: 0, 2379 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2380 RampUp: false, 2381 AggregatedQuantity: 7516192768, 2382 NumaAllocationResult: machine.NewCPUSet(0), 2383 TopologyAwareAllocations: map[int]uint64{ 2384 0: 7516192768, 2385 }, 2386 Annotations: map[string]string{ 2387 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2388 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2389 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 2390 }, 2391 Labels: map[string]string{ 2392 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2393 }, 2394 ExtraControlKnobInfo: map[string]commonstate.ControlKnobInfo{ 2395 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 2396 ControlKnobValue: "1516192768", 2397 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 2398 }, 2399 }, 2400 }, 2401 }, 2402 } 2403 2404 for _, tc := range testCases { 2405 setExtraControlKnobByConfigForAllocationInfo(tc.inputAllocationInfo, tc.extraControlKnobConfigs, tc.pod) 2406 as.Equalf(tc.outputAllocationInfo, tc.inputAllocationInfo, "failed in test case: %s", tc.description) 2407 } 2408 } 2409 2410 func TestSetExtraControlKnobByConfigs(t *testing.T) { 2411 t.Parallel() 2412 2413 as := require.New(t) 2414 2415 tmpDir, err := ioutil.TempDir("", "checkpoint-TestSetExtraControlKnobByConfigs") 2416 as.Nil(err) 2417 defer os.RemoveAll(tmpDir) 2418 2419 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2420 as.Nil(err) 2421 2422 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 2423 as.Nil(err) 2424 2425 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 2426 as.Nil(err) 2427 2428 pod1UID := string(uuid.NewUUID()) 2429 testName := "test" 2430 2431 dynamicPolicy.metaServer = &metaserver.MetaServer{ 2432 MetaAgent: &agent.MetaAgent{ 2433 PodFetcher: &pod.PodFetcherStub{ 2434 PodList: []*v1.Pod{ 2435 { 2436 ObjectMeta: metav1.ObjectMeta{ 2437 UID: types.UID(pod1UID), 2438 }, 2439 Spec: v1.PodSpec{ 2440 Containers: []v1.Container{ 2441 {Name: testName}, 2442 }, 2443 }, 2444 }, 2445 }, 2446 }, 2447 }, 2448 } 2449 2450 testMemLimitAnnoKey := "test_mem_limit_anno_key" 2451 dynamicPolicy.extraControlKnobConfigs = commonstate.ExtraControlKnobConfigs{ 2452 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 2453 PodExplicitlyAnnotationKey: testMemLimitAnnoKey, 2454 QoSLevelToDefaultValue: map[string]string{ 2455 consts.PodAnnotationQoSLevelDedicatedCores: "1516192768", 2456 consts.PodAnnotationQoSLevelSharedCores: "2516192768", 2457 consts.PodAnnotationQoSLevelReclaimedCores: "3516192768", 2458 }, 2459 ControlKnobInfo: commonstate.ControlKnobInfo{ 2460 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 2461 }, 2462 }, 2463 } 2464 2465 podEntries := state.PodEntries{ 2466 pod1UID: state.ContainerEntries{ 2467 testName: &state.AllocationInfo{ 2468 PodUid: pod1UID, 2469 PodNamespace: testName, 2470 PodName: testName, 2471 ContainerName: testName, 2472 ContainerType: pluginapi.ContainerType_MAIN.String(), 2473 ContainerIndex: 0, 2474 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2475 RampUp: false, 2476 AggregatedQuantity: 7516192768, 2477 NumaAllocationResult: machine.NewCPUSet(0), 2478 TopologyAwareAllocations: map[int]uint64{ 2479 0: 7516192768, 2480 }, 2481 Annotations: map[string]string{ 2482 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2483 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2484 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 2485 }, 2486 Labels: map[string]string{ 2487 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2488 }, 2489 }, 2490 }, 2491 } 2492 2493 podResourceEntries := state.PodResourceEntries{ 2494 v1.ResourceMemory: podEntries, 2495 } 2496 2497 reservedMemory, err := getReservedMemory(fakeConf, &metaserver.MetaServer{}, machineInfo) 2498 as.Nil(err) 2499 2500 reserved := map[v1.ResourceName]map[int]uint64{ 2501 v1.ResourceMemory: reservedMemory, 2502 } 2503 2504 resourcesMachineState, err := state.GenerateMachineStateFromPodEntries(machineInfo, podResourceEntries, reserved) 2505 as.Nil(err) 2506 2507 dynamicPolicy.state.SetPodResourceEntries(podResourceEntries) 2508 dynamicPolicy.state.SetMachineState(resourcesMachineState) 2509 2510 dynamicPolicy.setExtraControlKnobByConfigs(nil, nil, nil, nil, nil) 2511 2512 expectedAllocationInfo := &state.AllocationInfo{ 2513 PodUid: pod1UID, 2514 PodNamespace: testName, 2515 PodName: testName, 2516 ContainerName: testName, 2517 ContainerType: pluginapi.ContainerType_MAIN.String(), 2518 ContainerIndex: 0, 2519 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2520 RampUp: false, 2521 AggregatedQuantity: 7516192768, 2522 NumaAllocationResult: machine.NewCPUSet(0), 2523 TopologyAwareAllocations: map[int]uint64{ 2524 0: 7516192768, 2525 }, 2526 Annotations: map[string]string{ 2527 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2528 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2529 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 2530 }, 2531 Labels: map[string]string{ 2532 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2533 }, 2534 ExtraControlKnobInfo: map[string]commonstate.ControlKnobInfo{ 2535 string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): { 2536 ControlKnobValue: "1516192768", 2537 OciPropertyName: util.OCIPropertyNameMemoryLimitInBytes, 2538 }, 2539 }, 2540 } 2541 2542 as.Equal(expectedAllocationInfo, dynamicPolicy.state.GetPodResourceEntries()[v1.ResourceMemory][pod1UID][testName]) 2543 } 2544 2545 func makeMetaServer() *metaserver.MetaServer { 2546 cpuTopology, _ := machine.GenerateDummyCPUTopology(16, 2, 4) 2547 machineInfo, _ := machine.GenerateDummyMachineInfo(4, 32) 2548 2549 return &metaserver.MetaServer{ 2550 MetaAgent: &metaserveragent.MetaAgent{ 2551 KatalystMachineInfo: &machine.KatalystMachineInfo{ 2552 MachineInfo: machineInfo, 2553 CPUTopology: cpuTopology, 2554 ExtraNetworkInfo: &machine.ExtraNetworkInfo{}, 2555 }, 2556 PodFetcher: &pod.PodFetcherStub{}, 2557 }, 2558 ExternalManager: external.InitExternalManager(&pod.PodFetcherStub{}), 2559 } 2560 } 2561 2562 func makeTestGenericContext(t *testing.T) *appagent.GenericContext { 2563 genericCtx, err := katalystbase.GenerateFakeGenericContext([]runtime.Object{}) 2564 assert.NoError(t, err) 2565 2566 return &appagent.GenericContext{ 2567 GenericContext: genericCtx, 2568 MetaServer: makeMetaServer(), 2569 PluginManager: nil, 2570 } 2571 } 2572 2573 func TestNewAndStartDynamicPolicy(t *testing.T) { 2574 t.Parallel() 2575 2576 as := require.New(t) 2577 2578 tmpDir, err := ioutil.TempDir("", "checkpoint-TestNewAndStartDynamicPolicy") 2579 as.Nil(err) 2580 2581 agentCtx := makeTestGenericContext(t) 2582 _, component, err := NewDynamicPolicy(agentCtx, &config.Configuration{ 2583 GenericConfiguration: &generic.GenericConfiguration{}, 2584 AgentConfiguration: &configagent.AgentConfiguration{ 2585 GenericAgentConfiguration: &configagent.GenericAgentConfiguration{ 2586 QRMAdvisorConfiguration: &global.QRMAdvisorConfiguration{}, 2587 GenericQRMPluginConfiguration: &qrmconfig.GenericQRMPluginConfiguration{ 2588 StateFileDirectory: tmpDir, 2589 QRMPluginSocketDirs: []string{path.Join(tmpDir, "test.sock")}, 2590 }, 2591 }, 2592 StaticAgentConfiguration: &configagent.StaticAgentConfiguration{ 2593 QRMPluginsConfiguration: &qrmconfig.QRMPluginsConfiguration{ 2594 MemoryQRMPluginConfig: &qrmconfig.MemoryQRMPluginConfig{ 2595 PolicyName: memconsts.MemoryResourcePluginPolicyNameDynamic, 2596 ReservedMemoryGB: 4, 2597 SkipMemoryStateCorruption: true, 2598 EnableSettingMemoryMigrate: false, 2599 EnableMemoryAdvisor: false, 2600 ExtraControlKnobConfigFile: "", 2601 }, 2602 }, 2603 }, 2604 }, 2605 }, nil, "test_dynamic_policy") 2606 as.Nil(err) 2607 2608 ctx, cancel := context.WithCancel(context.Background()) 2609 var wg sync.WaitGroup 2610 wg.Add(1) 2611 2612 go func() { 2613 component.Run(ctx) 2614 wg.Done() 2615 }() 2616 2617 cancel() 2618 wg.Wait() 2619 } 2620 2621 func TestRemoveContainer(t *testing.T) { 2622 t.Parallel() 2623 2624 as := require.New(t) 2625 2626 tmpDir, err := ioutil.TempDir("", "checkpoint-TestRemoveContainer") 2627 as.Nil(err) 2628 defer os.RemoveAll(tmpDir) 2629 2630 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2631 as.Nil(err) 2632 2633 machineInfo := &info.MachineInfo{} 2634 2635 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 2636 as.Nil(err) 2637 2638 podUID := "podUID" 2639 containerName := "testName" 2640 2641 dynamicPolicy.state.SetPodResourceEntries(state.PodResourceEntries{ 2642 v1.ResourceMemory: state.PodEntries{ 2643 podUID: state.ContainerEntries{ 2644 containerName: &state.AllocationInfo{ 2645 PodUid: "podUID", 2646 PodNamespace: "testName", 2647 PodName: "testName", 2648 ContainerName: "testName", 2649 ContainerType: pluginapi.ContainerType_MAIN.String(), 2650 ContainerIndex: 0, 2651 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2652 RampUp: false, 2653 AggregatedQuantity: 9663676416, 2654 NumaAllocationResult: machine.NewCPUSet(0), 2655 TopologyAwareAllocations: map[int]uint64{ 2656 0: 9663676416, 2657 }, 2658 Annotations: map[string]string{ 2659 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2660 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2661 }, 2662 Labels: map[string]string{ 2663 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2664 }, 2665 }, 2666 }, 2667 }, 2668 }) 2669 2670 allocationInfo := dynamicPolicy.state.GetAllocationInfo(v1.ResourceMemory, podUID, containerName) 2671 as.NotNil(allocationInfo) 2672 2673 dynamicPolicy.removeContainer(podUID, containerName) 2674 2675 allocationInfo = dynamicPolicy.state.GetAllocationInfo(v1.ResourceMemory, podUID, containerName) 2676 as.Nil(allocationInfo) 2677 2678 dynamicPolicy.removeContainer(podUID, containerName) 2679 2680 allocationInfo = dynamicPolicy.state.GetAllocationInfo(v1.ResourceMemory, podUID, containerName) 2681 as.Nil(allocationInfo) 2682 } 2683 2684 func TestServeForAdvisor(t *testing.T) { 2685 t.Parallel() 2686 2687 as := require.New(t) 2688 2689 tmpDir, err := ioutil.TempDir("", "checkpoint-TestServeForAdvisor") 2690 as.Nil(err) 2691 defer os.RemoveAll(tmpDir) 2692 2693 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2694 as.Nil(err) 2695 2696 machineInfo := &info.MachineInfo{} 2697 2698 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 2699 as.Nil(err) 2700 2701 tmpMemPluginSvrSockDir, err := ioutil.TempDir("", "checkpoint-TestServeForAdvisor-MemSvrSock") 2702 as.Nil(err) 2703 2704 dynamicPolicy.memoryPluginSocketAbsPath = path.Join(tmpMemPluginSvrSockDir, "memSvr.sock") 2705 stopCh := make(chan struct{}) 2706 2707 go func() { 2708 select { 2709 case <-time.After(time.Second): 2710 close(stopCh) 2711 } 2712 }() 2713 dynamicPolicy.serveForAdvisor(stopCh) 2714 } 2715 2716 func TestDynamicPolicy_ListContainers(t *testing.T) { 2717 t.Parallel() 2718 2719 as := require.New(t) 2720 2721 tmpDir, err := ioutil.TempDir("", "checkpoint-TestDynamicPolicy_ListContainers") 2722 as.Nil(err) 2723 defer os.RemoveAll(tmpDir) 2724 2725 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2726 as.Nil(err) 2727 2728 machineInfo := &info.MachineInfo{} 2729 2730 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 2731 as.Nil(err) 2732 2733 allocationInfo := &state.AllocationInfo{ 2734 PodUid: "podUID", 2735 PodNamespace: "testName", 2736 PodName: "testName", 2737 ContainerName: "testName", 2738 ContainerType: pluginapi.ContainerType_MAIN.String(), 2739 ContainerIndex: 0, 2740 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2741 RampUp: false, 2742 AggregatedQuantity: 9663676416, 2743 NumaAllocationResult: machine.NewCPUSet(0), 2744 TopologyAwareAllocations: map[int]uint64{ 2745 0: 9663676416, 2746 }, 2747 Annotations: map[string]string{ 2748 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2749 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2750 }, 2751 Labels: map[string]string{ 2752 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2753 }, 2754 } 2755 2756 dynamicPolicy.state.SetPodResourceEntries(state.PodResourceEntries{ 2757 v1.ResourceMemory: state.PodEntries{ 2758 "podUID": state.ContainerEntries{ 2759 "testName": allocationInfo, 2760 }, 2761 }, 2762 }) 2763 2764 containerType, found := pluginapi.ContainerType_value[allocationInfo.ContainerType] 2765 as.True(found) 2766 2767 type args struct { 2768 in0 context.Context 2769 in1 *advisorsvc.Empty 2770 } 2771 tests := []struct { 2772 name string 2773 args args 2774 want *advisorsvc.ListContainersResponse 2775 wantErr bool 2776 }{ 2777 { 2778 name: "test list contaienrs", 2779 args: args{ 2780 in0: context.Background(), 2781 in1: &advisorsvc.Empty{}, 2782 }, 2783 want: &advisorsvc.ListContainersResponse{ 2784 Containers: []*advisorsvc.ContainerMetadata{ 2785 { 2786 PodUid: allocationInfo.PodUid, 2787 PodNamespace: allocationInfo.PodNamespace, 2788 PodName: allocationInfo.PodName, 2789 ContainerName: allocationInfo.ContainerName, 2790 ContainerIndex: allocationInfo.ContainerIndex, 2791 ContainerType: pluginapi.ContainerType(containerType), 2792 QosLevel: allocationInfo.QoSLevel, 2793 Labels: maputil.CopySS(allocationInfo.Labels), 2794 Annotations: maputil.CopySS(allocationInfo.Annotations), 2795 }, 2796 }, 2797 }, 2798 wantErr: false, 2799 }, 2800 } 2801 for _, tt := range tests { 2802 tt := tt 2803 t.Run(tt.name, func(t *testing.T) { 2804 t.Parallel() 2805 2806 got, err := dynamicPolicy.ListContainers(tt.args.in0, tt.args.in1) 2807 if (err != nil) != tt.wantErr { 2808 t.Errorf("DynamicPolicy.ListContainers() error = %v, wantErr %v", err, tt.wantErr) 2809 return 2810 } 2811 if !reflect.DeepEqual(got, tt.want) { 2812 t.Errorf("DynamicPolicy.ListContainers() = %v, want %v", got, tt.want) 2813 } 2814 }) 2815 } 2816 } 2817 2818 func TestDynamicPolicy_hasLastLevelEnhancementKey(t *testing.T) { 2819 t.Parallel() 2820 2821 as := require.New(t) 2822 2823 tmpDir, err := ioutil.TempDir("", "checkpoint-TestDynamicPolicy_hasLastLevelEnhancementKey") 2824 as.Nil(err) 2825 defer os.RemoveAll(tmpDir) 2826 2827 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2828 as.Nil(err) 2829 2830 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 2831 as.Nil(err) 2832 2833 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 2834 as.Nil(err) 2835 2836 dynamicPolicy.state.SetPodResourceEntries(state.PodResourceEntries{ 2837 v1.ResourceMemory: state.PodEntries{ 2838 "podUID": state.ContainerEntries{ 2839 "testName": &state.AllocationInfo{ 2840 PodUid: "podUID", 2841 PodNamespace: "testName", 2842 PodName: "testName", 2843 ContainerName: "testName", 2844 ContainerType: pluginapi.ContainerType_MAIN.String(), 2845 ContainerIndex: 0, 2846 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2847 RampUp: false, 2848 AggregatedQuantity: 9663676416, 2849 NumaAllocationResult: machine.NewCPUSet(0), 2850 TopologyAwareAllocations: map[int]uint64{ 2851 0: 9663676416, 2852 }, 2853 Annotations: map[string]string{ 2854 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2855 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 2856 consts.PodAnnotationMemoryEnhancementOOMPriority: strconv.Itoa(qos.DefaultDedicatedCoresOOMPriorityScore), 2857 }, 2858 Labels: map[string]string{ 2859 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 2860 }, 2861 }, 2862 }, 2863 "podUID-1": state.ContainerEntries{ 2864 "testName-1": &state.AllocationInfo{ 2865 PodUid: "podUID-1", 2866 PodNamespace: "testName-1", 2867 PodName: "testName-1", 2868 ContainerName: "testName-1", 2869 ContainerType: pluginapi.ContainerType_MAIN.String(), 2870 ContainerIndex: 0, 2871 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 2872 RampUp: false, 2873 AggregatedQuantity: 9663676416, 2874 NumaAllocationResult: machine.NewCPUSet(0), 2875 TopologyAwareAllocations: map[int]uint64{ 2876 0: 9663676416, 2877 }, 2878 }, 2879 }, 2880 }, 2881 }) 2882 2883 type args struct { 2884 lastLevelEnhancementKey string 2885 podUID string 2886 } 2887 tests := []struct { 2888 name string 2889 args args 2890 want bool 2891 }{ 2892 { 2893 name: "test hasLastLevelEnhancementKey with key", 2894 args: args{ 2895 lastLevelEnhancementKey: consts.PodAnnotationMemoryEnhancementOOMPriority, 2896 podUID: "podUID", 2897 }, 2898 want: true, 2899 }, 2900 { 2901 name: "test hasLastLevelEnhancementKey without key", 2902 args: args{ 2903 lastLevelEnhancementKey: consts.PodAnnotationMemoryEnhancementOOMPriority, 2904 podUID: "podUID-1", 2905 }, 2906 want: false, 2907 }, 2908 } 2909 for _, tt := range tests { 2910 tt := tt 2911 t.Run(tt.name, func(t *testing.T) { 2912 t.Parallel() 2913 if got := dynamicPolicy.hasLastLevelEnhancementKey(tt.args.lastLevelEnhancementKey, tt.args.podUID); got != tt.want { 2914 t.Errorf("DynamicPolicy.hasLastLevelEnhancementKey() = %v, want %v", got, tt.want) 2915 } 2916 }) 2917 } 2918 } 2919 2920 var once sync.Once 2921 2922 func bpfTestInit() { 2923 if err := rlimit.RemoveMemlock(); err != nil { 2924 fmt.Println("Error removing memlock:", err) 2925 } 2926 } 2927 2928 // TempBPFFS creates a temporary directory on a BPF FS. 2929 // 2930 // The directory is automatically cleaned up at the end of the test run. 2931 func TempBPFFS(tb testing.TB) string { 2932 tb.Helper() 2933 2934 if err := os.MkdirAll("/sys/fs/bpf", 0o755); err != nil { 2935 tb.Fatal("Failed to create /sys/fs/bpf directory:", err) 2936 } 2937 2938 tmp, err := os.MkdirTemp("/sys/fs/bpf", "ebpf-test") 2939 if err != nil { 2940 tb.Fatal("Create temporary directory on BPFFS:", err) 2941 } 2942 tb.Cleanup(func() { os.RemoveAll(tmp) }) 2943 2944 return tmp 2945 } 2946 2947 func createMap(t *testing.T) *ebpf.Map { 2948 t.Helper() 2949 2950 m, err := ebpf.NewMap(&ebpf.MapSpec{ 2951 Type: ebpf.Hash, 2952 KeySize: 8, 2953 ValueSize: 8, 2954 MaxEntries: 2, 2955 }) 2956 if err != nil { 2957 t.Fatal(err) 2958 } 2959 return m 2960 } 2961 2962 func TestClearResidualOOMPriority(t *testing.T) { 2963 t.Parallel() 2964 2965 as := require.New(t) 2966 2967 once.Do(bpfTestInit) 2968 2969 tmpDir, err := ioutil.TempDir("", "checkpoint-TestClearResidualOOMPriority") 2970 as.Nil(err) 2971 defer os.RemoveAll(tmpDir) 2972 2973 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 2974 as.Nil(err) 2975 2976 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 2977 as.Nil(err) 2978 2979 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 2980 as.Nil(err) 2981 2982 dynamicPolicy.metaServer = &metaserver.MetaServer{ 2983 MetaAgent: &agent.MetaAgent{ 2984 PodFetcher: &pod.PodFetcherStub{}, 2985 }, 2986 } 2987 2988 dynamicPolicy.enableOOMPriority = true 2989 dynamicPolicy.enhancementHandlers = make(util.ResourceEnhancementHandlerMap) 2990 2991 m := createMap(t) 2992 defer m.Close() 2993 2994 tmp := TempBPFFS(t) 2995 path := filepath.Join(tmp, "map") 2996 2997 if err := m.Pin(path); err != nil { 2998 t.Fatal(err) 2999 } 3000 3001 pinned := m.IsPinned() 3002 as.True(pinned) 3003 3004 m.Close() 3005 3006 dynamicPolicy.clearResidualOOMPriority(&config.Configuration{ 3007 GenericConfiguration: &generic.GenericConfiguration{ 3008 QoSConfiguration: dynamicPolicy.qosConfig, 3009 }, 3010 }, nil, nil, dynamicPolicy.emitter, dynamicPolicy.metaServer) 3011 3012 dynamicPolicy.oomPriorityMap, err = ebpf.LoadPinnedMap(path, &ebpf.LoadPinOptions{}) 3013 if err != nil { 3014 t.Fatalf("load oom priority map at: %s failed with error: %v", path, err) 3015 } 3016 defer dynamicPolicy.oomPriorityMap.Close() 3017 3018 time.Sleep(1 * time.Second) 3019 3020 if err := dynamicPolicy.oomPriorityMap.Put(uint64(0), int64(100)); err != nil { 3021 t.Fatal("Can't put:", err) 3022 } 3023 3024 dynamicPolicy.clearResidualOOMPriority(&config.Configuration{ 3025 GenericConfiguration: &generic.GenericConfiguration{ 3026 QoSConfiguration: dynamicPolicy.qosConfig, 3027 }, 3028 }, nil, nil, dynamicPolicy.emitter, dynamicPolicy.metaServer) 3029 } 3030 3031 func TestSyncOOMPriority(t *testing.T) { 3032 t.Parallel() 3033 3034 as := require.New(t) 3035 3036 once.Do(bpfTestInit) 3037 3038 tmpDir, err := ioutil.TempDir("", "checkpoint-TestSyncOOMPriority") 3039 as.Nil(err) 3040 defer os.RemoveAll(tmpDir) 3041 3042 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 3043 as.Nil(err) 3044 3045 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 3046 as.Nil(err) 3047 3048 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 3049 as.Nil(err) 3050 3051 dynamicPolicy.metaServer = &metaserver.MetaServer{ 3052 MetaAgent: &agent.MetaAgent{ 3053 PodFetcher: &pod.PodFetcherStub{}, 3054 }, 3055 } 3056 3057 m := createMap(t) 3058 defer m.Close() 3059 3060 tmp := TempBPFFS(t) 3061 path := filepath.Join(tmp, "map") 3062 3063 if err := m.Pin(path); err != nil { 3064 t.Fatal(err) 3065 } 3066 3067 pinned := m.IsPinned() 3068 as.True(pinned) 3069 3070 m.Close() 3071 3072 dynamicPolicy.syncOOMPriority(&config.Configuration{ 3073 GenericConfiguration: &generic.GenericConfiguration{ 3074 QoSConfiguration: dynamicPolicy.qosConfig, 3075 }, 3076 }, nil, nil, dynamicPolicy.emitter, dynamicPolicy.metaServer) 3077 3078 dynamicPolicy.oomPriorityMap, err = ebpf.LoadPinnedMap(path, &ebpf.LoadPinOptions{}) 3079 if err != nil { 3080 t.Fatalf("load oom priority map at: %s failed with error: %v", path, err) 3081 } 3082 defer dynamicPolicy.oomPriorityMap.Close() 3083 3084 time.Sleep(1 * time.Second) 3085 3086 dynamicPolicy.syncOOMPriority(&config.Configuration{ 3087 GenericConfiguration: &generic.GenericConfiguration{ 3088 QoSConfiguration: dynamicPolicy.qosConfig, 3089 }, 3090 }, nil, nil, dynamicPolicy.emitter, nil) 3091 3092 dynamicPolicy.syncOOMPriority(&config.Configuration{ 3093 GenericConfiguration: &generic.GenericConfiguration{ 3094 QoSConfiguration: dynamicPolicy.qosConfig, 3095 }, 3096 }, nil, nil, dynamicPolicy.emitter, dynamicPolicy.metaServer) 3097 } 3098 3099 func TestClearOOMPriority(t *testing.T) { 3100 t.Parallel() 3101 3102 as := require.New(t) 3103 3104 once.Do(bpfTestInit) 3105 3106 tmpDir, err := ioutil.TempDir("", "checkpoint-TestClearOOMPriority") 3107 as.Nil(err) 3108 defer os.RemoveAll(tmpDir) 3109 3110 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 3111 as.Nil(err) 3112 3113 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 3114 as.Nil(err) 3115 3116 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 3117 as.Nil(err) 3118 3119 dynamicPolicy.metaServer = &metaserver.MetaServer{ 3120 MetaAgent: &agent.MetaAgent{ 3121 PodFetcher: &pod.PodFetcherStub{}, 3122 }, 3123 ExternalManager: external.InitExternalManager(&pod.PodFetcherStub{}), 3124 } 3125 3126 m := createMap(t) 3127 defer m.Close() 3128 3129 tmp := TempBPFFS(t) 3130 path := filepath.Join(tmp, "map") 3131 3132 if err := m.Pin(path); err != nil { 3133 t.Fatal(err) 3134 } 3135 3136 pinned := m.IsPinned() 3137 as.True(pinned) 3138 3139 m.Close() 3140 3141 req1 := &pluginapi.ResourceRequest{ 3142 PodUid: "pod-1", 3143 } 3144 3145 req2 := &pluginapi.RemovePodRequest{ 3146 PodUid: "pod-1", 3147 } 3148 3149 dynamicPolicy.clearOOMPriority(context.Background(), dynamicPolicy.emitter, 3150 dynamicPolicy.metaServer, req1, nil) 3151 3152 dynamicPolicy.oomPriorityMap, err = ebpf.LoadPinnedMap(path, &ebpf.LoadPinOptions{}) 3153 if err != nil { 3154 t.Fatalf("load oom priority map at: %s failed with error: %v", path, err) 3155 } 3156 defer dynamicPolicy.oomPriorityMap.Close() 3157 3158 time.Sleep(1 * time.Second) 3159 3160 dynamicPolicy.clearOOMPriority(context.Background(), dynamicPolicy.emitter, 3161 dynamicPolicy.metaServer, req2, nil) 3162 } 3163 3164 func TestPollOOMBPFInit(t *testing.T) { 3165 t.Parallel() 3166 3167 as := require.New(t) 3168 3169 once.Do(bpfTestInit) 3170 3171 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 3172 as.Nil(err) 3173 3174 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 3175 as.Nil(err) 3176 3177 m := createMap(t) 3178 defer m.Close() 3179 3180 tmp := TempBPFFS(t) 3181 path := filepath.Join(tmp, "map") 3182 3183 if err := m.Pin(path); err != nil { 3184 t.Fatal(err) 3185 } 3186 3187 pinned := m.IsPinned() 3188 as.True(pinned) 3189 3190 m.Close() 3191 3192 type args struct { 3193 isNilMap bool 3194 isNilInitFunc bool 3195 } 3196 tests := []struct { 3197 name string 3198 args args 3199 isWantNil bool 3200 }{ 3201 { 3202 name: "test with nil ebpf map and nil initOOMPriorityBPF", 3203 args: args{ 3204 isNilMap: true, 3205 isNilInitFunc: true, 3206 }, 3207 isWantNil: true, 3208 }, 3209 { 3210 name: "test with nil ebpf map and dummyInitOOMPriorityBPF", 3211 args: args{ 3212 isNilMap: true, 3213 isNilInitFunc: false, 3214 }, 3215 isWantNil: true, 3216 }, 3217 { 3218 name: "test with loaded ebpf map and dummyInitOOMPriorityBPF", 3219 args: args{ 3220 isNilMap: false, 3221 isNilInitFunc: false, 3222 }, 3223 isWantNil: false, 3224 }, 3225 } 3226 for _, tt := range tests { 3227 tt := tt 3228 t.Run(tt.name, func(t *testing.T) { 3229 t.Parallel() 3230 3231 tmpDir, err := ioutil.TempDir("", "checkpoint-TestPollOOMBPFInit") 3232 as.Nil(err) 3233 defer os.RemoveAll(tmpDir) 3234 3235 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 3236 as.Nil(err) 3237 3238 if !tt.args.isNilMap { 3239 dynamicPolicy.oomPriorityMapPinnedPath = path 3240 } 3241 3242 if !tt.args.isNilInitFunc { 3243 oom.SetInitOOMPriorityBPFFunc(oom.DummyInitOOMPriorityBPF) 3244 } 3245 3246 dynamicPolicy.stopCh = make(chan struct{}) 3247 go dynamicPolicy.PollOOMBPFInit(dynamicPolicy.stopCh) 3248 time.Sleep(1 * time.Second) 3249 close(dynamicPolicy.stopCh) 3250 3251 dynamicPolicy.oomPriorityMapLock.Lock() 3252 defer func() { 3253 if dynamicPolicy.oomPriorityMap != nil { 3254 dynamicPolicy.oomPriorityMap.Close() 3255 } 3256 dynamicPolicy.oomPriorityMapLock.Unlock() 3257 }() 3258 3259 if tt.isWantNil && dynamicPolicy.oomPriorityMap != nil { 3260 t.Errorf("TestPollOOMBPFInit() failed: oomPriorityMap should be nil, but it's not") 3261 } else if !tt.isWantNil && dynamicPolicy.oomPriorityMap == nil { 3262 t.Errorf("TestPollOOMBPFInit() failed: oomPriorityMap should not be nil, but it is") 3263 } 3264 }) 3265 } 3266 } 3267 3268 func TestDynamicPolicy_adjustAllocationEntries(t *testing.T) { 3269 t.Parallel() 3270 3271 metaServer := makeMetaServer() 3272 tmpDir, err := ioutil.TempDir("", "checkpoint-TestDynamicPolicy_adjustAllocationEntries") 3273 assert.NoError(t, err) 3274 defer os.RemoveAll(tmpDir) 3275 3276 cpuTopology, err := machine.GenerateDummyCPUTopology(16, 2, 4) 3277 assert.NoError(t, err) 3278 3279 machineInfo, err := machine.GenerateDummyMachineInfo(4, 32) 3280 assert.NoError(t, err) 3281 3282 metaServer.PodFetcher = &pod.PodFetcherStub{ 3283 PodList: []*v1.Pod{ 3284 { 3285 ObjectMeta: metav1.ObjectMeta{ 3286 UID: types.UID("test-pod-1-uid"), 3287 Name: "test-pod-1", 3288 Annotations: map[string]string{ 3289 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 3290 }, 3291 Labels: map[string]string{ 3292 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 3293 }, 3294 }, 3295 Spec: v1.PodSpec{ 3296 Containers: []v1.Container{ 3297 { 3298 Name: "test-container-1", 3299 Resources: v1.ResourceRequirements{ 3300 Limits: v1.ResourceList{ 3301 v1.ResourceMemory: resource.MustParse("1Gi"), 3302 }, 3303 Requests: v1.ResourceList{ 3304 v1.ResourceMemory: resource.MustParse("1Gi"), 3305 }, 3306 }, 3307 }, 3308 }, 3309 }, 3310 Status: v1.PodStatus{ 3311 ContainerStatuses: []v1.ContainerStatus{ 3312 { 3313 Name: "test-container-1", 3314 ContainerID: "test-container-1-id", 3315 }, 3316 }, 3317 }, 3318 }, 3319 { 3320 ObjectMeta: metav1.ObjectMeta{ 3321 UID: types.UID("test-pod-2-uid"), 3322 Name: "test-pod-2", 3323 Annotations: map[string]string{ 3324 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 3325 }, 3326 Labels: map[string]string{ 3327 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 3328 }, 3329 }, 3330 Spec: v1.PodSpec{ 3331 Containers: []v1.Container{ 3332 { 3333 Name: "test-container-2", 3334 Resources: v1.ResourceRequirements{ 3335 Limits: v1.ResourceList{ 3336 v1.ResourceMemory: resource.MustParse("1Gi"), 3337 }, 3338 Requests: v1.ResourceList{ 3339 v1.ResourceMemory: resource.MustParse("1Gi"), 3340 }, 3341 }, 3342 }, 3343 }, 3344 }, 3345 Status: v1.PodStatus{ 3346 ContainerStatuses: []v1.ContainerStatus{ 3347 { 3348 Name: "test-container-2", 3349 ContainerID: "test-container-2-id", 3350 }, 3351 }, 3352 }, 3353 }, 3354 }, 3355 } 3356 podResourceEntries := state.PodResourceEntries{ 3357 v1.ResourceMemory: state.PodEntries{ 3358 "test-pod-2-uid": state.ContainerEntries{ 3359 "test-container-2": &state.AllocationInfo{ 3360 PodUid: "test-pod-2-uid", 3361 PodNamespace: "test", 3362 PodName: "test-pod-2", 3363 ContainerName: "test-container-2", 3364 ContainerType: pluginapi.ContainerType_MAIN.String(), 3365 ContainerIndex: 0, 3366 QoSLevel: consts.PodAnnotationQoSLevelDedicatedCores, 3367 RampUp: false, 3368 AggregatedQuantity: 7516192768, 3369 NumaAllocationResult: machine.NewCPUSet(0), 3370 TopologyAwareAllocations: map[int]uint64{ 3371 0: 7516192768, 3372 }, 3373 Annotations: map[string]string{ 3374 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 3375 consts.PodAnnotationMemoryEnhancementNumaBinding: consts.PodAnnotationMemoryEnhancementNumaBindingEnable, 3376 consts.PodAnnotationMemoryEnhancementNumaExclusive: consts.PodAnnotationMemoryEnhancementNumaExclusiveEnable, 3377 }, 3378 Labels: map[string]string{ 3379 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 3380 }, 3381 }, 3382 }, 3383 "test-pod-1-uid": state.ContainerEntries{ 3384 "test-container-1": &state.AllocationInfo{ 3385 PodUid: "test-pod-1-uid", 3386 PodNamespace: "test", 3387 PodName: "test-pod-1", 3388 ContainerName: "test-container-1", 3389 ContainerType: pluginapi.ContainerType_MAIN.String(), 3390 ContainerIndex: 0, 3391 QoSLevel: consts.PodAnnotationQoSLevelSharedCores, 3392 RampUp: false, 3393 NumaAllocationResult: machine.NewCPUSet(0, 1, 2, 3), 3394 Annotations: map[string]string{ 3395 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 3396 }, 3397 Labels: map[string]string{ 3398 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelSharedCores, 3399 }, 3400 }, 3401 }, 3402 }, 3403 } 3404 type fields struct { 3405 emitter metrics.MetricEmitter 3406 metaServer *metaserver.MetaServer 3407 } 3408 tests := []struct { 3409 name string 3410 fields fields 3411 wantErr bool 3412 }{ 3413 { 3414 name: "normal adjustment", 3415 fields: fields{ 3416 emitter: metrics.DummyMetrics{}, 3417 metaServer: metaServer, 3418 }, 3419 wantErr: false, 3420 }, 3421 } 3422 for _, tt := range tests { 3423 tt := tt 3424 t.Run(tt.name, func(t *testing.T) { 3425 t.Parallel() 3426 3427 dynamicPolicy, err := getTestDynamicPolicyWithInitialization(cpuTopology, machineInfo, tmpDir) 3428 assert.NoError(t, err) 3429 dynamicPolicy.emitter = tt.fields.emitter 3430 dynamicPolicy.metaServer = tt.fields.metaServer 3431 dynamicPolicy.asyncWorkers = asyncworker.NewAsyncWorkers(memoryPluginAsyncWorkersName, dynamicPolicy.emitter) 3432 dynamicPolicy.state.SetPodResourceEntries(podResourceEntries) 3433 reservedMemory, err := getReservedMemory(fakeConf, dynamicPolicy.metaServer, machineInfo) 3434 assert.NoError(t, err) 3435 3436 resourcesReservedMemory := map[v1.ResourceName]map[int]uint64{ 3437 v1.ResourceMemory: reservedMemory, 3438 } 3439 machineState, err := state.GenerateMachineStateFromPodEntries(machineInfo, podResourceEntries, resourcesReservedMemory) 3440 assert.NoError(t, err) 3441 dynamicPolicy.state.SetMachineState(machineState) 3442 3443 if err := dynamicPolicy.adjustAllocationEntries(); (err != nil) != tt.wantErr { 3444 t.Errorf("DynamicPolicy.adjustAllocationEntries() error = %v, wantErr %v", err, tt.wantErr) 3445 } 3446 }) 3447 } 3448 }