github.com/kubewharf/katalyst-core@v0.5.3/pkg/controller/vpa/util/resource.go (about) 1 /* 2 Copyright 2022 The Katalyst Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package util 18 19 import ( 20 "fmt" 21 "sort" 22 23 core "k8s.io/api/core/v1" 24 "k8s.io/klog/v2" 25 26 apis "github.com/kubewharf/katalyst-api/pkg/apis/autoscaling/v1alpha1" 27 "github.com/kubewharf/katalyst-core/pkg/consts" 28 katalystutil "github.com/kubewharf/katalyst-core/pkg/util" 29 "github.com/kubewharf/katalyst-core/pkg/util/native" 30 ) 31 32 /* 33 helper functions to sort slice-organized resources 34 */ 35 36 func sortPodResources(podResources []apis.PodResources) { 37 sort.SliceStable(podResources, func(i, j int) bool { 38 if podResources[i].PodName == nil { 39 return true 40 } 41 return *podResources[i].PodName > *podResources[j].PodName 42 }) 43 44 for _, pr := range podResources { 45 sortContainerResources(pr.ContainerResources) 46 } 47 } 48 49 func sortContainerResources(containerResources []apis.ContainerResources) { 50 sort.SliceStable(containerResources, func(i, j int) bool { 51 if containerResources[i].ContainerName == nil { 52 return true 53 } 54 return *containerResources[i].ContainerName > *containerResources[j].ContainerName 55 }) 56 } 57 58 func sortRecommendedPodResources(recommendedPodResources []apis.RecommendedPodResources) { 59 sort.SliceStable(recommendedPodResources, func(i, j int) bool { 60 if recommendedPodResources[i].PodName == nil { 61 return true 62 } 63 return *recommendedPodResources[i].PodName > *recommendedPodResources[j].PodName 64 }) 65 66 for _, pr := range recommendedPodResources { 67 sortRecommendedContainerResources(pr.ContainerRecommendations) 68 } 69 } 70 71 func sortRecommendedContainerResources(recommendedContainerResources []apis.RecommendedContainerResources) { 72 sort.SliceStable(recommendedContainerResources, func(i, j int) bool { 73 if recommendedContainerResources[i].ContainerName == nil { 74 return true 75 } 76 return *recommendedContainerResources[i].ContainerName > *recommendedContainerResources[j].ContainerName 77 }) 78 } 79 80 /* 81 helper functions to generate status for vpa 82 */ 83 84 // mergeResourceAndRecommendation merge RecommendedContainerResources into ContainerResources, 85 // it works both for requests and limits 86 func mergeResourceAndRecommendation(containerResource apis.ContainerResources, 87 containerRecommendation apis.RecommendedContainerResources, 88 ) apis.ContainerResources { 89 mergeFn := func(resource *apis.ContainerResourceList, recommendation *apis.RecommendedRequestResources) *apis.ContainerResourceList { 90 if recommendation == nil { 91 return resource 92 } 93 94 if resource == nil { 95 return &apis.ContainerResourceList{ 96 UncappedTarget: recommendation.Resources, 97 Target: recommendation.Resources, 98 } 99 } 100 101 resourceCopy := resource.DeepCopy() 102 resourceCopy.UncappedTarget = recommendation.Resources 103 resourceCopy.Target = recommendation.Resources 104 return resourceCopy 105 } 106 107 containerResource.Requests = mergeFn(containerResource.Requests, containerRecommendation.Requests) 108 containerResource.Limits = mergeFn(containerResource.Limits, containerRecommendation.Limits) 109 return containerResource 110 } 111 112 // mergeResourceAndCurrent merge pod current resources into ContainerResources, 113 // it works both for requests and limits 114 func mergeResourceAndCurrent(containerResource apis.ContainerResources, 115 containerCurrent apis.ContainerResources, 116 ) apis.ContainerResources { 117 mergeFn := func(resource *apis.ContainerResourceList, current *apis.ContainerResourceList) *apis.ContainerResourceList { 118 if current == nil { 119 return resource 120 } 121 122 if resource == nil { 123 return nil 124 } 125 126 resourceCopy := resource.DeepCopy() 127 resourceCopy.Current = current.Current 128 return resourceCopy 129 } 130 131 containerResource.Requests = mergeFn(containerResource.Requests, containerCurrent.Requests) 132 containerResource.Limits = mergeFn(containerResource.Limits, containerCurrent.Limits) 133 return containerResource 134 } 135 136 // cropResources change containerResources with containerPolicies 137 func cropResources(podResources map[consts.PodContainerName]apis.ContainerResources, 138 containerResources map[consts.ContainerName]apis.ContainerResources, containerPolicies map[string]apis.ContainerResourcePolicy, 139 ) error { 140 for key, containerResource := range podResources { 141 _, containerName, err := native.ParsePodContainerName(key) 142 if err != nil { 143 return err 144 } 145 146 if policy, ok := containerPolicies[containerName]; ok { 147 podResources[key] = cropResourcesWithPolicies(containerResource, policy) 148 } 149 } 150 151 for key, containerResource := range containerResources { 152 containerName := native.ParseContainerName(key) 153 if policy, ok := containerPolicies[containerName]; ok { 154 containerResources[key] = cropResourcesWithPolicies(containerResource, policy) 155 } 156 } 157 return nil 158 } 159 160 // cropResourcesWithPolicies check policy.ControllerValues and crop final recommendation to obey resource policy 161 func cropResourcesWithPolicies(resource apis.ContainerResources, policy apis.ContainerResourcePolicy) apis.ContainerResources { 162 cropRequests := func() { 163 if resource.Requests != nil { 164 resource.Requests.LowerBound = policy.MinAllowed.DeepCopy() 165 resource.Requests.UpperBound = policy.MaxAllowed.DeepCopy() 166 resource.Requests.Target = cropResourcesWithBounds(resource.Requests.UncappedTarget, 167 policy.MinAllowed, policy.MaxAllowed, policy.ControlledResources) 168 } 169 } 170 171 cropLimits := func() { 172 if resource.Limits != nil { 173 resource.Limits.LowerBound = policy.MinAllowed.DeepCopy() 174 resource.Limits.UpperBound = policy.MaxAllowed.DeepCopy() 175 resource.Limits.Target = cropResourcesWithBounds(resource.Limits.UncappedTarget, 176 policy.MinAllowed, policy.MaxAllowed, policy.ControlledResources) 177 } 178 } 179 180 resource = *resource.DeepCopy() 181 switch policy.ControlledValues { 182 case apis.ContainerControlledValuesRequestsOnly: 183 cropRequests() 184 resource.Limits = nil 185 case apis.ContainerControlledValuesLimitsOnly: 186 cropLimits() 187 resource.Requests = nil 188 case apis.ContainerControlledValuesRequestsAndLimits: 189 cropRequests() 190 cropLimits() 191 } 192 return resource 193 } 194 195 // cropResourcesWithBounds limit uncappedValue between lowBound and upBound and 196 // filter out resources which aren't in controlledResource 197 func cropResourcesWithBounds(uncappedValue core.ResourceList, lowBound core.ResourceList, upBound core.ResourceList, 198 controlledResource []core.ResourceName, 199 ) core.ResourceList { 200 finalValue := make(core.ResourceList) 201 202 for _, resourceName := range controlledResource { 203 resourceValue, ok := uncappedValue[resourceName] 204 if !ok { 205 continue 206 } 207 208 finalValue[resourceName] = resourceValue 209 210 if minValue, ok := lowBound[resourceName]; ok { 211 if resourceValue.Cmp(minValue) < 0 { 212 finalValue[resourceName] = minValue.DeepCopy() 213 } 214 } 215 216 if maxValue, ok := upBound[resourceName]; ok { 217 if resourceValue.Cmp(maxValue) > 0 { 218 finalValue[resourceName] = maxValue.DeepCopy() 219 } 220 } 221 } 222 223 return finalValue 224 } 225 226 // GetVPAResourceStatusWithRecommendation updates resource recommendation results from vpaRec to vpa 227 func GetVPAResourceStatusWithRecommendation(vpa *apis.KatalystVerticalPodAutoscaler, recPodResources []apis.RecommendedPodResources, 228 recContainerResources []apis.RecommendedContainerResources, 229 ) ([]apis.PodResources, []apis.ContainerResources, error) { 230 containerPolicies, err := katalystutil.GenerateVPAPolicyMap(vpa) 231 if err != nil { 232 return nil, nil, err 233 } 234 235 vpaPodResources, vpaContainerResources, err := katalystutil.GenerateVPAResourceMap(vpa) 236 if err != nil { 237 return nil, nil, err 238 } 239 240 podResources := make(map[consts.PodContainerName]apis.ContainerResources) 241 containerResources := make(map[consts.ContainerName]apis.ContainerResources) 242 243 for _, podRec := range recPodResources { 244 if podRec.PodName == nil { 245 klog.Errorf("recommended pod resource's podName can't be nil") 246 continue 247 } 248 249 for _, containerRec := range podRec.ContainerRecommendations { 250 if containerRec.ContainerName == nil { 251 klog.Errorf("recommended pod resource's containerName can't be nil") 252 continue 253 } 254 255 key := native.GeneratePodContainerName(*podRec.PodName, *containerRec.ContainerName) 256 if _, ok := podResources[key]; ok { 257 klog.Errorf("recommended pod %s already exists", key) 258 continue 259 } 260 261 podResources[key] = mergeResourceAndRecommendation(apis.ContainerResources{ 262 ContainerName: containerRec.ContainerName, 263 }, containerRec) 264 265 // if vpa status already had current pod resources, then merge it 266 if res, ok := vpaPodResources[key]; ok { 267 podResources[key] = mergeResourceAndCurrent(podResources[key], res) 268 } 269 } 270 } 271 272 for _, containerRec := range recContainerResources { 273 if containerRec.ContainerName == nil { 274 klog.Errorf("recommended container resource's containerName can't be nil") 275 continue 276 } 277 278 key := native.GenerateContainerName(*containerRec.ContainerName) 279 if _, ok := containerResources[key]; ok { 280 klog.Errorf("recommended container %s already exists", key) 281 continue 282 } 283 284 containerResources[key] = mergeResourceAndRecommendation(apis.ContainerResources{ 285 ContainerName: containerRec.ContainerName, 286 }, containerRec) 287 288 // if vpa status already had current container resources, then merge it 289 if res, ok := vpaContainerResources[key]; ok { 290 containerResources[key] = mergeResourceAndCurrent(containerResources[key], res) 291 } 292 } 293 294 // crop resources limit resource recommendation with boundaries 295 if err := cropResources(podResources, containerResources, containerPolicies); err != nil { 296 return nil, nil, fmt.Errorf("failed to set container resource by policies: %v", err) 297 } 298 299 podResourcesMap := make(map[string]*apis.PodResources) 300 for key, containerResource := range podResources { 301 podName, _, err := native.ParsePodContainerName(key) 302 if err != nil { 303 return nil, nil, err 304 } 305 306 if _, ok := podResourcesMap[podName]; !ok { 307 podResourcesMap[podName] = &apis.PodResources{ 308 PodName: &podName, 309 ContainerResources: make([]apis.ContainerResources, 0), 310 } 311 } 312 podResourcesMap[podName].ContainerResources = append(podResourcesMap[podName].ContainerResources, containerResource) 313 } 314 315 var ( 316 finalPodResources = make([]apis.PodResources, 0, len(podResourcesMap)) 317 finalContainerResources = make([]apis.ContainerResources, 0, len(containerResources)) 318 ) 319 for _, podResource := range podResourcesMap { 320 finalPodResources = append(finalPodResources, *podResource) 321 } 322 for _, containerResource := range containerResources { 323 finalContainerResources = append(finalContainerResources, containerResource) 324 } 325 326 // sort both finalPodResources and finalContainerResources to make sure result stable 327 sortPodResources(finalPodResources) 328 sortContainerResources(finalContainerResources) 329 330 return finalPodResources, finalContainerResources, nil 331 } 332 333 // GetVPAResourceStatusWithCurrent updates pod current resource results from vpaRec to vpa 334 func GetVPAResourceStatusWithCurrent(vpa *apis.KatalystVerticalPodAutoscaler, pods []*core.Pod) ([]apis.PodResources, []apis.ContainerResources, error) { 335 podResources, containerResources, err := katalystutil.GenerateVPAResourceMap(vpa) 336 if err != nil { 337 return nil, nil, err 338 } 339 340 updateContainerResourcesCurrent := func(targetResourceNames map[consts.ContainerName][]core.ResourceName, 341 containerName consts.ContainerName, 342 target core.ResourceList, current *core.ResourceList, 343 specResource core.ResourceList, 344 ) { 345 // if pod apply strategy is 'Pod', we not need update container current, 346 // because each pod has different recommendation. 347 if vpa.Spec.UpdatePolicy.PodApplyStrategy == apis.PodApplyStrategyStrategyPod { 348 *current = nil 349 return 350 } 351 352 _, ok := targetResourceNames[containerName] 353 if !ok { 354 *current = target.DeepCopy() 355 resourceNames := make([]core.ResourceName, 0, len(target)) 356 for resourceName := range target { 357 resourceNames = append(resourceNames, resourceName) 358 } 359 sort.SliceStable(resourceNames, func(i, j int) bool { 360 return string(resourceNames[i]) < string(resourceNames[j]) 361 }) 362 targetResourceNames[containerName] = resourceNames 363 } 364 365 specCurrent := cropResourcesWithBounds(specResource, nil, nil, targetResourceNames[containerName]) 366 if !native.ResourcesEqual(target, specCurrent) && 367 (native.ResourcesEqual(target, *current) || 368 !native.ResourcesLess(*current, specCurrent, targetResourceNames[containerName])) { 369 *current = specCurrent 370 } 371 } 372 373 for containerName := range containerResources { 374 // get container resource current requests 375 if containerResources[containerName].Requests != nil { 376 targetResourceNames := make(map[consts.ContainerName][]core.ResourceName) 377 func(pods []*core.Pod) { 378 for _, pod := range pods { 379 for _, container := range pod.Spec.Containers { 380 if container.Name != string(containerName) { 381 continue 382 } 383 384 updateContainerResourcesCurrent(targetResourceNames, containerName, containerResources[containerName].Requests.Target, 385 &containerResources[containerName].Requests.Current, container.Resources.Requests) 386 } 387 } 388 }(pods) 389 } 390 391 // get container resource current limits 392 if containerResources[containerName].Limits != nil { 393 targetResourceNames := make(map[consts.ContainerName][]core.ResourceName) 394 func(pods []*core.Pod) { 395 for _, pod := range pods { 396 for _, container := range pod.Spec.Containers { 397 if container.Name != string(containerName) { 398 continue 399 } 400 401 updateContainerResourcesCurrent(targetResourceNames, containerName, containerResources[containerName].Limits.Target, 402 &containerResources[containerName].Limits.Current, container.Resources.Limits) 403 } 404 } 405 }(pods) 406 } 407 } 408 409 getPodContainerResourcesCurrent := func(target core.ResourceList, specResource core.ResourceList) core.ResourceList { 410 resourceNames := make([]core.ResourceName, 0, len(target)) 411 for resourceName := range target { 412 resourceNames = append(resourceNames, resourceName) 413 } 414 415 return cropResourcesWithBounds(specResource, nil, nil, resourceNames) 416 } 417 418 for podContainerName := range podResources { 419 podName, containerName, err := native.ParsePodContainerName(podContainerName) 420 if err != nil { 421 return nil, nil, err 422 } 423 424 // find pod matched pod & container current requests 425 if podResources[podContainerName].Requests != nil { 426 func(pods []*core.Pod) { 427 for _, pod := range pods { 428 if pod.Name != podName { 429 continue 430 } 431 432 for _, container := range pod.Spec.Containers { 433 if container.Name != containerName { 434 continue 435 } 436 437 podResources[podContainerName].Requests.Current = getPodContainerResourcesCurrent(podResources[podContainerName].Requests.Target, container.Resources.Requests) 438 return 439 } 440 } 441 }(pods) 442 } 443 444 // find pod matched pod & container current limits 445 if podResources[podContainerName].Limits != nil { 446 func(pods []*core.Pod) { 447 for _, pod := range pods { 448 if pod.Name != podName { 449 continue 450 } 451 452 for _, container := range pod.Spec.Containers { 453 if container.Name != containerName { 454 continue 455 } 456 457 podResources[podContainerName].Limits.Current = getPodContainerResourcesCurrent(podResources[podContainerName].Limits.Target, container.Resources.Limits) 458 return 459 } 460 } 461 }(pods) 462 } 463 } 464 465 podResourcesMap := make(map[string]*apis.PodResources) 466 for key, containerResource := range podResources { 467 podName, _, err := native.ParsePodContainerName(key) 468 if err != nil { 469 return nil, nil, err 470 } 471 472 if _, ok := podResourcesMap[podName]; !ok { 473 podResourcesMap[podName] = &apis.PodResources{ 474 PodName: &podName, 475 ContainerResources: make([]apis.ContainerResources, 0), 476 } 477 } 478 podResourcesMap[podName].ContainerResources = append(podResourcesMap[podName].ContainerResources, containerResource) 479 } 480 481 var ( 482 finalPodResources = make([]apis.PodResources, 0, len(podResourcesMap)) 483 finalContainerResources = make([]apis.ContainerResources, 0, len(containerResources)) 484 ) 485 for _, podResource := range podResourcesMap { 486 finalPodResources = append(finalPodResources, *podResource) 487 } 488 for _, containerResource := range containerResources { 489 finalContainerResources = append(finalContainerResources, containerResource) 490 } 491 492 // sort both finalPodResources and finalContainerResources to make sure result stable 493 sortPodResources(finalPodResources) 494 sortContainerResources(finalContainerResources) 495 496 return finalPodResources, finalContainerResources, nil 497 } 498 499 /* 500 helper functions to generate status for vpaRec 501 */ 502 503 // GetVPARecResourceStatus updates resource recommendation results from vpa status to vpaRec status 504 func GetVPARecResourceStatus(vpaPodResources []apis.PodResources, vpaContainerResources []apis.ContainerResources) ( 505 []apis.RecommendedPodResources, []apis.RecommendedContainerResources, error, 506 ) { 507 recPodResources := make(map[consts.PodContainerName]apis.RecommendedContainerResources) 508 for _, podResource := range vpaPodResources { 509 if podResource.PodName == nil { 510 continue 511 } 512 513 for _, containerResource := range podResource.ContainerResources { 514 if containerResource.ContainerName == nil { 515 klog.Errorf("vpa pod resource's podName can't be nil") 516 continue 517 } 518 519 key := native.GeneratePodContainerName(*podResource.PodName, *containerResource.ContainerName) 520 if _, ok := recPodResources[key]; ok { 521 klog.Errorf("vpa pod %s already exists", key) 522 continue 523 } 524 recPodResources[key] = katalystutil.ConvertVPAContainerResourceToRecommendedContainerResources(containerResource) 525 } 526 } 527 528 recContainerResources := make(map[consts.ContainerName]apis.RecommendedContainerResources) 529 for _, containerResource := range vpaContainerResources { 530 if containerResource.ContainerName == nil { 531 continue 532 } 533 534 key := native.GenerateContainerName(*containerResource.ContainerName) 535 if _, ok := recContainerResources[key]; ok { 536 klog.Errorf("vpa container %s already exists", key) 537 continue 538 } 539 recContainerResources[key] = katalystutil.ConvertVPAContainerResourceToRecommendedContainerResources(containerResource) 540 } 541 542 podResourcesMap := make(map[string]*apis.RecommendedPodResources) 543 for key, containerResource := range recPodResources { 544 podName, _, err := native.ParsePodContainerName(key) 545 if err != nil { 546 return nil, nil, err 547 } 548 549 if _, ok := podResourcesMap[podName]; !ok { 550 podResourcesMap[podName] = &apis.RecommendedPodResources{ 551 PodName: &podName, 552 ContainerRecommendations: make([]apis.RecommendedContainerResources, 0), 553 } 554 } 555 podResourcesMap[podName].ContainerRecommendations = append(podResourcesMap[podName].ContainerRecommendations, containerResource) 556 } 557 558 var ( 559 finalPodResources = make([]apis.RecommendedPodResources, 0, len(podResourcesMap)) 560 finalContainerResources = make([]apis.RecommendedContainerResources, 0, len(recContainerResources)) 561 ) 562 for _, podResource := range podResourcesMap { 563 finalPodResources = append(finalPodResources, *podResource) 564 } 565 for _, containerResource := range recContainerResources { 566 finalContainerResources = append(finalContainerResources, containerResource) 567 } 568 569 // sort both finalPodResources and finalContainerResources to make sure result stable 570 sortRecommendedPodResources(finalPodResources) 571 sortRecommendedContainerResources(finalContainerResources) 572 573 return finalPodResources, finalContainerResources, nil 574 }