github.com/kubewharf/katalyst-core@v0.5.3/pkg/controller/resource-recommend/processor/percentile/process_gc.go (about)

     1  /*
     2  Copyright 2022 The Katalyst Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package percentile
    18  
    19  import (
    20  	"context"
    21  	"runtime/debug"
    22  	"time"
    23  
    24  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    25  	"k8s.io/apimachinery/pkg/types"
    26  	"k8s.io/klog/v2"
    27  	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
    28  
    29  	"github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1"
    30  	"github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task"
    31  	"github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log"
    32  	processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor"
    33  )
    34  
    35  func (p *Processor) GarbageCollector(ctx context.Context) {
    36  	defer func() {
    37  		if r := recover(); r != nil {
    38  			errMsg := "garbage collect goroutine panic"
    39  			log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack()))
    40  			panic(errMsg)
    41  		}
    42  	}()
    43  
    44  	ticker := time.Tick(DefaultGarbageCollectInterval)
    45  	for range ticker {
    46  		if err := p.garbageCollect(NewContext()); err != nil {
    47  			log.ErrorS(ctx, err, "garbageCollect failed")
    48  		}
    49  	}
    50  }
    51  
    52  func (p *Processor) garbageCollect(ctx context.Context) error {
    53  	defer func() {
    54  		if r := recover(); r != nil {
    55  			errMsg := "garbage collect run panic"
    56  			log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack()))
    57  		}
    58  	}()
    59  
    60  	log.InfoS(ctx, "garbage collect start")
    61  
    62  	p.mutex.Lock()
    63  	defer p.mutex.Unlock()
    64  
    65  	// Tasks that have not been running for too long are considered invalid and will be cleared
    66  	p.AggregateTasks.Range(func(ki, vi any) bool {
    67  		taskID := ki.(processortypes.TaskID)
    68  		t, ok := vi.(*task.HistogramTask)
    69  		if !ok {
    70  			p.AggregateTasks.Delete(taskID)
    71  			return true
    72  		}
    73  		if t.IsTimeoutNotExecute() {
    74  			p.AggregateTasks.Delete(taskID)
    75  		}
    76  		return true
    77  	})
    78  
    79  	// List all ResourceRecommend CR up to now
    80  	resourceRecommendList := &v1alpha1.ResourceRecommendList{}
    81  	err := p.Client.List(ctx, resourceRecommendList, &k8sclient.ListOptions{Raw: &metav1.ListOptions{ResourceVersion: "0"}})
    82  	if err != nil {
    83  		log.ErrorS(ctx, err, "garbage collect list all ResourceRecommend failed")
    84  		return err
    85  	}
    86  
    87  	// p.ClearingNoAttributionTask(resourceRecommendList)
    88  	klog.InfoS("percentile processor garbage collect list ResourceRecommend",
    89  		"ResourceRecommend Count", len(resourceRecommendList.Items))
    90  	// Convert the ResourceRecommend list to map for quick check whether it exists
    91  	existResourceRecommends := make(map[types.NamespacedName]v1alpha1.CrossVersionObjectReference)
    92  	for _, existResourceRecommend := range resourceRecommendList.Items {
    93  		namespacedName := types.NamespacedName{
    94  			Name:      existResourceRecommend.Name,
    95  			Namespace: existResourceRecommend.Namespace,
    96  		}
    97  		existResourceRecommends[namespacedName] = existResourceRecommend.Spec.TargetRef
    98  	}
    99  
   100  	// Delete tasks belong to not exist ResourceRecommend or task not match the targetRef in the ResourceRecommend
   101  	for namespacedName, tasks := range p.ResourceRecommendTaskIDsMap {
   102  		if tasks == nil {
   103  			delete(p.ResourceRecommendTaskIDsMap, namespacedName)
   104  			continue
   105  		}
   106  		targetRef, isExist := existResourceRecommends[namespacedName]
   107  		for metrics, taskID := range *tasks {
   108  			if !isExist ||
   109  				targetRef.Kind != metrics.Kind ||
   110  				targetRef.Name != metrics.WorkloadName ||
   111  				targetRef.APIVersion != metrics.APIVersion {
   112  				p.AggregateTasks.Delete(taskID)
   113  				delete(*tasks, metrics)
   114  			}
   115  			//  timeout task in p.AggregateTasks will clean,
   116  			// If taskID does not exist in p.AggregateTasks，mean is the task is cleared, this needs to be deleted from tasks
   117  			if _, found := p.AggregateTasks.Load(taskID); !found {
   118  				delete(*tasks, metrics)
   119  			}
   120  		}
   121  		if !isExist {
   122  			delete(p.ResourceRecommendTaskIDsMap, namespacedName)
   123  		}
   124  	}
   125  
   126  	log.InfoS(ctx, "garbage collect end")
   127  	return nil
   128  }