github.com/kubewharf/katalyst-core@v0.5.3/pkg/controller/resource-recommend/processor/percentile/process_tasks.go (about)

     1  /*
     2  Copyright 2022 The Katalyst Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package percentile
    18  
    19  import (
    20  	"context"
    21  	"runtime/debug"
    22  	"sync"
    23  
    24  	"github.com/pkg/errors"
    25  
    26  	"github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log"
    27  	processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor"
    28  )
    29  
    30  func (p *Processor) ProcessTasks(ctx context.Context) {
    31  	defer func() {
    32  		if r := recover(); r != nil {
    33  			errMsg := "process tasks goroutine run panic"
    34  			log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack()))
    35  			panic(errMsg)
    36  		}
    37  	}()
    38  
    39  	wg := &sync.WaitGroup{}
    40  	wg.Add(DefaultConcurrentTaskNum)
    41  	for i := 0; i < DefaultConcurrentTaskNum; i++ {
    42  		go func() {
    43  			defer wg.Done()
    44  			// Run a worker thread that just dequeues items, processes them, and marks them done.
    45  			for p.processTask(NewContext()) {
    46  			}
    47  		}()
    48  	}
    49  
    50  	log.InfoS(ctx, "process workers running")
    51  	wg.Wait()
    52  	log.InfoS(ctx, "all process workers finished")
    53  }
    54  
    55  func (p *Processor) processTask(ctx context.Context) bool {
    56  	obj, shutdown := p.TaskQueue.Get()
    57  	log.InfoS(ctx, "process task dequeue", "obj", obj, "shutdown", shutdown)
    58  	if shutdown {
    59  		err := errors.New("task queue is shutdown")
    60  		log.ErrorS(ctx, err, "process task failed")
    61  		// Stop working
    62  		return false
    63  	}
    64  
    65  	defer p.TaskQueue.Done(obj)
    66  
    67  	p.taskHandler(ctx, obj)
    68  	return true
    69  }
    70  
    71  func (p *Processor) taskHandler(ctx context.Context, obj interface{}) {
    72  	log.InfoS(ctx, "task handler begin")
    73  	taskID, ok := obj.(processortypes.TaskID)
    74  	if !ok {
    75  		err := errors.New("task key type error, drop")
    76  		log.ErrorS(ctx, err, "task err", "obj", obj)
    77  		p.TaskQueue.Forget(obj)
    78  		return
    79  	}
    80  	ctx = log.SetKeysAndValues(ctx, "taskID", taskID)
    81  
    82  	task, err := p.getTaskForTaskID(taskID)
    83  	if err != nil {
    84  		log.ErrorS(ctx, err, "task not found, drop it")
    85  		p.TaskQueue.Forget(obj)
    86  		return
    87  	}
    88  
    89  	if nextRunInterval, err := task.Run(ctx, p.DatasourceProxy); err != nil {
    90  		log.ErrorS(ctx, err, "task handler err")
    91  		p.TaskQueue.AddRateLimited(taskID)
    92  	} else {
    93  		log.InfoS(ctx, "task handler finished")
    94  		p.TaskQueue.Forget(obj)
    95  		if nextRunInterval > 0 {
    96  			p.TaskQueue.AddAfter(taskID, nextRunInterval)
    97  		}
    98  	}
    99  }