github.com/kubewharf/katalyst-core@v0.5.3/pkg/controller/resource-recommend/processor/percentile/process_tasks.go (about) 1 /* 2 Copyright 2022 The Katalyst Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package percentile 18 19 import ( 20 "context" 21 "runtime/debug" 22 "sync" 23 24 "github.com/pkg/errors" 25 26 "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log" 27 processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" 28 ) 29 30 func (p *Processor) ProcessTasks(ctx context.Context) { 31 defer func() { 32 if r := recover(); r != nil { 33 errMsg := "process tasks goroutine run panic" 34 log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack())) 35 panic(errMsg) 36 } 37 }() 38 39 wg := &sync.WaitGroup{} 40 wg.Add(DefaultConcurrentTaskNum) 41 for i := 0; i < DefaultConcurrentTaskNum; i++ { 42 go func() { 43 defer wg.Done() 44 // Run a worker thread that just dequeues items, processes them, and marks them done. 45 for p.processTask(NewContext()) { 46 } 47 }() 48 } 49 50 log.InfoS(ctx, "process workers running") 51 wg.Wait() 52 log.InfoS(ctx, "all process workers finished") 53 } 54 55 func (p *Processor) processTask(ctx context.Context) bool { 56 obj, shutdown := p.TaskQueue.Get() 57 log.InfoS(ctx, "process task dequeue", "obj", obj, "shutdown", shutdown) 58 if shutdown { 59 err := errors.New("task queue is shutdown") 60 log.ErrorS(ctx, err, "process task failed") 61 // Stop working 62 return false 63 } 64 65 defer p.TaskQueue.Done(obj) 66 67 p.taskHandler(ctx, obj) 68 return true 69 } 70 71 func (p *Processor) taskHandler(ctx context.Context, obj interface{}) { 72 log.InfoS(ctx, "task handler begin") 73 taskID, ok := obj.(processortypes.TaskID) 74 if !ok { 75 err := errors.New("task key type error, drop") 76 log.ErrorS(ctx, err, "task err", "obj", obj) 77 p.TaskQueue.Forget(obj) 78 return 79 } 80 ctx = log.SetKeysAndValues(ctx, "taskID", taskID) 81 82 task, err := p.getTaskForTaskID(taskID) 83 if err != nil { 84 log.ErrorS(ctx, err, "task not found, drop it") 85 p.TaskQueue.Forget(obj) 86 return 87 } 88 89 if nextRunInterval, err := task.Run(ctx, p.DatasourceProxy); err != nil { 90 log.ErrorS(ctx, err, "task handler err") 91 p.TaskQueue.AddRateLimited(taskID) 92 } else { 93 log.InfoS(ctx, "task handler finished") 94 p.TaskQueue.Forget(obj) 95 if nextRunInterval > 0 { 96 p.TaskQueue.AddAfter(taskID, nextRunInterval) 97 } 98 } 99 }