volcano.sh/volcano@v1.9.0/pkg/scheduler/plugins/overcommit/overcommit.go (about)

     1  /*
     2  Copyright 2021 The Volcano Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package overcommit
    18  
    19  import (
    20  	v1 "k8s.io/api/core/v1"
    21  	"k8s.io/klog/v2"
    22  
    23  	"volcano.sh/apis/pkg/apis/scheduling"
    24  	"volcano.sh/volcano/pkg/scheduler/api"
    25  	"volcano.sh/volcano/pkg/scheduler/framework"
    26  	"volcano.sh/volcano/pkg/scheduler/plugins/util"
    27  )
    28  
    29  const (
    30  	// PluginName is name of plugin
    31  	PluginName = "overcommit"
    32  	// overCommitFactor is resource overCommit factor for enqueue action
    33  	// It determines the number of `pending` pods that the scheduler will tolerate
    34  	// when the resources of the cluster is insufficient
    35  	overCommitFactor = "overcommit-factor"
    36  	// defaultOverCommitFactor defines the default overCommit resource factor for enqueue action
    37  	defaultOverCommitFactor = 1.2
    38  )
    39  
    40  type overcommitPlugin struct {
    41  	// Arguments given for the plugin
    42  	pluginArguments  framework.Arguments
    43  	totalResource    *api.Resource
    44  	idleResource     *api.Resource
    45  	inqueueResource  *api.Resource
    46  	overCommitFactor float64
    47  }
    48  
    49  // New function returns overcommit plugin object
    50  func New(arguments framework.Arguments) framework.Plugin {
    51  	return &overcommitPlugin{
    52  		pluginArguments:  arguments,
    53  		totalResource:    api.EmptyResource(),
    54  		idleResource:     api.EmptyResource(),
    55  		inqueueResource:  api.EmptyResource(),
    56  		overCommitFactor: defaultOverCommitFactor,
    57  	}
    58  }
    59  
    60  func (op *overcommitPlugin) Name() string {
    61  	return PluginName
    62  }
    63  
    64  /*
    65  User should give overcommit-factor through overcommit plugin arguments as format below:
    66  
    67  actions: "enqueue, allocate, backfill"
    68  tiers:
    69  - plugins:
    70    - name: overcommit
    71      arguments:
    72      overcommit-factor: 1.0
    73  */
    74  func (op *overcommitPlugin) OnSessionOpen(ssn *framework.Session) {
    75  	klog.V(5).Infof("Enter overcommit plugin ...")
    76  	defer klog.V(5).Infof("Leaving overcommit plugin.")
    77  
    78  	op.pluginArguments.GetFloat64(&op.overCommitFactor, overCommitFactor)
    79  	if op.overCommitFactor < 1.0 {
    80  		klog.Warningf("Invalid input %f for overcommit-factor, reason: overcommit-factor cannot be less than 1,"+
    81  			" using default value: %f.", op.overCommitFactor, defaultOverCommitFactor)
    82  		op.overCommitFactor = defaultOverCommitFactor
    83  	}
    84  
    85  	op.totalResource.Add(ssn.TotalResource)
    86  	// calculate idle resources of total cluster, overcommit resources included
    87  	used := api.EmptyResource()
    88  	for _, node := range ssn.Nodes {
    89  		used.Add(node.Used)
    90  	}
    91  	op.idleResource = op.totalResource.Clone().Multi(op.overCommitFactor).SubWithoutAssert(used)
    92  
    93  	for _, job := range ssn.Jobs {
    94  		// calculate inqueue job resources
    95  		if job.PodGroup.Status.Phase == scheduling.PodGroupInqueue && job.PodGroup.Spec.MinResources != nil {
    96  			op.inqueueResource.Add(api.NewResource(*job.PodGroup.Spec.MinResources))
    97  			continue
    98  		}
    99  		// calculate inqueue resource for running jobs
   100  		// the judgement 'job.PodGroup.Status.Running >= job.PodGroup.Spec.MinMember' will work on cases such as the following condition:
   101  		// Considering a Spark job is completed(driver pod is completed) while the podgroup keeps running, the allocated resource will be reserved again if without the judgement.
   102  		if job.PodGroup.Status.Phase == scheduling.PodGroupRunning &&
   103  			job.PodGroup.Spec.MinResources != nil &&
   104  			int32(util.CalculateAllocatedTaskNum(job)) >= job.PodGroup.Spec.MinMember {
   105  			inqueued := util.GetInqueueResource(job, job.Allocated)
   106  			op.inqueueResource.Add(inqueued)
   107  		}
   108  	}
   109  
   110  	ssn.AddJobEnqueueableFn(op.Name(), func(obj interface{}) int {
   111  		job := obj.(*api.JobInfo)
   112  		idle := op.idleResource
   113  		inqueue := api.EmptyResource()
   114  		inqueue.Add(op.inqueueResource)
   115  		if job.PodGroup.Spec.MinResources == nil {
   116  			klog.V(4).Infof("Job <%s/%s> is bestEffort, permit to be inqueue.", job.Namespace, job.Name)
   117  			return util.Permit
   118  		}
   119  
   120  		//TODO: if allow 1 more job to be inqueue beyond overcommit-factor, large job may be inqueue and create pods
   121  		jobMinReq := api.NewResource(*job.PodGroup.Spec.MinResources)
   122  		if inqueue.Add(jobMinReq).LessEqual(idle, api.Zero) {
   123  			klog.V(4).Infof("Sufficient resources, permit job <%s/%s> to be inqueue", job.Namespace, job.Name)
   124  			return util.Permit
   125  		}
   126  		klog.V(4).Infof("Resource in cluster is overused, reject job <%s/%s> to be inqueue",
   127  			job.Namespace, job.Name)
   128  		ssn.RecordPodGroupEvent(job.PodGroup, v1.EventTypeNormal, string(scheduling.PodGroupUnschedulableType), "resource in cluster is overused")
   129  		return util.Reject
   130  	})
   131  
   132  	ssn.AddJobEnqueuedFn(op.Name(), func(obj interface{}) {
   133  		job := obj.(*api.JobInfo)
   134  		if job.PodGroup.Spec.MinResources == nil {
   135  			return
   136  		}
   137  		jobMinReq := api.NewResource(*job.PodGroup.Spec.MinResources)
   138  		op.inqueueResource.Add(jobMinReq)
   139  	})
   140  }
   141  
   142  func (op *overcommitPlugin) OnSessionClose(ssn *framework.Session) {
   143  	op.totalResource = nil
   144  	op.idleResource = nil
   145  	op.inqueueResource = nil
   146  }