volcano.sh/volcano@v1.9.0/cmd/controller-manager/app/server.go (about)

     1  /*
     2  Copyright 2017 The Volcano Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package app
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"os"
    23  	"time"
    24  
    25  	v1 "k8s.io/api/core/v1"
    26  	"k8s.io/apimachinery/pkg/util/uuid"
    27  	"k8s.io/client-go/informers"
    28  	kubeclientset "k8s.io/client-go/kubernetes"
    29  	"k8s.io/client-go/kubernetes/scheme"
    30  	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
    31  	"k8s.io/client-go/rest"
    32  	"k8s.io/client-go/tools/leaderelection"
    33  	"k8s.io/client-go/tools/leaderelection/resourcelock"
    34  	"k8s.io/client-go/tools/record"
    35  	"k8s.io/klog/v2"
    36  
    37  	"volcano.sh/apis/pkg/apis/helpers"
    38  	vcclientset "volcano.sh/apis/pkg/client/clientset/versioned"
    39  	"volcano.sh/volcano/cmd/controller-manager/app/options"
    40  	"volcano.sh/volcano/pkg/controllers/framework"
    41  	"volcano.sh/volcano/pkg/kube"
    42  	"volcano.sh/volcano/pkg/signals"
    43  )
    44  
    45  const (
    46  	leaseDuration = 15 * time.Second
    47  	renewDeadline = 10 * time.Second
    48  	retryPeriod   = 5 * time.Second
    49  )
    50  
    51  // Run the controller.
    52  func Run(opt *options.ServerOption) error {
    53  	config, err := kube.BuildConfig(opt.KubeClientOptions)
    54  	if err != nil {
    55  		return err
    56  	}
    57  
    58  	if opt.EnableHealthz {
    59  		if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-controller", opt.CaCertData, opt.CertData, opt.KeyData); err != nil {
    60  			return err
    61  		}
    62  	}
    63  
    64  	run := startControllers(config, opt)
    65  
    66  	ctx := signals.SetupSignalContext()
    67  
    68  	if !opt.EnableLeaderElection {
    69  		run(ctx)
    70  		return fmt.Errorf("finished without leader elect")
    71  	}
    72  
    73  	leaderElectionClient, err := kubeclientset.NewForConfig(rest.AddUserAgent(config, "leader-election"))
    74  	if err != nil {
    75  		return err
    76  	}
    77  
    78  	// Prepare event clients.
    79  	broadcaster := record.NewBroadcaster()
    80  	broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: leaderElectionClient.CoreV1().Events(opt.LockObjectNamespace)})
    81  	eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "vc-controller-manager"})
    82  
    83  	hostname, err := os.Hostname()
    84  	if err != nil {
    85  		return fmt.Errorf("unable to get hostname: %v", err)
    86  	}
    87  	// add a uniquifier so that two processes on the same host don't accidentally both become active
    88  	id := hostname + "_" + string(uuid.NewUUID())
    89  
    90  	rl, err := resourcelock.New(resourcelock.LeasesResourceLock,
    91  		opt.LockObjectNamespace,
    92  		"vc-controller-manager",
    93  		leaderElectionClient.CoreV1(),
    94  		leaderElectionClient.CoordinationV1(),
    95  		resourcelock.ResourceLockConfig{
    96  			Identity:      id,
    97  			EventRecorder: eventRecorder,
    98  		})
    99  	if err != nil {
   100  		return fmt.Errorf("couldn't create resource lock: %v", err)
   101  	}
   102  
   103  	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
   104  		Lock:          rl,
   105  		LeaseDuration: leaseDuration,
   106  		RenewDeadline: renewDeadline,
   107  		RetryPeriod:   retryPeriod,
   108  		Callbacks: leaderelection.LeaderCallbacks{
   109  			OnStartedLeading: run,
   110  			OnStoppedLeading: func() {
   111  				klog.Fatalf("leaderelection lost")
   112  			},
   113  		},
   114  	})
   115  	return fmt.Errorf("lost lease")
   116  }
   117  
   118  func startControllers(config *rest.Config, opt *options.ServerOption) func(ctx context.Context) {
   119  	controllerOpt := &framework.ControllerOption{}
   120  
   121  	controllerOpt.SchedulerNames = opt.SchedulerNames
   122  	controllerOpt.WorkerNum = opt.WorkerThreads
   123  	controllerOpt.MaxRequeueNum = opt.MaxRequeueNum
   124  
   125  	// TODO: add user agent for different controllers
   126  	controllerOpt.KubeClient = kubeclientset.NewForConfigOrDie(config)
   127  	controllerOpt.VolcanoClient = vcclientset.NewForConfigOrDie(config)
   128  	controllerOpt.SharedInformerFactory = informers.NewSharedInformerFactory(controllerOpt.KubeClient, 0)
   129  	controllerOpt.InheritOwnerAnnotations = opt.InheritOwnerAnnotations
   130  	controllerOpt.WorkerThreadsForPG = opt.WorkerThreadsForPG
   131  
   132  	return func(ctx context.Context) {
   133  		framework.ForeachController(func(c framework.Controller) {
   134  			if err := c.Initialize(controllerOpt); err != nil {
   135  				klog.Errorf("Failed to initialize controller <%s>: %v", c.Name(), err)
   136  				return
   137  			}
   138  
   139  			go c.Run(ctx.Done())
   140  		})
   141  
   142  		<-ctx.Done()
   143  	}
   144  }