k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/cmd/kube-scheduler/app/options/options.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package options 18 19 import ( 20 "context" 21 "fmt" 22 "net" 23 "os" 24 "time" 25 26 corev1 "k8s.io/api/core/v1" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 "k8s.io/apimachinery/pkg/util/uuid" 29 apiserveroptions "k8s.io/apiserver/pkg/server/options" 30 utilfeature "k8s.io/apiserver/pkg/util/feature" 31 "k8s.io/client-go/dynamic" 32 "k8s.io/client-go/dynamic/dynamicinformer" 33 clientset "k8s.io/client-go/kubernetes" 34 restclient "k8s.io/client-go/rest" 35 "k8s.io/client-go/tools/clientcmd" 36 "k8s.io/client-go/tools/events" 37 "k8s.io/client-go/tools/leaderelection" 38 "k8s.io/client-go/tools/leaderelection/resourcelock" 39 "k8s.io/client-go/tools/record" 40 cliflag "k8s.io/component-base/cli/flag" 41 componentbaseconfig "k8s.io/component-base/config" 42 "k8s.io/component-base/config/options" 43 "k8s.io/component-base/logs" 44 logsapi "k8s.io/component-base/logs/api/v1" 45 "k8s.io/component-base/metrics" 46 "k8s.io/klog/v2" 47 schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" 48 "k8s.io/kubernetes/pkg/scheduler" 49 kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" 50 "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" 51 netutils "k8s.io/utils/net" 52 ) 53 54 // Options has all the params needed to run a Scheduler 55 type Options struct { 56 // The default values. 57 ComponentConfig *kubeschedulerconfig.KubeSchedulerConfiguration 58 59 SecureServing *apiserveroptions.SecureServingOptionsWithLoopback 60 Authentication *apiserveroptions.DelegatingAuthenticationOptions 61 Authorization *apiserveroptions.DelegatingAuthorizationOptions 62 Metrics *metrics.Options 63 Logs *logs.Options 64 Deprecated *DeprecatedOptions 65 LeaderElection *componentbaseconfig.LeaderElectionConfiguration 66 67 // ConfigFile is the location of the scheduler server's configuration file. 68 ConfigFile string 69 70 // WriteConfigTo is the path where the default configuration will be written. 71 WriteConfigTo string 72 73 Master string 74 75 // Flags hold the parsed CLI flags. 76 Flags *cliflag.NamedFlagSets 77 } 78 79 // NewOptions returns default scheduler app options. 80 func NewOptions() *Options { 81 o := &Options{ 82 SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), 83 Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), 84 Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), 85 Deprecated: &DeprecatedOptions{ 86 PodMaxInUnschedulablePodsDuration: 5 * time.Minute, 87 }, 88 LeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ 89 LeaderElect: true, 90 LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, 91 RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, 92 RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, 93 ResourceLock: "leases", 94 ResourceName: "kube-scheduler", 95 ResourceNamespace: "kube-system", 96 }, 97 Metrics: metrics.NewOptions(), 98 Logs: logs.NewOptions(), 99 } 100 101 o.Authentication.TolerateInClusterLookupFailure = true 102 o.Authentication.RemoteKubeConfigFileOptional = true 103 o.Authorization.RemoteKubeConfigFileOptional = true 104 105 // Set the PairName but leave certificate directory blank to generate in-memory by default 106 o.SecureServing.ServerCert.CertDirectory = "" 107 o.SecureServing.ServerCert.PairName = "kube-scheduler" 108 o.SecureServing.BindPort = kubeschedulerconfig.DefaultKubeSchedulerPort 109 110 o.initFlags() 111 112 return o 113 } 114 115 // ApplyDeprecated obtains the deprecated CLI args and set them to `o.ComponentConfig` if specified. 116 func (o *Options) ApplyDeprecated() { 117 if o.Flags == nil { 118 return 119 } 120 // Obtain deprecated CLI args. Set them to cfg if specified in command line. 121 deprecated := o.Flags.FlagSet("deprecated") 122 if deprecated.Changed("profiling") { 123 o.ComponentConfig.EnableProfiling = o.Deprecated.EnableProfiling 124 } 125 if deprecated.Changed("contention-profiling") { 126 o.ComponentConfig.EnableContentionProfiling = o.Deprecated.EnableContentionProfiling 127 } 128 if deprecated.Changed("kubeconfig") { 129 o.ComponentConfig.ClientConnection.Kubeconfig = o.Deprecated.Kubeconfig 130 } 131 if deprecated.Changed("kube-api-content-type") { 132 o.ComponentConfig.ClientConnection.ContentType = o.Deprecated.ContentType 133 } 134 if deprecated.Changed("kube-api-qps") { 135 o.ComponentConfig.ClientConnection.QPS = o.Deprecated.QPS 136 } 137 if deprecated.Changed("kube-api-burst") { 138 o.ComponentConfig.ClientConnection.Burst = o.Deprecated.Burst 139 } 140 } 141 142 // ApplyLeaderElectionTo obtains the CLI args related with leaderelection, and override the values in `cfg`. 143 // Then the `cfg` object is injected into the `options` object. 144 func (o *Options) ApplyLeaderElectionTo(cfg *kubeschedulerconfig.KubeSchedulerConfiguration) { 145 if o.Flags == nil { 146 return 147 } 148 // Obtain CLI args related with leaderelection. Set them to `cfg` if specified in command line. 149 leaderelection := o.Flags.FlagSet("leader election") 150 if leaderelection.Changed("leader-elect") { 151 cfg.LeaderElection.LeaderElect = o.LeaderElection.LeaderElect 152 } 153 if leaderelection.Changed("leader-elect-lease-duration") { 154 cfg.LeaderElection.LeaseDuration = o.LeaderElection.LeaseDuration 155 } 156 if leaderelection.Changed("leader-elect-renew-deadline") { 157 cfg.LeaderElection.RenewDeadline = o.LeaderElection.RenewDeadline 158 } 159 if leaderelection.Changed("leader-elect-retry-period") { 160 cfg.LeaderElection.RetryPeriod = o.LeaderElection.RetryPeriod 161 } 162 if leaderelection.Changed("leader-elect-resource-lock") { 163 cfg.LeaderElection.ResourceLock = o.LeaderElection.ResourceLock 164 } 165 if leaderelection.Changed("leader-elect-resource-name") { 166 cfg.LeaderElection.ResourceName = o.LeaderElection.ResourceName 167 } 168 if leaderelection.Changed("leader-elect-resource-namespace") { 169 cfg.LeaderElection.ResourceNamespace = o.LeaderElection.ResourceNamespace 170 } 171 172 o.ComponentConfig = cfg 173 } 174 175 // initFlags initializes flags by section name. 176 func (o *Options) initFlags() { 177 if o.Flags != nil { 178 return 179 } 180 181 nfs := cliflag.NamedFlagSets{} 182 fs := nfs.FlagSet("misc") 183 fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.") 184 fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the configuration values to this file and exit.") 185 fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") 186 187 o.SecureServing.AddFlags(nfs.FlagSet("secure serving")) 188 o.Authentication.AddFlags(nfs.FlagSet("authentication")) 189 o.Authorization.AddFlags(nfs.FlagSet("authorization")) 190 o.Deprecated.AddFlags(nfs.FlagSet("deprecated")) 191 options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election")) 192 utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate")) 193 o.Metrics.AddFlags(nfs.FlagSet("metrics")) 194 logsapi.AddFlags(o.Logs, nfs.FlagSet("logs")) 195 196 o.Flags = &nfs 197 } 198 199 // ApplyTo applies the scheduler options to the given scheduler app configuration. 200 func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) error { 201 if len(o.ConfigFile) == 0 { 202 // If the --config arg is not specified, honor the deprecated as well as leader election CLI args. 203 o.ApplyDeprecated() 204 o.ApplyLeaderElectionTo(o.ComponentConfig) 205 c.ComponentConfig = *o.ComponentConfig 206 } else { 207 cfg, err := LoadConfigFromFile(logger, o.ConfigFile) 208 if err != nil { 209 return err 210 } 211 // If the --config arg is specified, honor the leader election CLI args only. 212 o.ApplyLeaderElectionTo(cfg) 213 214 if err := validation.ValidateKubeSchedulerConfiguration(cfg); err != nil { 215 return err 216 } 217 218 c.ComponentConfig = *cfg 219 } 220 221 // Build kubeconfig first to so that if it fails, it doesn't cause leaking 222 // goroutines (started from initializing secure serving - which underneath 223 // creates a queue which in its constructor starts a goroutine). 224 kubeConfig, err := createKubeConfig(c.ComponentConfig.ClientConnection, o.Master) 225 if err != nil { 226 return err 227 } 228 c.KubeConfig = kubeConfig 229 230 if err := o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { 231 return err 232 } 233 if o.SecureServing != nil && (o.SecureServing.BindPort != 0 || o.SecureServing.Listener != nil) { 234 if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { 235 return err 236 } 237 if err := o.Authorization.ApplyTo(&c.Authorization); err != nil { 238 return err 239 } 240 } 241 o.Metrics.Apply() 242 243 // Apply value independently instead of using ApplyDeprecated() because it can't be configured via ComponentConfig. 244 if o.Deprecated != nil { 245 c.PodMaxInUnschedulablePodsDuration = o.Deprecated.PodMaxInUnschedulablePodsDuration 246 } 247 248 return nil 249 } 250 251 // Validate validates all the required options. 252 func (o *Options) Validate() []error { 253 var errs []error 254 255 if err := validation.ValidateKubeSchedulerConfiguration(o.ComponentConfig); err != nil { 256 errs = append(errs, err.Errors()...) 257 } 258 errs = append(errs, o.SecureServing.Validate()...) 259 errs = append(errs, o.Authentication.Validate()...) 260 errs = append(errs, o.Authorization.Validate()...) 261 errs = append(errs, o.Metrics.Validate()...) 262 263 return errs 264 } 265 266 // Config return a scheduler config object 267 func (o *Options) Config(ctx context.Context) (*schedulerappconfig.Config, error) { 268 logger := klog.FromContext(ctx) 269 if o.SecureServing != nil { 270 if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{netutils.ParseIPSloppy("127.0.0.1")}); err != nil { 271 return nil, fmt.Errorf("error creating self-signed certificates: %v", err) 272 } 273 } 274 275 c := &schedulerappconfig.Config{} 276 if err := o.ApplyTo(logger, c); err != nil { 277 return nil, err 278 } 279 280 // Prepare kube clients. 281 client, eventClient, err := createClients(c.KubeConfig) 282 if err != nil { 283 return nil, err 284 } 285 286 c.EventBroadcaster = events.NewEventBroadcasterAdapterWithContext(ctx, eventClient) 287 288 // Set up leader election if enabled. 289 var leaderElectionConfig *leaderelection.LeaderElectionConfig 290 if c.ComponentConfig.LeaderElection.LeaderElect { 291 // Use the scheduler name in the first profile to record leader election. 292 schedulerName := corev1.DefaultSchedulerName 293 if len(c.ComponentConfig.Profiles) != 0 { 294 schedulerName = c.ComponentConfig.Profiles[0].SchedulerName 295 } 296 coreRecorder := c.EventBroadcaster.DeprecatedNewLegacyRecorder(schedulerName) 297 leaderElectionConfig, err = makeLeaderElectionConfig(c.ComponentConfig.LeaderElection, c.KubeConfig, coreRecorder) 298 if err != nil { 299 return nil, err 300 } 301 } 302 303 c.Client = client 304 c.InformerFactory = scheduler.NewInformerFactory(client, 0) 305 dynClient := dynamic.NewForConfigOrDie(c.KubeConfig) 306 c.DynInformerFactory = dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynClient, 0, corev1.NamespaceAll, nil) 307 c.LeaderElection = leaderElectionConfig 308 309 return c, nil 310 } 311 312 // makeLeaderElectionConfig builds a leader election configuration. It will 313 // create a new resource lock associated with the configuration. 314 func makeLeaderElectionConfig(config componentbaseconfig.LeaderElectionConfiguration, kubeConfig *restclient.Config, recorder record.EventRecorder) (*leaderelection.LeaderElectionConfig, error) { 315 hostname, err := os.Hostname() 316 if err != nil { 317 return nil, fmt.Errorf("unable to get hostname: %v", err) 318 } 319 // add a uniquifier so that two processes on the same host don't accidentally both become active 320 id := hostname + "_" + string(uuid.NewUUID()) 321 322 rl, err := resourcelock.NewFromKubeconfig(config.ResourceLock, 323 config.ResourceNamespace, 324 config.ResourceName, 325 resourcelock.ResourceLockConfig{ 326 Identity: id, 327 EventRecorder: recorder, 328 }, 329 kubeConfig, 330 config.RenewDeadline.Duration) 331 if err != nil { 332 return nil, fmt.Errorf("couldn't create resource lock: %v", err) 333 } 334 335 return &leaderelection.LeaderElectionConfig{ 336 Lock: rl, 337 LeaseDuration: config.LeaseDuration.Duration, 338 RenewDeadline: config.RenewDeadline.Duration, 339 RetryPeriod: config.RetryPeriod.Duration, 340 WatchDog: leaderelection.NewLeaderHealthzAdaptor(time.Second * 20), 341 Name: "kube-scheduler", 342 ReleaseOnCancel: true, 343 }, nil 344 } 345 346 // createKubeConfig creates a kubeConfig from the given config and masterOverride. 347 // TODO remove masterOverride when CLI flags are removed. 348 func createKubeConfig(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string) (*restclient.Config, error) { 349 kubeConfig, err := clientcmd.BuildConfigFromFlags(masterOverride, config.Kubeconfig) 350 if err != nil { 351 return nil, err 352 } 353 354 kubeConfig.DisableCompression = true 355 kubeConfig.AcceptContentTypes = config.AcceptContentTypes 356 kubeConfig.ContentType = config.ContentType 357 kubeConfig.QPS = config.QPS 358 kubeConfig.Burst = int(config.Burst) 359 360 return kubeConfig, nil 361 } 362 363 // createClients creates a kube client and an event client from the given kubeConfig 364 func createClients(kubeConfig *restclient.Config) (clientset.Interface, clientset.Interface, error) { 365 client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeConfig, "scheduler")) 366 if err != nil { 367 return nil, nil, err 368 } 369 370 eventClient, err := clientset.NewForConfig(kubeConfig) 371 if err != nil { 372 return nil, nil, err 373 } 374 375 return client, eventClient, nil 376 }