k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/cmd/kube-scheduler/app/server_test.go (about) 1 /* 2 Copyright 2020 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package app 18 19 import ( 20 "context" 21 "fmt" 22 "net" 23 "net/http" 24 "net/http/httptest" 25 "os" 26 "path/filepath" 27 "testing" 28 "time" 29 30 "github.com/google/go-cmp/cmp" 31 "github.com/spf13/pflag" 32 v1 "k8s.io/api/core/v1" 33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 34 "k8s.io/apimachinery/pkg/runtime" 35 "k8s.io/apiserver/pkg/util/feature" 36 componentbaseconfig "k8s.io/component-base/config" 37 "k8s.io/component-base/featuregate" 38 featuregatetesting "k8s.io/component-base/featuregate/testing" 39 configv1 "k8s.io/kube-scheduler/config/v1" 40 "k8s.io/kubernetes/cmd/kube-scheduler/app/options" 41 "k8s.io/kubernetes/pkg/features" 42 "k8s.io/kubernetes/pkg/scheduler/apis/config" 43 "k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults" 44 "k8s.io/kubernetes/pkg/scheduler/framework" 45 ) 46 47 func TestSetup(t *testing.T) { 48 // temp dir 49 tmpDir, err := os.MkdirTemp("", "scheduler-options") 50 if err != nil { 51 t.Fatal(err) 52 } 53 defer os.RemoveAll(tmpDir) 54 55 // https server 56 server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 57 w.WriteHeader(200) 58 w.Write([]byte(`ok`)) 59 })) 60 defer server.Close() 61 62 configKubeconfig := filepath.Join(tmpDir, "config.kubeconfig") 63 if err := os.WriteFile(configKubeconfig, []byte(fmt.Sprintf(` 64 apiVersion: v1 65 kind: Config 66 clusters: 67 - cluster: 68 insecure-skip-tls-verify: true 69 server: %s 70 name: default 71 contexts: 72 - context: 73 cluster: default 74 user: default 75 name: default 76 current-context: default 77 users: 78 - name: default 79 user: 80 username: config 81 `, server.URL)), os.FileMode(0600)); err != nil { 82 t.Fatal(err) 83 } 84 85 // plugin config 86 pluginConfigFilev1 := filepath.Join(tmpDir, "pluginv1.yaml") 87 if err := os.WriteFile(pluginConfigFilev1, []byte(fmt.Sprintf(` 88 apiVersion: kubescheduler.config.k8s.io/v1 89 kind: KubeSchedulerConfiguration 90 clientConnection: 91 kubeconfig: '%s' 92 profiles: 93 - plugins: 94 multiPoint: 95 enabled: 96 - name: SchedulingGates 97 - name: DefaultBinder 98 - name: PrioritySort 99 - name: DefaultPreemption 100 - name: VolumeBinding 101 - name: NodeResourcesFit 102 - name: NodePorts 103 - name: InterPodAffinity 104 - name: TaintToleration 105 disabled: 106 - name: "*" 107 preFilter: 108 disabled: 109 - name: VolumeBinding 110 - name: InterPodAffinity 111 filter: 112 disabled: 113 - name: VolumeBinding 114 - name: InterPodAffinity 115 - name: TaintToleration 116 score: 117 disabled: 118 - name: VolumeBinding 119 - name: NodeResourcesFit 120 `, configKubeconfig)), os.FileMode(0600)); err != nil { 121 t.Fatal(err) 122 } 123 124 // plugin config 125 simplifiedPluginConfigFilev1 := filepath.Join(tmpDir, "simplifiedPluginv1.yaml") 126 if err := os.WriteFile(simplifiedPluginConfigFilev1, []byte(fmt.Sprintf(` 127 apiVersion: kubescheduler.config.k8s.io/v1 128 kind: KubeSchedulerConfiguration 129 clientConnection: 130 kubeconfig: '%s' 131 profiles: 132 - schedulerName: simplified-scheduler 133 `, configKubeconfig)), os.FileMode(0600)); err != nil { 134 t.Fatal(err) 135 } 136 137 // out-of-tree plugin config v1 138 outOfTreePluginConfigFilev1 := filepath.Join(tmpDir, "outOfTreePluginv1.yaml") 139 if err := os.WriteFile(outOfTreePluginConfigFilev1, []byte(fmt.Sprintf(` 140 apiVersion: kubescheduler.config.k8s.io/v1 141 kind: KubeSchedulerConfiguration 142 clientConnection: 143 kubeconfig: '%s' 144 profiles: 145 - plugins: 146 preFilter: 147 enabled: 148 - name: Foo 149 filter: 150 enabled: 151 - name: Foo 152 `, configKubeconfig)), os.FileMode(0600)); err != nil { 153 t.Fatal(err) 154 } 155 156 // multiple profiles config 157 multiProfilesConfig := filepath.Join(tmpDir, "multi-profiles.yaml") 158 if err := os.WriteFile(multiProfilesConfig, []byte(fmt.Sprintf(` 159 apiVersion: kubescheduler.config.k8s.io/v1 160 kind: KubeSchedulerConfiguration 161 clientConnection: 162 kubeconfig: '%s' 163 profiles: 164 - schedulerName: "profile-default-plugins" 165 - schedulerName: "profile-disable-all-filter-and-score-plugins" 166 plugins: 167 preFilter: 168 disabled: 169 - name: "*" 170 filter: 171 disabled: 172 - name: "*" 173 postFilter: 174 disabled: 175 - name: "*" 176 preScore: 177 disabled: 178 - name: "*" 179 score: 180 disabled: 181 - name: "*" 182 `, configKubeconfig)), os.FileMode(0600)); err != nil { 183 t.Fatal(err) 184 } 185 186 // empty leader-election config 187 emptyLeaderElectionConfig := filepath.Join(tmpDir, "empty-leader-election-config.yaml") 188 if err := os.WriteFile(emptyLeaderElectionConfig, []byte(fmt.Sprintf(` 189 apiVersion: kubescheduler.config.k8s.io/v1 190 kind: KubeSchedulerConfiguration 191 clientConnection: 192 kubeconfig: '%s' 193 `, configKubeconfig)), os.FileMode(0600)); err != nil { 194 t.Fatal(err) 195 } 196 197 // leader-election config 198 leaderElectionConfig := filepath.Join(tmpDir, "leader-election-config.yaml") 199 if err := os.WriteFile(leaderElectionConfig, []byte(fmt.Sprintf(` 200 apiVersion: kubescheduler.config.k8s.io/v1 201 kind: KubeSchedulerConfiguration 202 clientConnection: 203 kubeconfig: '%s' 204 leaderElection: 205 leaseDuration: 1h 206 `, configKubeconfig)), os.FileMode(0600)); err != nil { 207 t.Fatal(err) 208 } 209 210 testcases := []struct { 211 name string 212 flags []string 213 registryOptions []Option 214 restoreFeatures map[featuregate.Feature]bool 215 wantPlugins map[string]*config.Plugins 216 wantLeaderElection *componentbaseconfig.LeaderElectionConfiguration 217 wantClientConnection *componentbaseconfig.ClientConnectionConfiguration 218 }{ 219 { 220 name: "default config with an alpha feature enabled", 221 flags: []string{ 222 "--kubeconfig", configKubeconfig, 223 "--feature-gates=VolumeCapacityPriority=true", 224 }, 225 wantPlugins: map[string]*config.Plugins{ 226 "default-scheduler": defaults.ExpandedPluginsV1, 227 }, 228 restoreFeatures: map[featuregate.Feature]bool{ 229 features.VolumeCapacityPriority: false, 230 }, 231 }, 232 { 233 name: "component configuration v1 with only scheduler name configured", 234 flags: []string{ 235 "--config", simplifiedPluginConfigFilev1, 236 "--kubeconfig", configKubeconfig, 237 }, 238 wantPlugins: map[string]*config.Plugins{ 239 "simplified-scheduler": defaults.ExpandedPluginsV1, 240 }, 241 }, 242 { 243 name: "default config", 244 flags: []string{ 245 "--kubeconfig", configKubeconfig, 246 }, 247 wantPlugins: map[string]*config.Plugins{ 248 "default-scheduler": defaults.ExpandedPluginsV1, 249 }, 250 }, 251 { 252 name: "component configuration v1", 253 flags: []string{ 254 "--config", pluginConfigFilev1, 255 "--kubeconfig", configKubeconfig, 256 }, 257 wantPlugins: map[string]*config.Plugins{ 258 "default-scheduler": func() *config.Plugins { 259 plugins := defaults.ExpandedPluginsV1.DeepCopy() 260 plugins.Filter.Enabled = []config.Plugin{ 261 {Name: "NodeResourcesFit"}, 262 {Name: "NodePorts"}, 263 } 264 plugins.PreFilter.Enabled = []config.Plugin{ 265 {Name: "NodeResourcesFit"}, 266 {Name: "NodePorts"}, 267 } 268 plugins.PreScore.Enabled = []config.Plugin{ 269 {Name: "VolumeBinding"}, 270 {Name: "NodeResourcesFit"}, 271 {Name: "InterPodAffinity"}, 272 {Name: "TaintToleration"}, 273 } 274 plugins.Score.Enabled = []config.Plugin{ 275 {Name: "InterPodAffinity", Weight: 1}, 276 {Name: "TaintToleration", Weight: 1}, 277 } 278 return plugins 279 }(), 280 }, 281 }, 282 { 283 name: "out-of-tree component configuration v1", 284 flags: []string{ 285 "--config", outOfTreePluginConfigFilev1, 286 "--kubeconfig", configKubeconfig, 287 }, 288 registryOptions: []Option{WithPlugin("Foo", newFoo)}, 289 wantPlugins: map[string]*config.Plugins{ 290 "default-scheduler": func() *config.Plugins { 291 plugins := defaults.ExpandedPluginsV1.DeepCopy() 292 plugins.PreFilter.Enabled = append(plugins.PreFilter.Enabled, config.Plugin{Name: "Foo"}) 293 plugins.Filter.Enabled = append(plugins.Filter.Enabled, config.Plugin{Name: "Foo"}) 294 return plugins 295 }(), 296 }, 297 }, 298 { 299 name: "leader election CLI args, along with --config arg", 300 flags: []string{ 301 "--leader-elect=false", 302 "--leader-elect-lease-duration=2h", // CLI args are favored over the fields in ComponentConfig 303 "--kubeconfig=foo", // deprecated CLI arg will be ignored if --config is specified 304 "--config", emptyLeaderElectionConfig, 305 }, 306 wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ 307 LeaderElect: false, // from CLI args 308 LeaseDuration: metav1.Duration{Duration: 2 * time.Hour}, // from CLI args 309 RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, 310 RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, 311 ResourceLock: "leases", 312 ResourceName: configv1.SchedulerDefaultLockObjectName, 313 ResourceNamespace: configv1.SchedulerDefaultLockObjectNamespace, 314 }, 315 wantClientConnection: &componentbaseconfig.ClientConnectionConfiguration{ 316 Kubeconfig: configKubeconfig, 317 ContentType: "application/vnd.kubernetes.protobuf", 318 QPS: 50, 319 Burst: 100, 320 }, 321 }, 322 { 323 name: "leader election CLI args, without --config arg", 324 flags: []string{ 325 "--leader-elect=false", 326 "--leader-elect-lease-duration=2h", 327 "--leader-elect-resource-namespace=default", 328 "--kubeconfig", configKubeconfig, // deprecated CLI arg is honored if --config is not specified 329 }, 330 wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ 331 LeaderElect: false, // from CLI args 332 LeaseDuration: metav1.Duration{Duration: 2 * time.Hour}, // from CLI args 333 RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, 334 RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, 335 ResourceLock: "leases", 336 ResourceName: configv1.SchedulerDefaultLockObjectName, 337 ResourceNamespace: "default", // from CLI args 338 }, 339 wantClientConnection: &componentbaseconfig.ClientConnectionConfiguration{ 340 Kubeconfig: configKubeconfig, // from deprecated CLI args 341 ContentType: "application/vnd.kubernetes.protobuf", 342 QPS: 50, 343 Burst: 100, 344 }, 345 }, 346 { 347 name: "leader election settings specified by ComponentConfig only", 348 flags: []string{ 349 "--config", leaderElectionConfig, 350 }, 351 wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ 352 LeaderElect: true, 353 LeaseDuration: metav1.Duration{Duration: 1 * time.Hour}, // from CC 354 RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, 355 RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, 356 ResourceLock: "leases", 357 ResourceName: configv1.SchedulerDefaultLockObjectName, 358 ResourceNamespace: configv1.SchedulerDefaultLockObjectNamespace, 359 }, 360 }, 361 { 362 name: "leader election settings specified by CLI args and ComponentConfig", 363 flags: []string{ 364 "--leader-elect=true", 365 "--leader-elect-renew-deadline=5s", 366 "--leader-elect-retry-period=1s", 367 "--config", leaderElectionConfig, 368 }, 369 wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ 370 LeaderElect: true, 371 LeaseDuration: metav1.Duration{Duration: 1 * time.Hour}, // from CC 372 RenewDeadline: metav1.Duration{Duration: 5 * time.Second}, // from CLI args 373 RetryPeriod: metav1.Duration{Duration: 1 * time.Second}, // from CLI args 374 ResourceLock: "leases", 375 ResourceName: configv1.SchedulerDefaultLockObjectName, 376 ResourceNamespace: configv1.SchedulerDefaultLockObjectNamespace, 377 }, 378 }, 379 } 380 381 makeListener := func(t *testing.T) net.Listener { 382 t.Helper() 383 l, err := net.Listen("tcp", ":0") 384 if err != nil { 385 t.Fatal(err) 386 } 387 return l 388 } 389 390 for _, tc := range testcases { 391 t.Run(tc.name, func(t *testing.T) { 392 for k, v := range tc.restoreFeatures { 393 featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v) 394 } 395 396 fs := pflag.NewFlagSet("test", pflag.PanicOnError) 397 opts := options.NewOptions() 398 399 // use listeners instead of static ports so parallel test runs don't conflict 400 opts.SecureServing.Listener = makeListener(t) 401 defer opts.SecureServing.Listener.Close() 402 403 nfs := opts.Flags 404 for _, f := range nfs.FlagSets { 405 fs.AddFlagSet(f) 406 } 407 if err := fs.Parse(tc.flags); err != nil { 408 t.Fatal(err) 409 } 410 411 // use listeners instead of static ports so parallel test runs don't conflict 412 opts.SecureServing.Listener = makeListener(t) 413 defer opts.SecureServing.Listener.Close() 414 415 ctx, cancel := context.WithCancel(context.Background()) 416 defer cancel() 417 _, sched, err := Setup(ctx, opts, tc.registryOptions...) 418 if err != nil { 419 t.Fatal(err) 420 } 421 422 if tc.wantPlugins != nil { 423 gotPlugins := make(map[string]*config.Plugins) 424 for n, p := range sched.Profiles { 425 gotPlugins[n] = p.ListPlugins() 426 } 427 428 if diff := cmp.Diff(tc.wantPlugins, gotPlugins); diff != "" { 429 t.Errorf("Unexpected plugins diff (-want, +got): %s", diff) 430 } 431 } 432 433 if tc.wantLeaderElection != nil { 434 gotLeaderElection := opts.ComponentConfig.LeaderElection 435 if diff := cmp.Diff(*tc.wantLeaderElection, gotLeaderElection); diff != "" { 436 t.Errorf("Unexpected leaderElection diff (-want, +got): %s", diff) 437 } 438 } 439 440 if tc.wantClientConnection != nil { 441 gotClientConnection := opts.ComponentConfig.ClientConnection 442 if diff := cmp.Diff(*tc.wantClientConnection, gotClientConnection); diff != "" { 443 t.Errorf("Unexpected clientConnection diff (-want, +got): %s", diff) 444 } 445 } 446 }) 447 } 448 } 449 450 // Simulates an out-of-tree plugin. 451 type foo struct{} 452 453 var _ framework.PreFilterPlugin = &foo{} 454 var _ framework.FilterPlugin = &foo{} 455 456 func (*foo) Name() string { 457 return "Foo" 458 } 459 460 func newFoo(_ context.Context, _ runtime.Object, _ framework.Handle) (framework.Plugin, error) { 461 return &foo{}, nil 462 } 463 464 func (*foo) PreFilter(_ context.Context, _ *framework.CycleState, _ *v1.Pod) (*framework.PreFilterResult, *framework.Status) { 465 return nil, nil 466 } 467 468 func (*foo) PreFilterExtensions() framework.PreFilterExtensions { 469 return nil 470 } 471 472 func (*foo) Filter(_ context.Context, _ *framework.CycleState, _ *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { 473 return nil 474 }