k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controlplane/instance_test.go (about) 1 /* 2 Copyright 2014 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package controlplane 18 19 import ( 20 "context" 21 "crypto/tls" 22 "encoding/json" 23 "io" 24 "net" 25 "net/http" 26 "net/http/httptest" 27 "reflect" 28 "strings" 29 "testing" 30 31 "github.com/stretchr/testify/assert" 32 33 autoscalingapiv2beta1 "k8s.io/api/autoscaling/v2beta1" 34 autoscalingapiv2beta2 "k8s.io/api/autoscaling/v2beta2" 35 batchapiv1beta1 "k8s.io/api/batch/v1beta1" 36 certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1" 37 discoveryv1beta1 "k8s.io/api/discovery/v1beta1" 38 eventsv1beta1 "k8s.io/api/events/v1beta1" 39 nodev1beta1 "k8s.io/api/node/v1beta1" 40 policyapiv1beta1 "k8s.io/api/policy/v1beta1" 41 storageapiv1beta1 "k8s.io/api/storage/v1beta1" 42 extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" 43 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 44 "k8s.io/apimachinery/pkg/runtime/schema" 45 utilnet "k8s.io/apimachinery/pkg/util/net" 46 "k8s.io/apimachinery/pkg/util/sets" 47 "k8s.io/apimachinery/pkg/version" 48 "k8s.io/apiserver/pkg/authorization/authorizerfactory" 49 openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" 50 genericapiserver "k8s.io/apiserver/pkg/server" 51 "k8s.io/apiserver/pkg/server/options" 52 "k8s.io/apiserver/pkg/server/resourceconfig" 53 serverstorage "k8s.io/apiserver/pkg/server/storage" 54 etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" 55 "k8s.io/apiserver/pkg/util/openapi" 56 "k8s.io/client-go/discovery" 57 "k8s.io/client-go/informers" 58 "k8s.io/client-go/kubernetes" 59 restclient "k8s.io/client-go/rest" 60 kubeversion "k8s.io/component-base/version" 61 aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" 62 netutils "k8s.io/utils/net" 63 64 "k8s.io/kubernetes/pkg/api/legacyscheme" 65 flowcontrolv1bet3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3" 66 controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" 67 "k8s.io/kubernetes/pkg/controlplane/reconcilers" 68 "k8s.io/kubernetes/pkg/controlplane/storageversionhashdata" 69 generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" 70 "k8s.io/kubernetes/pkg/kubeapiserver" 71 kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" 72 certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest" 73 corerest "k8s.io/kubernetes/pkg/registry/core/rest" 74 "k8s.io/kubernetes/pkg/registry/registrytest" 75 ) 76 77 // setUp is a convenience function for setting up for (most) tests. 78 func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertions) { 79 server, storageConfig := etcd3testing.NewUnsecuredEtcd3TestClientServer(t) 80 81 config := &Config{ 82 ControlPlane: controlplaneapiserver.Config{ 83 Generic: genericapiserver.NewConfig(legacyscheme.Codecs), 84 Extra: controlplaneapiserver.Extra{ 85 APIResourceConfigSource: DefaultAPIResourceConfigSource(), 86 }, 87 }, 88 Extra: Extra{ 89 APIServerServicePort: 443, 90 MasterCount: 1, 91 EndpointReconcilerType: reconcilers.MasterCountReconcilerType, 92 ServiceIPRange: net.IPNet{IP: netutils.ParseIPSloppy("10.0.0.0"), Mask: net.CIDRMask(24, 32)}, 93 }, 94 } 95 96 storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() 97 storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker 98 resourceEncoding := resourceconfig.MergeResourceEncodingConfigs(storageFactoryConfig.DefaultResourceEncoding, storageFactoryConfig.ResourceEncodingOverrides) 99 storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, "application/vnd.kubernetes.protobuf", storageFactoryConfig.Serializer, resourceEncoding, DefaultAPIResourceConfigSource(), nil) 100 etcdOptions := options.NewEtcdOptions(storageConfig) 101 // unit tests don't need watch cache and it leaks lots of goroutines with etcd testing functions during unit tests 102 etcdOptions.EnableWatchCache = false 103 err := etcdOptions.ApplyWithStorageFactoryTo(storageFactory, config.ControlPlane.Generic) 104 if err != nil { 105 t.Fatal(err) 106 } 107 108 kubeVersion := kubeversion.Get() 109 config.ControlPlane.Generic.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() 110 config.ControlPlane.Generic.Version = &kubeVersion 111 config.ControlPlane.StorageFactory = storageFactory 112 config.ControlPlane.Generic.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} 113 config.ControlPlane.Generic.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") 114 config.ControlPlane.Generic.LegacyAPIGroupPrefixes = sets.NewString("/api") 115 config.Extra.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250} 116 config.ControlPlane.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{ 117 DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, nil }, 118 TLSClientConfig: &tls.Config{}, 119 }) 120 121 // set fake SecureServingInfo because the listener port is needed for the kubernetes service 122 config.ControlPlane.Generic.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}} 123 124 getOpenAPIDefinitions := openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(generatedopenapi.GetOpenAPIDefinitions) 125 namer := openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme) 126 config.ControlPlane.Generic.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, namer) 127 128 clientset, err := kubernetes.NewForConfig(config.ControlPlane.Generic.LoopbackClientConfig) 129 if err != nil { 130 t.Fatalf("unable to create client set due to %v", err) 131 } 132 config.ControlPlane.VersionedInformers = informers.NewSharedInformerFactory(clientset, config.ControlPlane.Generic.LoopbackClientConfig.Timeout) 133 134 return server, *config, assert.New(t) 135 } 136 137 type fakeLocalhost443Listener struct{} 138 139 func (fakeLocalhost443Listener) Accept() (net.Conn, error) { 140 return nil, nil 141 } 142 143 func (fakeLocalhost443Listener) Close() error { 144 return nil 145 } 146 147 func (fakeLocalhost443Listener) Addr() net.Addr { 148 return &net.TCPAddr{ 149 IP: net.IPv4(127, 0, 0, 1), 150 Port: 443, 151 } 152 } 153 154 // TestLegacyRestStorageStrategies ensures that all Storage objects which are using the generic registry Store have 155 // their various strategies properly wired up. This surfaced as a bug where strategies defined Export functions, but 156 // they were never used outside of unit tests because the export strategies were not assigned inside the Store. 157 func TestLegacyRestStorageStrategies(t *testing.T) { 158 _, etcdserver, apiserverCfg, _ := newInstance(t) 159 defer etcdserver.Terminate(t) 160 161 storageProvider, err := corerest.New(corerest.Config{ 162 GenericConfig: corerest.GenericConfig{ 163 StorageFactory: apiserverCfg.ControlPlane.Extra.StorageFactory, 164 EventTTL: apiserverCfg.ControlPlane.Extra.EventTTL, 165 LoopbackClientConfig: apiserverCfg.ControlPlane.Generic.LoopbackClientConfig, 166 Informers: apiserverCfg.ControlPlane.Extra.VersionedInformers, 167 }, 168 Proxy: corerest.ProxyConfig{ 169 Transport: apiserverCfg.ControlPlane.Extra.ProxyTransport, 170 KubeletClientConfig: apiserverCfg.Extra.KubeletClientConfig, 171 }, 172 Services: corerest.ServicesConfig{ 173 ClusterIPRange: apiserverCfg.Extra.ServiceIPRange, 174 NodePortRange: apiserverCfg.Extra.ServiceNodePortRange, 175 }, 176 }) 177 if err != nil { 178 t.Fatalf("unexpected error from REST storage: %v", err) 179 } 180 181 apiGroupInfo, err := storageProvider.NewRESTStorage(serverstorage.NewResourceConfig(), apiserverCfg.ControlPlane.Generic.RESTOptionsGetter) 182 if err != nil { 183 t.Errorf("failed to create legacy REST storage: %v", err) 184 } 185 186 strategyErrors := registrytest.ValidateStorageStrategies(apiGroupInfo.VersionedResourcesStorageMap["v1"]) 187 for _, err := range strategyErrors { 188 t.Error(err) 189 } 190 } 191 192 func TestCertificatesRestStorageStrategies(t *testing.T) { 193 _, etcdserver, apiserverCfg, _ := newInstance(t) 194 defer etcdserver.Terminate(t) 195 196 certStorageProvider := certificatesrest.RESTStorageProvider{} 197 apiGroupInfo, err := certStorageProvider.NewRESTStorage(apiserverCfg.ControlPlane.APIResourceConfigSource, apiserverCfg.ControlPlane.Generic.RESTOptionsGetter) 198 if err != nil { 199 t.Fatalf("unexpected error from REST storage: %v", err) 200 } 201 202 strategyErrors := registrytest.ValidateStorageStrategies( 203 apiGroupInfo.VersionedResourcesStorageMap[certificatesapiv1beta1.SchemeGroupVersion.Version]) 204 for _, err := range strategyErrors { 205 t.Error(err) 206 } 207 } 208 209 func newInstance(t *testing.T) (*Instance, *etcd3testing.EtcdTestServer, Config, *assert.Assertions) { 210 etcdserver, config, assert := setUp(t) 211 212 apiserver, err := config.Complete().New(genericapiserver.NewEmptyDelegate()) 213 if err != nil { 214 t.Fatalf("Error in bringing up the master: %v", err) 215 } 216 217 return apiserver, etcdserver, config, assert 218 } 219 220 // TestVersion tests /version 221 func TestVersion(t *testing.T) { 222 s, etcdserver, _, _ := newInstance(t) 223 defer etcdserver.Terminate(t) 224 225 req, _ := http.NewRequest("GET", "/version", nil) 226 resp := httptest.NewRecorder() 227 s.ControlPlane.GenericAPIServer.Handler.ServeHTTP(resp, req) 228 if resp.Code != 200 { 229 t.Fatalf("expected http 200, got: %d", resp.Code) 230 } 231 232 var info version.Info 233 err := json.NewDecoder(resp.Body).Decode(&info) 234 if err != nil { 235 t.Errorf("unexpected error: %v", err) 236 } 237 238 if !reflect.DeepEqual(kubeversion.Get(), info) { 239 t.Errorf("Expected %#v, Got %#v", kubeversion.Get(), info) 240 } 241 } 242 243 func decodeResponse(resp *http.Response, obj interface{}) error { 244 defer resp.Body.Close() 245 246 data, err := io.ReadAll(resp.Body) 247 if err != nil { 248 return err 249 } 250 if err := json.Unmarshal(data, obj); err != nil { 251 return err 252 } 253 return nil 254 } 255 256 // Because we need to be backwards compatible with release 1.1, at endpoints 257 // that exist in release 1.1, the responses should have empty APIVersion. 258 func TestAPIVersionOfDiscoveryEndpoints(t *testing.T) { 259 apiserver, etcdserver, _, assert := newInstance(t) 260 defer etcdserver.Terminate(t) 261 262 server := httptest.NewServer(apiserver.ControlPlane.GenericAPIServer.Handler.GoRestfulContainer.ServeMux) 263 264 // /api exists in release-1.1 265 resp, err := http.Get(server.URL + "/api") 266 if err != nil { 267 t.Errorf("unexpected error: %v", err) 268 } 269 apiVersions := metav1.APIVersions{} 270 assert.NoError(decodeResponse(resp, &apiVersions)) 271 assert.Equal(apiVersions.APIVersion, "") 272 273 // /api/v1 exists in release-1.1 274 resp, err = http.Get(server.URL + "/api/v1") 275 if err != nil { 276 t.Errorf("unexpected error: %v", err) 277 } 278 resourceList := metav1.APIResourceList{} 279 assert.NoError(decodeResponse(resp, &resourceList)) 280 assert.Equal(resourceList.APIVersion, "") 281 282 // /apis exists in release-1.1 283 resp, err = http.Get(server.URL + "/apis") 284 if err != nil { 285 t.Errorf("unexpected error: %v", err) 286 } 287 groupList := metav1.APIGroupList{} 288 assert.NoError(decodeResponse(resp, &groupList)) 289 assert.Equal(groupList.APIVersion, "") 290 291 // /apis/autoscaling doesn't exist in release-1.1, so the APIVersion field 292 // should be non-empty in the results returned by the server. 293 resp, err = http.Get(server.URL + "/apis/autoscaling") 294 if err != nil { 295 t.Errorf("unexpected error: %v", err) 296 } 297 group := metav1.APIGroup{} 298 assert.NoError(decodeResponse(resp, &group)) 299 assert.Equal(group.APIVersion, "v1") 300 301 // apis/autoscaling/v1 doesn't exist in release-1.1, so the APIVersion field 302 // should be non-empty in the results returned by the server. 303 304 resp, err = http.Get(server.URL + "/apis/autoscaling/v1") 305 if err != nil { 306 t.Errorf("unexpected error: %v", err) 307 } 308 resourceList = metav1.APIResourceList{} 309 assert.NoError(decodeResponse(resp, &resourceList)) 310 assert.Equal(resourceList.APIVersion, "v1") 311 312 } 313 314 // This test doesn't cover the apiregistration and apiextensions group, as they are installed by other apiservers. 315 func TestStorageVersionHashes(t *testing.T) { 316 apiserver, etcdserver, _, _ := newInstance(t) 317 defer etcdserver.Terminate(t) 318 319 server := httptest.NewServer(apiserver.ControlPlane.GenericAPIServer.Handler.GoRestfulContainer.ServeMux) 320 321 c := &restclient.Config{ 322 Host: server.URL, 323 APIPath: "/api", 324 ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}, 325 } 326 discover := discovery.NewDiscoveryClientForConfigOrDie(c).WithLegacy() 327 _, all, err := discover.ServerGroupsAndResources() 328 if err != nil { 329 t.Error(err) 330 } 331 var count int 332 apiResources := sets.NewString() 333 for _, g := range all { 334 for _, r := range g.APIResources { 335 apiResources.Insert(g.GroupVersion + "/" + r.Name) 336 if strings.Contains(r.Name, "/") || 337 storageversionhashdata.NoStorageVersionHash.Has(g.GroupVersion+"/"+r.Name) { 338 if r.StorageVersionHash != "" { 339 t.Errorf("expect resource %s/%s to have empty storageVersionHash, got hash %q", g.GroupVersion, r.Name, r.StorageVersionHash) 340 } 341 continue 342 } 343 if r.StorageVersionHash == "" { 344 t.Errorf("expect the storageVersionHash of %s/%s to exist", g.GroupVersion, r.Name) 345 continue 346 } 347 // Uncomment the following line if you want to update storageversionhash/data.go 348 // fmt.Printf("\"%s/%s\": \"%s\",\n", g.GroupVersion, r.Name, r.StorageVersionHash) 349 expected := storageversionhashdata.GVRToStorageVersionHash[g.GroupVersion+"/"+r.Name] 350 if r.StorageVersionHash != expected { 351 t.Errorf("expect the storageVersionHash of %s/%s to be %q, got %q", g.GroupVersion, r.Name, expected, r.StorageVersionHash) 352 } 353 count++ 354 } 355 } 356 if count != len(storageversionhashdata.GVRToStorageVersionHash) { 357 knownResources := sets.StringKeySet(storageversionhashdata.GVRToStorageVersionHash) 358 t.Errorf("please remove the redundant entries from GVRToStorageVersionHash: %v", knownResources.Difference(apiResources).List()) 359 } 360 } 361 362 func TestNoAlphaVersionsEnabledByDefault(t *testing.T) { 363 config := DefaultAPIResourceConfigSource() 364 for gv, enable := range config.GroupVersionConfigs { 365 if enable && strings.Contains(gv.Version, "alpha") { 366 t.Errorf("Alpha API version %s enabled by default", gv.String()) 367 } 368 } 369 370 for gvr, enabled := range config.ResourceConfigs { 371 if !strings.Contains(gvr.Version, "alpha") || !enabled { 372 continue 373 } 374 375 // we have enabled an alpha api by resource {g,v,r}, we also expect the 376 // alpha api by version {g,v} to be disabled. This is so a programmer 377 // remembers to add the new alpha version to alphaAPIGroupVersionsDisabledByDefault. 378 gr := gvr.GroupVersion() 379 if enabled, found := config.GroupVersionConfigs[gr]; !found || enabled { 380 t.Errorf("Alpha API version %q should be disabled by default", gr.String()) 381 } 382 } 383 } 384 385 func TestNoBetaVersionsEnabledByDefault(t *testing.T) { 386 config := DefaultAPIResourceConfigSource() 387 for gv, enable := range config.GroupVersionConfigs { 388 if enable && strings.Contains(gv.Version, "beta") { 389 t.Errorf("Beta API version %s enabled by default", gv.String()) 390 } 391 } 392 393 for gvr, enabled := range config.ResourceConfigs { 394 if !strings.Contains(gvr.Version, "beta") || !enabled { 395 continue 396 } 397 398 // we have enabled a beta api by resource {g,v,r}, we also expect the 399 // beta api by version {g,v} to be disabled. This is so a programmer 400 // remembers to add the new beta version to betaAPIGroupVersionsDisabledByDefault. 401 gr := gvr.GroupVersion() 402 if enabled, found := config.GroupVersionConfigs[gr]; !found || enabled { 403 t.Errorf("Beta API version %q should be disabled by default", gr.String()) 404 } 405 } 406 } 407 408 func TestDefaultVars(t *testing.T) { 409 // stableAPIGroupVersionsEnabledByDefault should not contain beta or alpha 410 for i := range stableAPIGroupVersionsEnabledByDefault { 411 gv := stableAPIGroupVersionsEnabledByDefault[i] 412 if strings.Contains(gv.Version, "beta") || strings.Contains(gv.Version, "alpha") { 413 t.Errorf("stableAPIGroupVersionsEnabledByDefault should contain stable version, but found: %q", gv.String()) 414 } 415 } 416 417 // legacyBetaEnabledByDefaultResources should contain only beta version 418 for i := range legacyBetaEnabledByDefaultResources { 419 gv := legacyBetaEnabledByDefaultResources[i] 420 if !strings.Contains(gv.Version, "beta") { 421 t.Errorf("legacyBetaEnabledByDefaultResources should contain beta version, but found: %q", gv.String()) 422 } 423 } 424 425 // betaAPIGroupVersionsDisabledByDefault should contain only beta version 426 for i := range betaAPIGroupVersionsDisabledByDefault { 427 gv := betaAPIGroupVersionsDisabledByDefault[i] 428 if !strings.Contains(gv.Version, "beta") { 429 t.Errorf("betaAPIGroupVersionsDisabledByDefault should contain beta version, but found: %q", gv.String()) 430 } 431 } 432 433 // alphaAPIGroupVersionsDisabledByDefault should contain only alpha version 434 for i := range alphaAPIGroupVersionsDisabledByDefault { 435 gv := alphaAPIGroupVersionsDisabledByDefault[i] 436 if !strings.Contains(gv.Version, "alpha") { 437 t.Errorf("alphaAPIGroupVersionsDisabledByDefault should contain alpha version, but found: %q", gv.String()) 438 } 439 } 440 } 441 442 func TestNewBetaResourcesEnabledByDefault(t *testing.T) { 443 // legacyEnabledBetaResources is nearly a duplication from elsewhere. This is intentional. These types already have 444 // GA equivalents available and should therefore never have a beta enabled by default again. 445 legacyEnabledBetaResources := map[schema.GroupVersionResource]bool{ 446 autoscalingapiv2beta1.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true, 447 autoscalingapiv2beta2.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true, 448 batchapiv1beta1.SchemeGroupVersion.WithResource("cronjobs"): true, 449 discoveryv1beta1.SchemeGroupVersion.WithResource("endpointslices"): true, 450 eventsv1beta1.SchemeGroupVersion.WithResource("events"): true, 451 nodev1beta1.SchemeGroupVersion.WithResource("runtimeclasses"): true, 452 policyapiv1beta1.SchemeGroupVersion.WithResource("poddisruptionbudgets"): true, 453 policyapiv1beta1.SchemeGroupVersion.WithResource("podsecuritypolicies"): true, 454 storageapiv1beta1.SchemeGroupVersion.WithResource("csinodes"): true, 455 } 456 457 // legacyBetaResourcesWithoutStableEquivalents contains those groupresources that were enabled by default as beta 458 // before we changed that policy and do not have stable versions. These resources are allowed to have additional 459 // beta versions enabled by default. Nothing new should be added here. There are no future exceptions because there 460 // are no more beta resources enabled by default. 461 legacyBetaResourcesWithoutStableEquivalents := map[schema.GroupResource]bool{ 462 flowcontrolv1bet3.SchemeGroupVersion.WithResource("flowschemas").GroupResource(): true, 463 flowcontrolv1bet3.SchemeGroupVersion.WithResource("prioritylevelconfigurations").GroupResource(): true, 464 } 465 466 config := DefaultAPIResourceConfigSource() 467 for gvr, enable := range config.ResourceConfigs { 468 if !strings.Contains(gvr.Version, "beta") { 469 continue // only check beta things 470 } 471 if !enable { 472 continue // only check things that are enabled 473 } 474 if legacyEnabledBetaResources[gvr] { 475 continue // this is a legacy beta resource 476 } 477 if legacyBetaResourcesWithoutStableEquivalents[gvr.GroupResource()] { 478 continue // this is another beta of a legacy beta resource with no stable equivalent 479 } 480 t.Errorf("no new beta resources can be enabled by default, see https://github.com/kubernetes/enhancements/blob/0ad0fc8269165ca300d05ca51c7ce190a79976a5/keps/sig-architecture/3136-beta-apis-off-by-default/README.md: %v", gvr) 481 } 482 }