k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controlplane/instance_test.go (about)

     1  /*
     2  Copyright 2014 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controlplane
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"encoding/json"
    23  	"io"
    24  	"net"
    25  	"net/http"
    26  	"net/http/httptest"
    27  	"reflect"
    28  	"strings"
    29  	"testing"
    30  
    31  	"github.com/stretchr/testify/assert"
    32  
    33  	autoscalingapiv2beta1 "k8s.io/api/autoscaling/v2beta1"
    34  	autoscalingapiv2beta2 "k8s.io/api/autoscaling/v2beta2"
    35  	batchapiv1beta1 "k8s.io/api/batch/v1beta1"
    36  	certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1"
    37  	discoveryv1beta1 "k8s.io/api/discovery/v1beta1"
    38  	eventsv1beta1 "k8s.io/api/events/v1beta1"
    39  	nodev1beta1 "k8s.io/api/node/v1beta1"
    40  	policyapiv1beta1 "k8s.io/api/policy/v1beta1"
    41  	storageapiv1beta1 "k8s.io/api/storage/v1beta1"
    42  	extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
    43  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    44  	"k8s.io/apimachinery/pkg/runtime/schema"
    45  	utilnet "k8s.io/apimachinery/pkg/util/net"
    46  	"k8s.io/apimachinery/pkg/util/sets"
    47  	"k8s.io/apimachinery/pkg/version"
    48  	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
    49  	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
    50  	genericapiserver "k8s.io/apiserver/pkg/server"
    51  	"k8s.io/apiserver/pkg/server/options"
    52  	"k8s.io/apiserver/pkg/server/resourceconfig"
    53  	serverstorage "k8s.io/apiserver/pkg/server/storage"
    54  	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
    55  	"k8s.io/apiserver/pkg/util/openapi"
    56  	"k8s.io/client-go/discovery"
    57  	"k8s.io/client-go/informers"
    58  	"k8s.io/client-go/kubernetes"
    59  	restclient "k8s.io/client-go/rest"
    60  	kubeversion "k8s.io/component-base/version"
    61  	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
    62  	netutils "k8s.io/utils/net"
    63  
    64  	"k8s.io/kubernetes/pkg/api/legacyscheme"
    65  	flowcontrolv1bet3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3"
    66  	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
    67  	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
    68  	"k8s.io/kubernetes/pkg/controlplane/storageversionhashdata"
    69  	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
    70  	"k8s.io/kubernetes/pkg/kubeapiserver"
    71  	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
    72  	certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest"
    73  	corerest "k8s.io/kubernetes/pkg/registry/core/rest"
    74  	"k8s.io/kubernetes/pkg/registry/registrytest"
    75  )
    76  
    77  // setUp is a convenience function for setting up for (most) tests.
    78  func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertions) {
    79  	server, storageConfig := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
    80  
    81  	config := &Config{
    82  		ControlPlane: controlplaneapiserver.Config{
    83  			Generic: genericapiserver.NewConfig(legacyscheme.Codecs),
    84  			Extra: controlplaneapiserver.Extra{
    85  				APIResourceConfigSource: DefaultAPIResourceConfigSource(),
    86  			},
    87  		},
    88  		Extra: Extra{
    89  			APIServerServicePort:   443,
    90  			MasterCount:            1,
    91  			EndpointReconcilerType: reconcilers.MasterCountReconcilerType,
    92  			ServiceIPRange:         net.IPNet{IP: netutils.ParseIPSloppy("10.0.0.0"), Mask: net.CIDRMask(24, 32)},
    93  		},
    94  	}
    95  
    96  	storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
    97  	storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker
    98  	resourceEncoding := resourceconfig.MergeResourceEncodingConfigs(storageFactoryConfig.DefaultResourceEncoding, storageFactoryConfig.ResourceEncodingOverrides)
    99  	storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, "application/vnd.kubernetes.protobuf", storageFactoryConfig.Serializer, resourceEncoding, DefaultAPIResourceConfigSource(), nil)
   100  	etcdOptions := options.NewEtcdOptions(storageConfig)
   101  	// unit tests don't need watch cache and it leaks lots of goroutines with etcd testing functions during unit tests
   102  	etcdOptions.EnableWatchCache = false
   103  	err := etcdOptions.ApplyWithStorageFactoryTo(storageFactory, config.ControlPlane.Generic)
   104  	if err != nil {
   105  		t.Fatal(err)
   106  	}
   107  
   108  	kubeVersion := kubeversion.Get()
   109  	config.ControlPlane.Generic.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
   110  	config.ControlPlane.Generic.Version = &kubeVersion
   111  	config.ControlPlane.StorageFactory = storageFactory
   112  	config.ControlPlane.Generic.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
   113  	config.ControlPlane.Generic.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
   114  	config.ControlPlane.Generic.LegacyAPIGroupPrefixes = sets.NewString("/api")
   115  	config.Extra.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250}
   116  	config.ControlPlane.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{
   117  		DialContext:     func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, nil },
   118  		TLSClientConfig: &tls.Config{},
   119  	})
   120  
   121  	// set fake SecureServingInfo because the listener port is needed for the kubernetes service
   122  	config.ControlPlane.Generic.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}}
   123  
   124  	getOpenAPIDefinitions := openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(generatedopenapi.GetOpenAPIDefinitions)
   125  	namer := openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme)
   126  	config.ControlPlane.Generic.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, namer)
   127  
   128  	clientset, err := kubernetes.NewForConfig(config.ControlPlane.Generic.LoopbackClientConfig)
   129  	if err != nil {
   130  		t.Fatalf("unable to create client set due to %v", err)
   131  	}
   132  	config.ControlPlane.VersionedInformers = informers.NewSharedInformerFactory(clientset, config.ControlPlane.Generic.LoopbackClientConfig.Timeout)
   133  
   134  	return server, *config, assert.New(t)
   135  }
   136  
   137  type fakeLocalhost443Listener struct{}
   138  
   139  func (fakeLocalhost443Listener) Accept() (net.Conn, error) {
   140  	return nil, nil
   141  }
   142  
   143  func (fakeLocalhost443Listener) Close() error {
   144  	return nil
   145  }
   146  
   147  func (fakeLocalhost443Listener) Addr() net.Addr {
   148  	return &net.TCPAddr{
   149  		IP:   net.IPv4(127, 0, 0, 1),
   150  		Port: 443,
   151  	}
   152  }
   153  
   154  // TestLegacyRestStorageStrategies ensures that all Storage objects which are using the generic registry Store have
   155  // their various strategies properly wired up. This surfaced as a bug where strategies defined Export functions, but
   156  // they were never used outside of unit tests because the export strategies were not assigned inside the Store.
   157  func TestLegacyRestStorageStrategies(t *testing.T) {
   158  	_, etcdserver, apiserverCfg, _ := newInstance(t)
   159  	defer etcdserver.Terminate(t)
   160  
   161  	storageProvider, err := corerest.New(corerest.Config{
   162  		GenericConfig: corerest.GenericConfig{
   163  			StorageFactory:       apiserverCfg.ControlPlane.Extra.StorageFactory,
   164  			EventTTL:             apiserverCfg.ControlPlane.Extra.EventTTL,
   165  			LoopbackClientConfig: apiserverCfg.ControlPlane.Generic.LoopbackClientConfig,
   166  			Informers:            apiserverCfg.ControlPlane.Extra.VersionedInformers,
   167  		},
   168  		Proxy: corerest.ProxyConfig{
   169  			Transport:           apiserverCfg.ControlPlane.Extra.ProxyTransport,
   170  			KubeletClientConfig: apiserverCfg.Extra.KubeletClientConfig,
   171  		},
   172  		Services: corerest.ServicesConfig{
   173  			ClusterIPRange: apiserverCfg.Extra.ServiceIPRange,
   174  			NodePortRange:  apiserverCfg.Extra.ServiceNodePortRange,
   175  		},
   176  	})
   177  	if err != nil {
   178  		t.Fatalf("unexpected error from REST storage: %v", err)
   179  	}
   180  
   181  	apiGroupInfo, err := storageProvider.NewRESTStorage(serverstorage.NewResourceConfig(), apiserverCfg.ControlPlane.Generic.RESTOptionsGetter)
   182  	if err != nil {
   183  		t.Errorf("failed to create legacy REST storage: %v", err)
   184  	}
   185  
   186  	strategyErrors := registrytest.ValidateStorageStrategies(apiGroupInfo.VersionedResourcesStorageMap["v1"])
   187  	for _, err := range strategyErrors {
   188  		t.Error(err)
   189  	}
   190  }
   191  
   192  func TestCertificatesRestStorageStrategies(t *testing.T) {
   193  	_, etcdserver, apiserverCfg, _ := newInstance(t)
   194  	defer etcdserver.Terminate(t)
   195  
   196  	certStorageProvider := certificatesrest.RESTStorageProvider{}
   197  	apiGroupInfo, err := certStorageProvider.NewRESTStorage(apiserverCfg.ControlPlane.APIResourceConfigSource, apiserverCfg.ControlPlane.Generic.RESTOptionsGetter)
   198  	if err != nil {
   199  		t.Fatalf("unexpected error from REST storage: %v", err)
   200  	}
   201  
   202  	strategyErrors := registrytest.ValidateStorageStrategies(
   203  		apiGroupInfo.VersionedResourcesStorageMap[certificatesapiv1beta1.SchemeGroupVersion.Version])
   204  	for _, err := range strategyErrors {
   205  		t.Error(err)
   206  	}
   207  }
   208  
   209  func newInstance(t *testing.T) (*Instance, *etcd3testing.EtcdTestServer, Config, *assert.Assertions) {
   210  	etcdserver, config, assert := setUp(t)
   211  
   212  	apiserver, err := config.Complete().New(genericapiserver.NewEmptyDelegate())
   213  	if err != nil {
   214  		t.Fatalf("Error in bringing up the master: %v", err)
   215  	}
   216  
   217  	return apiserver, etcdserver, config, assert
   218  }
   219  
   220  // TestVersion tests /version
   221  func TestVersion(t *testing.T) {
   222  	s, etcdserver, _, _ := newInstance(t)
   223  	defer etcdserver.Terminate(t)
   224  
   225  	req, _ := http.NewRequest("GET", "/version", nil)
   226  	resp := httptest.NewRecorder()
   227  	s.ControlPlane.GenericAPIServer.Handler.ServeHTTP(resp, req)
   228  	if resp.Code != 200 {
   229  		t.Fatalf("expected http 200, got: %d", resp.Code)
   230  	}
   231  
   232  	var info version.Info
   233  	err := json.NewDecoder(resp.Body).Decode(&info)
   234  	if err != nil {
   235  		t.Errorf("unexpected error: %v", err)
   236  	}
   237  
   238  	if !reflect.DeepEqual(kubeversion.Get(), info) {
   239  		t.Errorf("Expected %#v, Got %#v", kubeversion.Get(), info)
   240  	}
   241  }
   242  
   243  func decodeResponse(resp *http.Response, obj interface{}) error {
   244  	defer resp.Body.Close()
   245  
   246  	data, err := io.ReadAll(resp.Body)
   247  	if err != nil {
   248  		return err
   249  	}
   250  	if err := json.Unmarshal(data, obj); err != nil {
   251  		return err
   252  	}
   253  	return nil
   254  }
   255  
   256  // Because we need to be backwards compatible with release 1.1, at endpoints
   257  // that exist in release 1.1, the responses should have empty APIVersion.
   258  func TestAPIVersionOfDiscoveryEndpoints(t *testing.T) {
   259  	apiserver, etcdserver, _, assert := newInstance(t)
   260  	defer etcdserver.Terminate(t)
   261  
   262  	server := httptest.NewServer(apiserver.ControlPlane.GenericAPIServer.Handler.GoRestfulContainer.ServeMux)
   263  
   264  	// /api exists in release-1.1
   265  	resp, err := http.Get(server.URL + "/api")
   266  	if err != nil {
   267  		t.Errorf("unexpected error: %v", err)
   268  	}
   269  	apiVersions := metav1.APIVersions{}
   270  	assert.NoError(decodeResponse(resp, &apiVersions))
   271  	assert.Equal(apiVersions.APIVersion, "")
   272  
   273  	// /api/v1 exists in release-1.1
   274  	resp, err = http.Get(server.URL + "/api/v1")
   275  	if err != nil {
   276  		t.Errorf("unexpected error: %v", err)
   277  	}
   278  	resourceList := metav1.APIResourceList{}
   279  	assert.NoError(decodeResponse(resp, &resourceList))
   280  	assert.Equal(resourceList.APIVersion, "")
   281  
   282  	// /apis exists in release-1.1
   283  	resp, err = http.Get(server.URL + "/apis")
   284  	if err != nil {
   285  		t.Errorf("unexpected error: %v", err)
   286  	}
   287  	groupList := metav1.APIGroupList{}
   288  	assert.NoError(decodeResponse(resp, &groupList))
   289  	assert.Equal(groupList.APIVersion, "")
   290  
   291  	// /apis/autoscaling doesn't exist in release-1.1, so the APIVersion field
   292  	// should be non-empty in the results returned by the server.
   293  	resp, err = http.Get(server.URL + "/apis/autoscaling")
   294  	if err != nil {
   295  		t.Errorf("unexpected error: %v", err)
   296  	}
   297  	group := metav1.APIGroup{}
   298  	assert.NoError(decodeResponse(resp, &group))
   299  	assert.Equal(group.APIVersion, "v1")
   300  
   301  	// apis/autoscaling/v1 doesn't exist in release-1.1, so the APIVersion field
   302  	// should be non-empty in the results returned by the server.
   303  
   304  	resp, err = http.Get(server.URL + "/apis/autoscaling/v1")
   305  	if err != nil {
   306  		t.Errorf("unexpected error: %v", err)
   307  	}
   308  	resourceList = metav1.APIResourceList{}
   309  	assert.NoError(decodeResponse(resp, &resourceList))
   310  	assert.Equal(resourceList.APIVersion, "v1")
   311  
   312  }
   313  
   314  // This test doesn't cover the apiregistration and apiextensions group, as they are installed by other apiservers.
   315  func TestStorageVersionHashes(t *testing.T) {
   316  	apiserver, etcdserver, _, _ := newInstance(t)
   317  	defer etcdserver.Terminate(t)
   318  
   319  	server := httptest.NewServer(apiserver.ControlPlane.GenericAPIServer.Handler.GoRestfulContainer.ServeMux)
   320  
   321  	c := &restclient.Config{
   322  		Host:          server.URL,
   323  		APIPath:       "/api",
   324  		ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs},
   325  	}
   326  	discover := discovery.NewDiscoveryClientForConfigOrDie(c).WithLegacy()
   327  	_, all, err := discover.ServerGroupsAndResources()
   328  	if err != nil {
   329  		t.Error(err)
   330  	}
   331  	var count int
   332  	apiResources := sets.NewString()
   333  	for _, g := range all {
   334  		for _, r := range g.APIResources {
   335  			apiResources.Insert(g.GroupVersion + "/" + r.Name)
   336  			if strings.Contains(r.Name, "/") ||
   337  				storageversionhashdata.NoStorageVersionHash.Has(g.GroupVersion+"/"+r.Name) {
   338  				if r.StorageVersionHash != "" {
   339  					t.Errorf("expect resource %s/%s to have empty storageVersionHash, got hash %q", g.GroupVersion, r.Name, r.StorageVersionHash)
   340  				}
   341  				continue
   342  			}
   343  			if r.StorageVersionHash == "" {
   344  				t.Errorf("expect the storageVersionHash of %s/%s to exist", g.GroupVersion, r.Name)
   345  				continue
   346  			}
   347  			// Uncomment the following line if you want to update storageversionhash/data.go
   348  			// fmt.Printf("\"%s/%s\": \"%s\",\n", g.GroupVersion, r.Name, r.StorageVersionHash)
   349  			expected := storageversionhashdata.GVRToStorageVersionHash[g.GroupVersion+"/"+r.Name]
   350  			if r.StorageVersionHash != expected {
   351  				t.Errorf("expect the storageVersionHash of %s/%s to be %q, got %q", g.GroupVersion, r.Name, expected, r.StorageVersionHash)
   352  			}
   353  			count++
   354  		}
   355  	}
   356  	if count != len(storageversionhashdata.GVRToStorageVersionHash) {
   357  		knownResources := sets.StringKeySet(storageversionhashdata.GVRToStorageVersionHash)
   358  		t.Errorf("please remove the redundant entries from GVRToStorageVersionHash: %v", knownResources.Difference(apiResources).List())
   359  	}
   360  }
   361  
   362  func TestNoAlphaVersionsEnabledByDefault(t *testing.T) {
   363  	config := DefaultAPIResourceConfigSource()
   364  	for gv, enable := range config.GroupVersionConfigs {
   365  		if enable && strings.Contains(gv.Version, "alpha") {
   366  			t.Errorf("Alpha API version %s enabled by default", gv.String())
   367  		}
   368  	}
   369  
   370  	for gvr, enabled := range config.ResourceConfigs {
   371  		if !strings.Contains(gvr.Version, "alpha") || !enabled {
   372  			continue
   373  		}
   374  
   375  		// we have enabled an alpha api by resource {g,v,r}, we also expect the
   376  		// alpha api by version {g,v} to be disabled. This is so a programmer
   377  		// remembers to add the new alpha version to alphaAPIGroupVersionsDisabledByDefault.
   378  		gr := gvr.GroupVersion()
   379  		if enabled, found := config.GroupVersionConfigs[gr]; !found || enabled {
   380  			t.Errorf("Alpha API version %q should be disabled by default", gr.String())
   381  		}
   382  	}
   383  }
   384  
   385  func TestNoBetaVersionsEnabledByDefault(t *testing.T) {
   386  	config := DefaultAPIResourceConfigSource()
   387  	for gv, enable := range config.GroupVersionConfigs {
   388  		if enable && strings.Contains(gv.Version, "beta") {
   389  			t.Errorf("Beta API version %s enabled by default", gv.String())
   390  		}
   391  	}
   392  
   393  	for gvr, enabled := range config.ResourceConfigs {
   394  		if !strings.Contains(gvr.Version, "beta") || !enabled {
   395  			continue
   396  		}
   397  
   398  		// we have enabled a beta api by resource {g,v,r}, we also expect the
   399  		// beta api by version {g,v} to be disabled. This is so a programmer
   400  		// remembers to add the new beta version to betaAPIGroupVersionsDisabledByDefault.
   401  		gr := gvr.GroupVersion()
   402  		if enabled, found := config.GroupVersionConfigs[gr]; !found || enabled {
   403  			t.Errorf("Beta API version %q should be disabled by default", gr.String())
   404  		}
   405  	}
   406  }
   407  
   408  func TestDefaultVars(t *testing.T) {
   409  	// stableAPIGroupVersionsEnabledByDefault should not contain beta or alpha
   410  	for i := range stableAPIGroupVersionsEnabledByDefault {
   411  		gv := stableAPIGroupVersionsEnabledByDefault[i]
   412  		if strings.Contains(gv.Version, "beta") || strings.Contains(gv.Version, "alpha") {
   413  			t.Errorf("stableAPIGroupVersionsEnabledByDefault should contain stable version, but found: %q", gv.String())
   414  		}
   415  	}
   416  
   417  	// legacyBetaEnabledByDefaultResources should contain only beta version
   418  	for i := range legacyBetaEnabledByDefaultResources {
   419  		gv := legacyBetaEnabledByDefaultResources[i]
   420  		if !strings.Contains(gv.Version, "beta") {
   421  			t.Errorf("legacyBetaEnabledByDefaultResources should contain beta version, but found: %q", gv.String())
   422  		}
   423  	}
   424  
   425  	// betaAPIGroupVersionsDisabledByDefault should contain only beta version
   426  	for i := range betaAPIGroupVersionsDisabledByDefault {
   427  		gv := betaAPIGroupVersionsDisabledByDefault[i]
   428  		if !strings.Contains(gv.Version, "beta") {
   429  			t.Errorf("betaAPIGroupVersionsDisabledByDefault should contain beta version, but found: %q", gv.String())
   430  		}
   431  	}
   432  
   433  	// alphaAPIGroupVersionsDisabledByDefault should contain only alpha version
   434  	for i := range alphaAPIGroupVersionsDisabledByDefault {
   435  		gv := alphaAPIGroupVersionsDisabledByDefault[i]
   436  		if !strings.Contains(gv.Version, "alpha") {
   437  			t.Errorf("alphaAPIGroupVersionsDisabledByDefault should contain alpha version, but found: %q", gv.String())
   438  		}
   439  	}
   440  }
   441  
   442  func TestNewBetaResourcesEnabledByDefault(t *testing.T) {
   443  	// legacyEnabledBetaResources is nearly a duplication from elsewhere.  This is intentional.  These types already have
   444  	// GA equivalents available and should therefore never have a beta enabled by default again.
   445  	legacyEnabledBetaResources := map[schema.GroupVersionResource]bool{
   446  		autoscalingapiv2beta1.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true,
   447  		autoscalingapiv2beta2.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true,
   448  		batchapiv1beta1.SchemeGroupVersion.WithResource("cronjobs"):                       true,
   449  		discoveryv1beta1.SchemeGroupVersion.WithResource("endpointslices"):                true,
   450  		eventsv1beta1.SchemeGroupVersion.WithResource("events"):                           true,
   451  		nodev1beta1.SchemeGroupVersion.WithResource("runtimeclasses"):                     true,
   452  		policyapiv1beta1.SchemeGroupVersion.WithResource("poddisruptionbudgets"):          true,
   453  		policyapiv1beta1.SchemeGroupVersion.WithResource("podsecuritypolicies"):           true,
   454  		storageapiv1beta1.SchemeGroupVersion.WithResource("csinodes"):                     true,
   455  	}
   456  
   457  	// legacyBetaResourcesWithoutStableEquivalents contains those groupresources that were enabled by default as beta
   458  	// before we changed that policy and do not have stable versions. These resources are allowed to have additional
   459  	// beta versions enabled by default.  Nothing new should be added here.  There are no future exceptions because there
   460  	// are no more beta resources enabled by default.
   461  	legacyBetaResourcesWithoutStableEquivalents := map[schema.GroupResource]bool{
   462  		flowcontrolv1bet3.SchemeGroupVersion.WithResource("flowschemas").GroupResource():                 true,
   463  		flowcontrolv1bet3.SchemeGroupVersion.WithResource("prioritylevelconfigurations").GroupResource(): true,
   464  	}
   465  
   466  	config := DefaultAPIResourceConfigSource()
   467  	for gvr, enable := range config.ResourceConfigs {
   468  		if !strings.Contains(gvr.Version, "beta") {
   469  			continue // only check beta things
   470  		}
   471  		if !enable {
   472  			continue // only check things that are enabled
   473  		}
   474  		if legacyEnabledBetaResources[gvr] {
   475  			continue // this is a legacy beta resource
   476  		}
   477  		if legacyBetaResourcesWithoutStableEquivalents[gvr.GroupResource()] {
   478  			continue // this is another beta of a legacy beta resource with no stable equivalent
   479  		}
   480  		t.Errorf("no new beta resources can be enabled by default, see https://github.com/kubernetes/enhancements/blob/0ad0fc8269165ca300d05ca51c7ce190a79976a5/keps/sig-architecture/3136-beta-apis-off-by-default/README.md: %v", gvr)
   481  	}
   482  }