k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controlplane/instance.go (about)

     1  /*
     2  Copyright 2014 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controlplane
    18  
    19  import (
    20  	"fmt"
    21  	"net"
    22  	"reflect"
    23  	"strconv"
    24  	"time"
    25  
    26  	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
    27  	admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1"
    28  	admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
    29  	apiserverinternalv1alpha1 "k8s.io/api/apiserverinternal/v1alpha1"
    30  	appsv1 "k8s.io/api/apps/v1"
    31  	authenticationv1 "k8s.io/api/authentication/v1"
    32  	authenticationv1alpha1 "k8s.io/api/authentication/v1alpha1"
    33  	authenticationv1beta1 "k8s.io/api/authentication/v1beta1"
    34  	authorizationapiv1 "k8s.io/api/authorization/v1"
    35  	autoscalingapiv1 "k8s.io/api/autoscaling/v1"
    36  	autoscalingapiv2 "k8s.io/api/autoscaling/v2"
    37  	batchapiv1 "k8s.io/api/batch/v1"
    38  	certificatesapiv1 "k8s.io/api/certificates/v1"
    39  	certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
    40  	coordinationapiv1 "k8s.io/api/coordination/v1"
    41  	apiv1 "k8s.io/api/core/v1"
    42  	discoveryv1 "k8s.io/api/discovery/v1"
    43  	eventsv1 "k8s.io/api/events/v1"
    44  	networkingapiv1 "k8s.io/api/networking/v1"
    45  	networkingapiv1alpha1 "k8s.io/api/networking/v1alpha1"
    46  	nodev1 "k8s.io/api/node/v1"
    47  	policyapiv1 "k8s.io/api/policy/v1"
    48  	rbacv1 "k8s.io/api/rbac/v1"
    49  	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
    50  	schedulingapiv1 "k8s.io/api/scheduling/v1"
    51  	storageapiv1 "k8s.io/api/storage/v1"
    52  	storageapiv1alpha1 "k8s.io/api/storage/v1alpha1"
    53  	storageapiv1beta1 "k8s.io/api/storage/v1beta1"
    54  	svmv1alpha1 "k8s.io/api/storagemigration/v1alpha1"
    55  	"k8s.io/apimachinery/pkg/runtime/schema"
    56  	utilnet "k8s.io/apimachinery/pkg/util/net"
    57  	"k8s.io/apiserver/pkg/endpoints/discovery"
    58  	genericapiserver "k8s.io/apiserver/pkg/server"
    59  	serverstorage "k8s.io/apiserver/pkg/server/storage"
    60  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    61  	"k8s.io/client-go/kubernetes"
    62  	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
    63  	discoveryclient "k8s.io/client-go/kubernetes/typed/discovery/v1"
    64  	"k8s.io/klog/v2"
    65  	api "k8s.io/kubernetes/pkg/apis/core"
    66  	flowcontrolv1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1"
    67  	flowcontrolv1beta1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta1"
    68  	flowcontrolv1beta2 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta2"
    69  	flowcontrolv1beta3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3"
    70  	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
    71  	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
    72  	"k8s.io/kubernetes/pkg/controlplane/controller/defaultservicecidr"
    73  	"k8s.io/kubernetes/pkg/controlplane/controller/kubernetesservice"
    74  	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
    75  	"k8s.io/kubernetes/pkg/features"
    76  	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
    77  	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
    78  
    79  	// RESTStorage installers
    80  	admissionregistrationrest "k8s.io/kubernetes/pkg/registry/admissionregistration/rest"
    81  	apiserverinternalrest "k8s.io/kubernetes/pkg/registry/apiserverinternal/rest"
    82  	appsrest "k8s.io/kubernetes/pkg/registry/apps/rest"
    83  	authenticationrest "k8s.io/kubernetes/pkg/registry/authentication/rest"
    84  	authorizationrest "k8s.io/kubernetes/pkg/registry/authorization/rest"
    85  	autoscalingrest "k8s.io/kubernetes/pkg/registry/autoscaling/rest"
    86  	batchrest "k8s.io/kubernetes/pkg/registry/batch/rest"
    87  	certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest"
    88  	coordinationrest "k8s.io/kubernetes/pkg/registry/coordination/rest"
    89  	corerest "k8s.io/kubernetes/pkg/registry/core/rest"
    90  	discoveryrest "k8s.io/kubernetes/pkg/registry/discovery/rest"
    91  	eventsrest "k8s.io/kubernetes/pkg/registry/events/rest"
    92  	flowcontrolrest "k8s.io/kubernetes/pkg/registry/flowcontrol/rest"
    93  	networkingrest "k8s.io/kubernetes/pkg/registry/networking/rest"
    94  	noderest "k8s.io/kubernetes/pkg/registry/node/rest"
    95  	policyrest "k8s.io/kubernetes/pkg/registry/policy/rest"
    96  	rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
    97  	resourcerest "k8s.io/kubernetes/pkg/registry/resource/rest"
    98  	schedulingrest "k8s.io/kubernetes/pkg/registry/scheduling/rest"
    99  	storagerest "k8s.io/kubernetes/pkg/registry/storage/rest"
   100  	svmrest "k8s.io/kubernetes/pkg/registry/storagemigration/rest"
   101  )
   102  
   103  const (
   104  	// DefaultEndpointReconcilerInterval is the default amount of time for how often the endpoints for
   105  	// the kubernetes Service are reconciled.
   106  	DefaultEndpointReconcilerInterval = 10 * time.Second
   107  	// DefaultEndpointReconcilerTTL is the default TTL timeout for the storage layer
   108  	DefaultEndpointReconcilerTTL = 15 * time.Second
   109  	// IdentityLeaseComponentLabelKey is used to apply a component label to identity lease objects, indicating:
   110  	//   1. the lease is an identity lease (different from leader election leases)
   111  	//   2. which component owns this lease
   112  	// TODO(sttts): remove this indirection
   113  	IdentityLeaseComponentLabelKey = controlplaneapiserver.IdentityLeaseComponentLabelKey
   114  	// KubeAPIServer defines variable used internally when referring to kube-apiserver component
   115  	KubeAPIServer = "kube-apiserver"
   116  	// repairLoopInterval defines the interval used to run the Services ClusterIP and NodePort repair loops
   117  	repairLoopInterval = 3 * time.Minute
   118  )
   119  
   120  // Extra defines extra configuration for kube-apiserver
   121  type Extra struct {
   122  	EndpointReconcilerConfig EndpointReconcilerConfig
   123  	KubeletClientConfig      kubeletclient.KubeletClientConfig
   124  
   125  	// Values to build the IP addresses used by discovery
   126  	// The range of IPs to be assigned to services with type=ClusterIP or greater
   127  	ServiceIPRange net.IPNet
   128  	// The IP address for the GenericAPIServer service (must be inside ServiceIPRange)
   129  	APIServerServiceIP net.IP
   130  
   131  	// dual stack services, the range represents an alternative IP range for service IP
   132  	// must be of different family than primary (ServiceIPRange)
   133  	SecondaryServiceIPRange net.IPNet
   134  	// the secondary IP address the GenericAPIServer service (must be inside SecondaryServiceIPRange)
   135  	SecondaryAPIServerServiceIP net.IP
   136  
   137  	// Port for the apiserver service.
   138  	APIServerServicePort int
   139  
   140  	// TODO, we can probably group service related items into a substruct to make it easier to configure
   141  	// the API server items and `Extra*` fields likely fit nicely together.
   142  
   143  	// The range of ports to be assigned to services with type=NodePort or greater
   144  	ServiceNodePortRange utilnet.PortRange
   145  	// If non-zero, the "kubernetes" services uses this port as NodePort.
   146  	KubernetesServiceNodePort int
   147  
   148  	// Number of masters running; all masters must be started with the
   149  	// same value for this field. (Numbers > 1 currently untested.)
   150  	MasterCount int
   151  
   152  	// MasterEndpointReconcileTTL sets the time to live in seconds of an
   153  	// endpoint record recorded by each master. The endpoints are checked at an
   154  	// interval that is 2/3 of this value and this value defaults to 15s if
   155  	// unset. In very large clusters, this value may be increased to reduce the
   156  	// possibility that the master endpoint record expires (due to other load
   157  	// on the etcd server) and causes masters to drop in and out of the
   158  	// kubernetes service record. It is not recommended to set this value below
   159  	// 15s.
   160  	MasterEndpointReconcileTTL time.Duration
   161  
   162  	// Selects which reconciler to use
   163  	EndpointReconcilerType reconcilers.Type
   164  
   165  	// RepairServicesInterval interval used by the repair loops for
   166  	// the Services NodePort and ClusterIP resources
   167  	RepairServicesInterval time.Duration
   168  }
   169  
   170  // Config defines configuration for the master
   171  type Config struct {
   172  	ControlPlane controlplaneapiserver.Config
   173  	Extra
   174  }
   175  
   176  type completedConfig struct {
   177  	ControlPlane controlplaneapiserver.CompletedConfig
   178  	*Extra
   179  }
   180  
   181  // CompletedConfig embeds a private pointer that cannot be instantiated outside of this package
   182  type CompletedConfig struct {
   183  	*completedConfig
   184  }
   185  
   186  // EndpointReconcilerConfig holds the endpoint reconciler and endpoint reconciliation interval to be
   187  // used by the master.
   188  type EndpointReconcilerConfig struct {
   189  	Reconciler reconcilers.EndpointReconciler
   190  	Interval   time.Duration
   191  }
   192  
   193  // Instance contains state for a Kubernetes cluster api server instance.
   194  type Instance struct {
   195  	ControlPlane *controlplaneapiserver.Server
   196  }
   197  
   198  func (c *Config) createMasterCountReconciler() reconcilers.EndpointReconciler {
   199  	endpointClient := corev1client.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig)
   200  	endpointSliceClient := discoveryclient.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig)
   201  	endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient)
   202  
   203  	return reconcilers.NewMasterCountEndpointReconciler(c.Extra.MasterCount, endpointsAdapter)
   204  }
   205  
   206  func (c *Config) createNoneReconciler() reconcilers.EndpointReconciler {
   207  	return reconcilers.NewNoneEndpointReconciler()
   208  }
   209  
   210  func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler {
   211  	endpointClient := corev1client.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig)
   212  	endpointSliceClient := discoveryclient.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig)
   213  	endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient)
   214  
   215  	ttl := c.Extra.MasterEndpointReconcileTTL
   216  	config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo"))
   217  	if err != nil {
   218  		klog.Fatalf("Error creating storage factory config: %v", err)
   219  	}
   220  	masterLeases, err := reconcilers.NewLeases(config, "/masterleases/", ttl)
   221  	if err != nil {
   222  		klog.Fatalf("Error creating leases: %v", err)
   223  	}
   224  
   225  	return reconcilers.NewLeaseEndpointReconciler(endpointsAdapter, masterLeases)
   226  }
   227  
   228  func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler {
   229  	klog.Infof("Using reconciler: %v", c.Extra.EndpointReconcilerType)
   230  	switch c.Extra.EndpointReconcilerType {
   231  	// there are numerous test dependencies that depend on a default controller
   232  	case reconcilers.MasterCountReconcilerType:
   233  		return c.createMasterCountReconciler()
   234  	case "", reconcilers.LeaseEndpointReconcilerType:
   235  		return c.createLeaseReconciler()
   236  	case reconcilers.NoneEndpointReconcilerType:
   237  		return c.createNoneReconciler()
   238  	default:
   239  		klog.Fatalf("Reconciler not implemented: %v", c.Extra.EndpointReconcilerType)
   240  	}
   241  	return nil
   242  }
   243  
   244  // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
   245  func (c *Config) Complete() CompletedConfig {
   246  	if c.ControlPlane.PeerEndpointReconcileInterval == 0 && c.EndpointReconcilerConfig.Interval != 0 {
   247  		// default this to the endpoint reconciler value before the generic
   248  		// controlplane completion can kick in
   249  		c.ControlPlane.PeerEndpointReconcileInterval = c.EndpointReconcilerConfig.Interval
   250  	}
   251  
   252  	cfg := completedConfig{
   253  		c.ControlPlane.Complete(),
   254  		&c.Extra,
   255  	}
   256  
   257  	serviceIPRange, apiServerServiceIP, err := options.ServiceIPRange(cfg.Extra.ServiceIPRange)
   258  	if err != nil {
   259  		klog.Fatalf("Error determining service IP ranges: %v", err)
   260  	}
   261  	if cfg.Extra.ServiceIPRange.IP == nil {
   262  		cfg.Extra.ServiceIPRange = serviceIPRange
   263  	}
   264  	if cfg.Extra.APIServerServiceIP == nil {
   265  		cfg.Extra.APIServerServiceIP = apiServerServiceIP
   266  	}
   267  
   268  	// override the default discovery addresses in the generic controlplane adding service IP support
   269  	discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: cfg.ControlPlane.Generic.ExternalAddress}
   270  	discoveryAddresses.CIDRRules = append(discoveryAddresses.CIDRRules,
   271  		discovery.CIDRRule{IPRange: cfg.Extra.ServiceIPRange, Address: net.JoinHostPort(cfg.Extra.APIServerServiceIP.String(), strconv.Itoa(cfg.Extra.APIServerServicePort))})
   272  	cfg.ControlPlane.Generic.DiscoveryAddresses = discoveryAddresses
   273  
   274  	if cfg.Extra.ServiceNodePortRange.Size == 0 {
   275  		// TODO: Currently no way to specify an empty range (do we need to allow this?)
   276  		// We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE)
   277  		// but then that breaks the strict nestedness of ServiceType.
   278  		// Review post-v1
   279  		cfg.Extra.ServiceNodePortRange = kubeoptions.DefaultServiceNodePortRange
   280  		klog.Infof("Node port range unspecified. Defaulting to %v.", cfg.Extra.ServiceNodePortRange)
   281  	}
   282  
   283  	if cfg.Extra.EndpointReconcilerConfig.Interval == 0 {
   284  		cfg.Extra.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval
   285  	}
   286  
   287  	if cfg.Extra.MasterEndpointReconcileTTL == 0 {
   288  		cfg.Extra.MasterEndpointReconcileTTL = DefaultEndpointReconcilerTTL
   289  	}
   290  
   291  	if cfg.Extra.EndpointReconcilerConfig.Reconciler == nil {
   292  		cfg.Extra.EndpointReconcilerConfig.Reconciler = c.createEndpointReconciler()
   293  	}
   294  
   295  	if cfg.Extra.RepairServicesInterval == 0 {
   296  		cfg.Extra.RepairServicesInterval = repairLoopInterval
   297  	}
   298  
   299  	return CompletedConfig{&cfg}
   300  }
   301  
   302  // New returns a new instance of Master from the given config.
   303  // Certain config fields will be set to a default value if unset.
   304  // Certain config fields must be specified, including:
   305  // KubeletClientConfig
   306  func (c CompletedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) {
   307  	if reflect.DeepEqual(c.Extra.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) {
   308  		return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig")
   309  	}
   310  
   311  	cp, err := c.ControlPlane.New(KubeAPIServer, delegationTarget)
   312  	if err != nil {
   313  		return nil, err
   314  	}
   315  
   316  	s := &Instance{
   317  		ControlPlane: cp,
   318  	}
   319  
   320  	client, err := kubernetes.NewForConfig(c.ControlPlane.Generic.LoopbackClientConfig)
   321  	if err != nil {
   322  		return nil, err
   323  	}
   324  
   325  	// TODO: update to a version that caches success but will recheck on failure, unlike memcache discovery
   326  	discoveryClientForAdmissionRegistration := client.Discovery()
   327  
   328  	legacyRESTStorageProvider, err := corerest.New(corerest.Config{
   329  		GenericConfig: corerest.GenericConfig{
   330  			StorageFactory:              c.ControlPlane.Extra.StorageFactory,
   331  			EventTTL:                    c.ControlPlane.Extra.EventTTL,
   332  			LoopbackClientConfig:        c.ControlPlane.Generic.LoopbackClientConfig,
   333  			ServiceAccountIssuer:        c.ControlPlane.Extra.ServiceAccountIssuer,
   334  			ExtendExpiration:            c.ControlPlane.Extra.ExtendExpiration,
   335  			ServiceAccountMaxExpiration: c.ControlPlane.Extra.ServiceAccountMaxExpiration,
   336  			APIAudiences:                c.ControlPlane.Generic.Authentication.APIAudiences,
   337  			Informers:                   c.ControlPlane.Extra.VersionedInformers,
   338  		},
   339  		Proxy: corerest.ProxyConfig{
   340  			Transport:           c.ControlPlane.Extra.ProxyTransport,
   341  			KubeletClientConfig: c.Extra.KubeletClientConfig,
   342  		},
   343  		Services: corerest.ServicesConfig{
   344  			ClusterIPRange:          c.Extra.ServiceIPRange,
   345  			SecondaryClusterIPRange: c.Extra.SecondaryServiceIPRange,
   346  			NodePortRange:           c.Extra.ServiceNodePortRange,
   347  			IPRepairInterval:        c.Extra.RepairServicesInterval,
   348  		},
   349  	})
   350  	if err != nil {
   351  		return nil, err
   352  	}
   353  
   354  	// The order here is preserved in discovery.
   355  	// If resources with identical names exist in more than one of these groups (e.g. "deployments.apps"" and "deployments.extensions"),
   356  	// the order of this list determines which group an unqualified resource name (e.g. "deployments") should prefer.
   357  	// This priority order is used for local discovery, but it ends up aggregated in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go
   358  	// with specific priorities.
   359  	// TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery
   360  	// handlers that we have.
   361  	restStorageProviders := []controlplaneapiserver.RESTStorageProvider{
   362  		legacyRESTStorageProvider,
   363  		apiserverinternalrest.StorageProvider{},
   364  		authenticationrest.RESTStorageProvider{Authenticator: c.ControlPlane.Generic.Authentication.Authenticator, APIAudiences: c.ControlPlane.Generic.Authentication.APIAudiences},
   365  		authorizationrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer, RuleResolver: c.ControlPlane.Generic.RuleResolver},
   366  		autoscalingrest.RESTStorageProvider{},
   367  		batchrest.RESTStorageProvider{},
   368  		certificatesrest.RESTStorageProvider{},
   369  		coordinationrest.RESTStorageProvider{},
   370  		discoveryrest.StorageProvider{},
   371  		networkingrest.RESTStorageProvider{},
   372  		noderest.RESTStorageProvider{},
   373  		policyrest.RESTStorageProvider{},
   374  		rbacrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer},
   375  		schedulingrest.RESTStorageProvider{},
   376  		storagerest.RESTStorageProvider{},
   377  		svmrest.RESTStorageProvider{},
   378  		flowcontrolrest.RESTStorageProvider{InformerFactory: c.ControlPlane.Generic.SharedInformerFactory},
   379  		// keep apps after extensions so legacy clients resolve the extensions versions of shared resource names.
   380  		// See https://github.com/kubernetes/kubernetes/issues/42392
   381  		appsrest.StorageProvider{},
   382  		admissionregistrationrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer, DiscoveryClient: discoveryClientForAdmissionRegistration},
   383  		eventsrest.RESTStorageProvider{TTL: c.ControlPlane.EventTTL},
   384  		resourcerest.RESTStorageProvider{},
   385  	}
   386  	if err := s.ControlPlane.InstallAPIs(restStorageProviders...); err != nil {
   387  		return nil, err
   388  	}
   389  
   390  	_, publicServicePort, err := c.ControlPlane.Generic.SecureServing.HostPort()
   391  	if err != nil {
   392  		return nil, fmt.Errorf("failed to get listener address: %w", err)
   393  	}
   394  	kubernetesServiceCtrl := kubernetesservice.New(kubernetesservice.Config{
   395  		PublicIP: c.ControlPlane.Generic.PublicAddress,
   396  
   397  		EndpointReconciler: c.Extra.EndpointReconcilerConfig.Reconciler,
   398  		EndpointInterval:   c.Extra.EndpointReconcilerConfig.Interval,
   399  
   400  		ServiceIP:                 c.Extra.APIServerServiceIP,
   401  		ServicePort:               c.Extra.APIServerServicePort,
   402  		PublicServicePort:         publicServicePort,
   403  		KubernetesServiceNodePort: c.Extra.KubernetesServiceNodePort,
   404  	}, client, c.ControlPlane.Extra.VersionedInformers.Core().V1().Services())
   405  	s.ControlPlane.GenericAPIServer.AddPostStartHookOrDie("bootstrap-controller", func(hookContext genericapiserver.PostStartHookContext) error {
   406  		kubernetesServiceCtrl.Start(hookContext.StopCh)
   407  		return nil
   408  	})
   409  	s.ControlPlane.GenericAPIServer.AddPreShutdownHookOrDie("stop-kubernetes-service-controller", func() error {
   410  		kubernetesServiceCtrl.Stop()
   411  		return nil
   412  	})
   413  
   414  	if utilfeature.DefaultFeatureGate.Enabled(features.MultiCIDRServiceAllocator) {
   415  		s.ControlPlane.GenericAPIServer.AddPostStartHookOrDie("start-kubernetes-service-cidr-controller", func(hookContext genericapiserver.PostStartHookContext) error {
   416  			controller := defaultservicecidr.NewController(
   417  				c.Extra.ServiceIPRange,
   418  				c.Extra.SecondaryServiceIPRange,
   419  				client,
   420  			)
   421  			// The default serviceCIDR must exist before the apiserver is healthy
   422  			// otherwise the allocators for Services will not work.
   423  			controller.Start(hookContext)
   424  			return nil
   425  		})
   426  	}
   427  
   428  	return s, nil
   429  }
   430  
   431  var (
   432  	// stableAPIGroupVersionsEnabledByDefault is a list of our stable versions.
   433  	stableAPIGroupVersionsEnabledByDefault = []schema.GroupVersion{
   434  		admissionregistrationv1.SchemeGroupVersion,
   435  		apiv1.SchemeGroupVersion,
   436  		appsv1.SchemeGroupVersion,
   437  		authenticationv1.SchemeGroupVersion,
   438  		authorizationapiv1.SchemeGroupVersion,
   439  		autoscalingapiv1.SchemeGroupVersion,
   440  		autoscalingapiv2.SchemeGroupVersion,
   441  		batchapiv1.SchemeGroupVersion,
   442  		certificatesapiv1.SchemeGroupVersion,
   443  		coordinationapiv1.SchemeGroupVersion,
   444  		discoveryv1.SchemeGroupVersion,
   445  		eventsv1.SchemeGroupVersion,
   446  		networkingapiv1.SchemeGroupVersion,
   447  		nodev1.SchemeGroupVersion,
   448  		policyapiv1.SchemeGroupVersion,
   449  		rbacv1.SchemeGroupVersion,
   450  		storageapiv1.SchemeGroupVersion,
   451  		schedulingapiv1.SchemeGroupVersion,
   452  		flowcontrolv1.SchemeGroupVersion,
   453  	}
   454  
   455  	// legacyBetaEnabledByDefaultResources is the list of beta resources we enable.  You may only add to this list
   456  	// if your resource is already enabled by default in a beta level we still serve AND there is no stable API for it.
   457  	// see https://github.com/kubernetes/enhancements/tree/master/keps/sig-architecture/3136-beta-apis-off-by-default
   458  	// for more details.
   459  	legacyBetaEnabledByDefaultResources = []schema.GroupVersionResource{
   460  		flowcontrolv1beta3.SchemeGroupVersion.WithResource("flowschemas"),                 // deprecate in 1.29, remove in 1.32
   461  		flowcontrolv1beta3.SchemeGroupVersion.WithResource("prioritylevelconfigurations"), // deprecate in 1.29, remove in 1.32
   462  	}
   463  	// betaAPIGroupVersionsDisabledByDefault is for all future beta groupVersions.
   464  	betaAPIGroupVersionsDisabledByDefault = []schema.GroupVersion{
   465  		admissionregistrationv1beta1.SchemeGroupVersion,
   466  		authenticationv1beta1.SchemeGroupVersion,
   467  		storageapiv1beta1.SchemeGroupVersion,
   468  		flowcontrolv1beta1.SchemeGroupVersion,
   469  		flowcontrolv1beta2.SchemeGroupVersion,
   470  		flowcontrolv1beta3.SchemeGroupVersion,
   471  	}
   472  
   473  	// alphaAPIGroupVersionsDisabledByDefault holds the alpha APIs we have.  They are always disabled by default.
   474  	alphaAPIGroupVersionsDisabledByDefault = []schema.GroupVersion{
   475  		admissionregistrationv1alpha1.SchemeGroupVersion,
   476  		apiserverinternalv1alpha1.SchemeGroupVersion,
   477  		authenticationv1alpha1.SchemeGroupVersion,
   478  		resourcev1alpha2.SchemeGroupVersion,
   479  		certificatesv1alpha1.SchemeGroupVersion,
   480  		networkingapiv1alpha1.SchemeGroupVersion,
   481  		storageapiv1alpha1.SchemeGroupVersion,
   482  		svmv1alpha1.SchemeGroupVersion,
   483  	}
   484  )
   485  
   486  // DefaultAPIResourceConfigSource returns default configuration for an APIResource.
   487  func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig {
   488  	ret := serverstorage.NewResourceConfig()
   489  	// NOTE: GroupVersions listed here will be enabled by default. Don't put alpha or beta versions in the list.
   490  	ret.EnableVersions(stableAPIGroupVersionsEnabledByDefault...)
   491  
   492  	// disable alpha and beta versions explicitly so we have a full list of what's possible to serve
   493  	ret.DisableVersions(betaAPIGroupVersionsDisabledByDefault...)
   494  	ret.DisableVersions(alphaAPIGroupVersionsDisabledByDefault...)
   495  
   496  	// enable the legacy beta resources that were present before stopped serving new beta APIs by default.
   497  	ret.EnableResources(legacyBetaEnabledByDefaultResources...)
   498  
   499  	return ret
   500  }