k8s.io/apiserver@v0.31.1/pkg/server/config.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package server
    18  
    19  import (
    20  	"context"
    21  	"crypto/sha256"
    22  	"encoding/base32"
    23  	"fmt"
    24  	"net"
    25  	"net/http"
    26  	"os"
    27  	goruntime "runtime"
    28  	"runtime/debug"
    29  	"sort"
    30  	"strconv"
    31  	"strings"
    32  	"sync/atomic"
    33  	"time"
    34  
    35  	"github.com/google/uuid"
    36  	"golang.org/x/crypto/cryptobyte"
    37  	jsonpatch "gopkg.in/evanphx/json-patch.v4"
    38  
    39  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    40  	"k8s.io/apimachinery/pkg/runtime"
    41  	"k8s.io/apimachinery/pkg/runtime/schema"
    42  	"k8s.io/apimachinery/pkg/runtime/serializer"
    43  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    44  	"k8s.io/apimachinery/pkg/util/sets"
    45  	"k8s.io/apimachinery/pkg/util/version"
    46  	utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
    47  	"k8s.io/apiserver/pkg/admission"
    48  	"k8s.io/apiserver/pkg/audit"
    49  	"k8s.io/apiserver/pkg/authentication/authenticator"
    50  	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
    51  	authenticatorunion "k8s.io/apiserver/pkg/authentication/request/union"
    52  	"k8s.io/apiserver/pkg/authentication/user"
    53  	"k8s.io/apiserver/pkg/authorization/authorizer"
    54  	"k8s.io/apiserver/pkg/endpoints/discovery"
    55  	discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
    56  	"k8s.io/apiserver/pkg/endpoints/filterlatency"
    57  	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
    58  	apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
    59  	apirequest "k8s.io/apiserver/pkg/endpoints/request"
    60  	genericfeatures "k8s.io/apiserver/pkg/features"
    61  	genericregistry "k8s.io/apiserver/pkg/registry/generic"
    62  	"k8s.io/apiserver/pkg/server/dynamiccertificates"
    63  	"k8s.io/apiserver/pkg/server/egressselector"
    64  	genericfilters "k8s.io/apiserver/pkg/server/filters"
    65  	"k8s.io/apiserver/pkg/server/healthz"
    66  	"k8s.io/apiserver/pkg/server/routes"
    67  	"k8s.io/apiserver/pkg/server/routine"
    68  	serverstore "k8s.io/apiserver/pkg/server/storage"
    69  	storagevalue "k8s.io/apiserver/pkg/storage/value"
    70  	"k8s.io/apiserver/pkg/storageversion"
    71  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    72  	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
    73  	flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
    74  	utilversion "k8s.io/apiserver/pkg/util/version"
    75  	"k8s.io/client-go/informers"
    76  	restclient "k8s.io/client-go/rest"
    77  	"k8s.io/component-base/featuregate"
    78  	"k8s.io/component-base/logs"
    79  	"k8s.io/component-base/metrics/features"
    80  	"k8s.io/component-base/metrics/prometheus/slis"
    81  	"k8s.io/component-base/tracing"
    82  	"k8s.io/klog/v2"
    83  	openapicommon "k8s.io/kube-openapi/pkg/common"
    84  	"k8s.io/kube-openapi/pkg/spec3"
    85  	"k8s.io/kube-openapi/pkg/validation/spec"
    86  	"k8s.io/utils/clock"
    87  	utilsnet "k8s.io/utils/net"
    88  
    89  	// install apis
    90  	_ "k8s.io/apiserver/pkg/apis/apiserver/install"
    91  )
    92  
    93  // hostnameFunc is a function to set the hostnameFunc of this apiserver.
    94  // To be used for testing purpose only, to simulate scenarios where multiple apiservers
    95  // exist. In such cases we want to ensure unique apiserver IDs which are a hash of hostnameFunc.
    96  var (
    97  	hostnameFunc = os.Hostname
    98  )
    99  
   100  const (
   101  	// DefaultLegacyAPIPrefix is where the legacy APIs will be located.
   102  	DefaultLegacyAPIPrefix = "/api"
   103  
   104  	// APIGroupPrefix is where non-legacy API group will be located.
   105  	APIGroupPrefix = "/apis"
   106  )
   107  
   108  // Config is a structure used to configure a GenericAPIServer.
   109  // Its members are sorted roughly in order of importance for composers.
   110  type Config struct {
   111  	// SecureServing is required to serve https
   112  	SecureServing *SecureServingInfo
   113  
   114  	// Authentication is the configuration for authentication
   115  	Authentication AuthenticationInfo
   116  
   117  	// Authorization is the configuration for authorization
   118  	Authorization AuthorizationInfo
   119  
   120  	// LoopbackClientConfig is a config for a privileged loopback connection to the API server
   121  	// This is required for proper functioning of the PostStartHooks on a GenericAPIServer
   122  	// TODO: move into SecureServing(WithLoopback) as soon as insecure serving is gone
   123  	LoopbackClientConfig *restclient.Config
   124  
   125  	// EgressSelector provides a lookup mechanism for dialing outbound connections.
   126  	// It does so based on a EgressSelectorConfiguration which was read at startup.
   127  	EgressSelector *egressselector.EgressSelector
   128  
   129  	// RuleResolver is required to get the list of rules that apply to a given user
   130  	// in a given namespace
   131  	RuleResolver authorizer.RuleResolver
   132  	// AdmissionControl performs deep inspection of a given request (including content)
   133  	// to set values and determine whether its allowed
   134  	AdmissionControl      admission.Interface
   135  	CorsAllowedOriginList []string
   136  	HSTSDirectives        []string
   137  	// FlowControl, if not nil, gives priority and fairness to request handling
   138  	FlowControl utilflowcontrol.Interface
   139  
   140  	EnableIndex     bool
   141  	EnableProfiling bool
   142  	DebugSocketPath string
   143  	EnableDiscovery bool
   144  
   145  	// Requires generic profiling enabled
   146  	EnableContentionProfiling bool
   147  	EnableMetrics             bool
   148  
   149  	DisabledPostStartHooks sets.String
   150  	// done values in this values for this map are ignored.
   151  	PostStartHooks map[string]PostStartHookConfigEntry
   152  
   153  	// EffectiveVersion determines which apis and features are available
   154  	// based on when the api/feature lifecyle.
   155  	EffectiveVersion utilversion.EffectiveVersion
   156  	// FeatureGate is a way to plumb feature gate through if you have them.
   157  	FeatureGate featuregate.FeatureGate
   158  	// AuditBackend is where audit events are sent to.
   159  	AuditBackend audit.Backend
   160  	// AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request.
   161  	AuditPolicyRuleEvaluator audit.PolicyRuleEvaluator
   162  	// ExternalAddress is the host name to use for external (public internet) facing URLs (e.g. Swagger)
   163  	// Will default to a value based on secure serving info and available ipv4 IPs.
   164  	ExternalAddress string
   165  
   166  	// TracerProvider can provide a tracer, which records spans for distributed tracing.
   167  	TracerProvider tracing.TracerProvider
   168  
   169  	//===========================================================================
   170  	// Fields you probably don't care about changing
   171  	//===========================================================================
   172  
   173  	// BuildHandlerChainFunc allows you to build custom handler chains by decorating the apiHandler.
   174  	BuildHandlerChainFunc func(apiHandler http.Handler, c *Config) (secure http.Handler)
   175  	// NonLongRunningRequestWaitGroup allows you to wait for all chain
   176  	// handlers associated with non long-running requests
   177  	// to complete while the server is shuting down.
   178  	NonLongRunningRequestWaitGroup *utilwaitgroup.SafeWaitGroup
   179  	// WatchRequestWaitGroup allows us to wait for all chain
   180  	// handlers associated with active watch requests to
   181  	// complete while the server is shuting down.
   182  	WatchRequestWaitGroup *utilwaitgroup.RateLimitedSafeWaitGroup
   183  	// DiscoveryAddresses is used to build the IPs pass to discovery. If nil, the ExternalAddress is
   184  	// always reported
   185  	DiscoveryAddresses discovery.Addresses
   186  	// The default set of healthz checks. There might be more added via AddHealthChecks dynamically.
   187  	HealthzChecks []healthz.HealthChecker
   188  	// The default set of livez checks. There might be more added via AddHealthChecks dynamically.
   189  	LivezChecks []healthz.HealthChecker
   190  	// The default set of readyz-only checks. There might be more added via AddReadyzChecks dynamically.
   191  	ReadyzChecks []healthz.HealthChecker
   192  	// LegacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
   193  	// to InstallLegacyAPIGroup. New API servers don't generally have legacy groups at all.
   194  	LegacyAPIGroupPrefixes sets.String
   195  	// RequestInfoResolver is used to assign attributes (used by admission and authorization) based on a request URL.
   196  	// Use-cases that are like kubelets may need to customize this.
   197  	RequestInfoResolver apirequest.RequestInfoResolver
   198  	// Serializer is required and provides the interface for serializing and converting objects to and from the wire
   199  	// The default (api.Codecs) usually works fine.
   200  	Serializer runtime.NegotiatedSerializer
   201  	// OpenAPIConfig will be used in generating OpenAPI spec. This is nil by default. Use DefaultOpenAPIConfig for "working" defaults.
   202  	OpenAPIConfig *openapicommon.Config
   203  	// OpenAPIV3Config will be used in generating OpenAPI V3 spec. This is nil by default. Use DefaultOpenAPIV3Config for "working" defaults.
   204  	OpenAPIV3Config *openapicommon.OpenAPIV3Config
   205  	// SkipOpenAPIInstallation avoids installing the OpenAPI handler if set to true.
   206  	SkipOpenAPIInstallation bool
   207  
   208  	// ResourceTransformers are used to transform resources from and to etcd, e.g. encryption.
   209  	ResourceTransformers storagevalue.ResourceTransformers
   210  	// RESTOptionsGetter is used to construct RESTStorage types via the generic registry.
   211  	RESTOptionsGetter genericregistry.RESTOptionsGetter
   212  
   213  	// If specified, all requests except those which match the LongRunningFunc predicate will timeout
   214  	// after this duration.
   215  	RequestTimeout time.Duration
   216  	// If specified, long running requests such as watch will be allocated a random timeout between this value, and
   217  	// twice this value.  Note that it is up to the request handlers to ignore or honor this timeout. In seconds.
   218  	MinRequestTimeout int
   219  
   220  	// StorageInitializationTimeout defines the maximum amount of time to wait for storage initialization
   221  	// before declaring apiserver ready.
   222  	StorageInitializationTimeout time.Duration
   223  
   224  	// This represents the maximum amount of time it should take for apiserver to complete its startup
   225  	// sequence and become healthy. From apiserver's start time to when this amount of time has
   226  	// elapsed, /livez will assume that unfinished post-start hooks will complete successfully and
   227  	// therefore return true.
   228  	LivezGracePeriod time.Duration
   229  	// ShutdownDelayDuration allows to block shutdown for some time, e.g. until endpoints pointing to this API server
   230  	// have converged on all node. During this time, the API server keeps serving, /healthz will return 200,
   231  	// but /readyz will return failure.
   232  	ShutdownDelayDuration time.Duration
   233  
   234  	// The limit on the total size increase all "copy" operations in a json
   235  	// patch may cause.
   236  	// This affects all places that applies json patch in the binary.
   237  	JSONPatchMaxCopyBytes int64
   238  	// The limit on the request size that would be accepted and decoded in a write request
   239  	// 0 means no limit.
   240  	MaxRequestBodyBytes int64
   241  	// MaxRequestsInFlight is the maximum number of parallel non-long-running requests. Every further
   242  	// request has to wait. Applies only to non-mutating requests.
   243  	MaxRequestsInFlight int
   244  	// MaxMutatingRequestsInFlight is the maximum number of parallel mutating requests. Every further
   245  	// request has to wait.
   246  	MaxMutatingRequestsInFlight int
   247  	// Predicate which is true for paths of long-running http requests
   248  	LongRunningFunc apirequest.LongRunningRequestCheck
   249  
   250  	// GoawayChance is the probability that send a GOAWAY to HTTP/2 clients. When client received
   251  	// GOAWAY, the in-flight requests will not be affected and new requests will use
   252  	// a new TCP connection to triggering re-balancing to another server behind the load balance.
   253  	// Default to 0, means never send GOAWAY. Max is 0.02 to prevent break the apiserver.
   254  	GoawayChance float64
   255  
   256  	// MergedResourceConfig indicates which groupVersion enabled and its resources enabled/disabled.
   257  	// This is composed of genericapiserver defaultAPIResourceConfig and those parsed from flags.
   258  	// If not specify any in flags, then genericapiserver will only enable defaultAPIResourceConfig.
   259  	MergedResourceConfig *serverstore.ResourceConfig
   260  
   261  	// lifecycleSignals provides access to the various signals
   262  	// that happen during lifecycle of the apiserver.
   263  	// it's intentionally marked private as it should never be overridden.
   264  	lifecycleSignals lifecycleSignals
   265  
   266  	// StorageObjectCountTracker is used to keep track of the total number of objects
   267  	// in the storage per resource, so we can estimate width of incoming requests.
   268  	StorageObjectCountTracker flowcontrolrequest.StorageObjectCountTracker
   269  
   270  	// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
   271  	// Server during the graceful termination of the apiserver. If true, we wait
   272  	// for non longrunning requests in flight to be drained and then initiate a
   273  	// shutdown of the HTTP Server. If false, we initiate a shutdown of the HTTP
   274  	// Server as soon as ShutdownDelayDuration has elapsed.
   275  	// If enabled, after ShutdownDelayDuration elapses, any incoming request is
   276  	// rejected with a 429 status code and a 'Retry-After' response.
   277  	ShutdownSendRetryAfter bool
   278  
   279  	//===========================================================================
   280  	// values below here are targets for removal
   281  	//===========================================================================
   282  
   283  	// PublicAddress is the IP address where members of the cluster (kubelet,
   284  	// kube-proxy, services, etc.) can reach the GenericAPIServer.
   285  	// If nil or 0.0.0.0, the host's default interface will be used.
   286  	PublicAddress net.IP
   287  
   288  	// EquivalentResourceRegistry provides information about resources equivalent to a given resource,
   289  	// and the kind associated with a given resource. As resources are installed, they are registered here.
   290  	EquivalentResourceRegistry runtime.EquivalentResourceRegistry
   291  
   292  	// APIServerID is the ID of this API server
   293  	APIServerID string
   294  
   295  	// StorageVersionManager holds the storage versions of the API resources installed by this server.
   296  	StorageVersionManager storageversion.Manager
   297  
   298  	// AggregatedDiscoveryGroupManager serves /apis in an aggregated form.
   299  	AggregatedDiscoveryGroupManager discoveryendpoint.ResourceManager
   300  
   301  	// ShutdownWatchTerminationGracePeriod, if set to a positive value,
   302  	// is the maximum duration the apiserver will wait for all active
   303  	// watch request(s) to drain.
   304  	// Once this grace period elapses, the apiserver will no longer
   305  	// wait for any active watch request(s) in flight to drain, it will
   306  	// proceed to the next step in the graceful server shutdown process.
   307  	// If set to a positive value, the apiserver will keep track of the
   308  	// number of active watch request(s) in flight and during shutdown
   309  	// it will wait, at most, for the specified duration and allow these
   310  	// active watch requests to drain with some rate limiting in effect.
   311  	// The default is zero, which implies the apiserver will not keep
   312  	// track of active watch request(s) in flight and will not wait
   313  	// for them to drain, this maintains backward compatibility.
   314  	// This grace period is orthogonal to other grace periods, and
   315  	// it is not overridden by any other grace period.
   316  	ShutdownWatchTerminationGracePeriod time.Duration
   317  }
   318  
   319  type RecommendedConfig struct {
   320  	Config
   321  
   322  	// SharedInformerFactory provides shared informers for Kubernetes resources. This value is set by
   323  	// RecommendedOptions.CoreAPI.ApplyTo called by RecommendedOptions.ApplyTo. It uses an in-cluster client config
   324  	// by default, or the kubeconfig given with kubeconfig command line flag.
   325  	SharedInformerFactory informers.SharedInformerFactory
   326  
   327  	// ClientConfig holds the kubernetes client configuration.
   328  	// This value is set by RecommendedOptions.CoreAPI.ApplyTo called by RecommendedOptions.ApplyTo.
   329  	// By default in-cluster client config is used.
   330  	ClientConfig *restclient.Config
   331  }
   332  
   333  type SecureServingInfo struct {
   334  	// Listener is the secure server network listener.
   335  	Listener net.Listener
   336  
   337  	// Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is
   338  	// allowed to be in SNICerts.
   339  	Cert dynamiccertificates.CertKeyContentProvider
   340  
   341  	// SNICerts are the TLS certificates used for SNI.
   342  	SNICerts []dynamiccertificates.SNICertKeyContentProvider
   343  
   344  	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
   345  	ClientCA dynamiccertificates.CAContentProvider
   346  
   347  	// MinTLSVersion optionally overrides the minimum TLS version supported.
   348  	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
   349  	MinTLSVersion uint16
   350  
   351  	// CipherSuites optionally overrides the list of allowed cipher suites for the server.
   352  	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
   353  	CipherSuites []uint16
   354  
   355  	// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
   356  	// A value of zero means to use the default provided by golang's HTTP/2 support.
   357  	HTTP2MaxStreamsPerConnection int
   358  
   359  	// DisableHTTP2 indicates that http2 should not be enabled.
   360  	DisableHTTP2 bool
   361  }
   362  
   363  type AuthenticationInfo struct {
   364  	// APIAudiences is a list of identifier that the API identifies as. This is
   365  	// used by some authenticators to validate audience bound credentials.
   366  	APIAudiences authenticator.Audiences
   367  	// Authenticator determines which subject is making the request
   368  	Authenticator authenticator.Request
   369  
   370  	RequestHeaderConfig *authenticatorfactory.RequestHeaderConfig
   371  }
   372  
   373  type AuthorizationInfo struct {
   374  	// Authorizer determines whether the subject is allowed to make the request based only
   375  	// on the RequestURI
   376  	Authorizer authorizer.Authorizer
   377  }
   378  
   379  func init() {
   380  	utilruntime.Must(features.AddFeatureGates(utilfeature.DefaultMutableFeatureGate))
   381  }
   382  
   383  // NewConfig returns a Config struct with the default values
   384  func NewConfig(codecs serializer.CodecFactory) *Config {
   385  	defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}
   386  	var id string
   387  	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
   388  		hostname, err := hostnameFunc()
   389  		if err != nil {
   390  			klog.Fatalf("error getting hostname for apiserver identity: %v", err)
   391  		}
   392  
   393  		// Since the hash needs to be unique across each kube-apiserver and aggregated apiservers,
   394  		// the hash used for the identity should include both the hostname and the identity value.
   395  		// TODO: receive the identity value as a parameter once the apiserver identity lease controller
   396  		// post start hook is moved to generic apiserver.
   397  		b := cryptobyte.NewBuilder(nil)
   398  		b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
   399  			b.AddBytes([]byte(hostname))
   400  		})
   401  		b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
   402  			b.AddBytes([]byte("kube-apiserver"))
   403  		})
   404  		hashData, err := b.Bytes()
   405  		if err != nil {
   406  			klog.Fatalf("error building hash data for apiserver identity: %v", err)
   407  		}
   408  
   409  		hash := sha256.Sum256(hashData)
   410  		id = "apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16]))
   411  	}
   412  	lifecycleSignals := newLifecycleSignals()
   413  
   414  	return &Config{
   415  		Serializer:                     codecs,
   416  		BuildHandlerChainFunc:          DefaultBuildHandlerChain,
   417  		NonLongRunningRequestWaitGroup: new(utilwaitgroup.SafeWaitGroup),
   418  		WatchRequestWaitGroup:          &utilwaitgroup.RateLimitedSafeWaitGroup{},
   419  		LegacyAPIGroupPrefixes:         sets.NewString(DefaultLegacyAPIPrefix),
   420  		DisabledPostStartHooks:         sets.NewString(),
   421  		PostStartHooks:                 map[string]PostStartHookConfigEntry{},
   422  		HealthzChecks:                  append([]healthz.HealthChecker{}, defaultHealthChecks...),
   423  		ReadyzChecks:                   append([]healthz.HealthChecker{}, defaultHealthChecks...),
   424  		LivezChecks:                    append([]healthz.HealthChecker{}, defaultHealthChecks...),
   425  		EnableIndex:                    true,
   426  		EnableDiscovery:                true,
   427  		EnableProfiling:                true,
   428  		DebugSocketPath:                "",
   429  		EnableMetrics:                  true,
   430  		MaxRequestsInFlight:            400,
   431  		MaxMutatingRequestsInFlight:    200,
   432  		RequestTimeout:                 time.Duration(60) * time.Second,
   433  		MinRequestTimeout:              1800,
   434  		StorageInitializationTimeout:   time.Minute,
   435  		LivezGracePeriod:               time.Duration(0),
   436  		ShutdownDelayDuration:          time.Duration(0),
   437  		// 1.5MB is the default client request size in bytes
   438  		// the etcd server should accept. See
   439  		// https://github.com/etcd-io/etcd/blob/release-3.4/embed/config.go#L56.
   440  		// A request body might be encoded in json, and is converted to
   441  		// proto when persisted in etcd, so we allow 2x as the largest size
   442  		// increase the "copy" operations in a json patch may cause.
   443  		JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
   444  		// 1.5MB is the recommended client request size in byte
   445  		// the etcd server should accept. See
   446  		// https://github.com/etcd-io/etcd/blob/release-3.4/embed/config.go#L56.
   447  		// A request body might be encoded in json, and is converted to
   448  		// proto when persisted in etcd, so we allow 2x as the largest request
   449  		// body size to be accepted and decoded in a write request.
   450  		// If this constant is changed, DefaultMaxRequestSizeBytes in k8s.io/apiserver/pkg/cel/limits.go
   451  		// should be changed to reflect the new value, if the two haven't
   452  		// been wired together already somehow.
   453  		MaxRequestBodyBytes: int64(3 * 1024 * 1024),
   454  
   455  		// Default to treating watch as a long-running operation
   456  		// Generic API servers have no inherent long-running subresources
   457  		LongRunningFunc:                     genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()),
   458  		lifecycleSignals:                    lifecycleSignals,
   459  		StorageObjectCountTracker:           flowcontrolrequest.NewStorageObjectCountTracker(),
   460  		ShutdownWatchTerminationGracePeriod: time.Duration(0),
   461  
   462  		APIServerID:           id,
   463  		StorageVersionManager: storageversion.NewDefaultManager(),
   464  		TracerProvider:        tracing.NewNoopTracerProvider(),
   465  	}
   466  }
   467  
   468  // NewRecommendedConfig returns a RecommendedConfig struct with the default values
   469  func NewRecommendedConfig(codecs serializer.CodecFactory) *RecommendedConfig {
   470  	return &RecommendedConfig{
   471  		Config: *NewConfig(codecs),
   472  	}
   473  }
   474  
   475  // DefaultOpenAPIConfig provides the default OpenAPIConfig used to build the OpenAPI V2 spec
   476  func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.Config {
   477  	return &openapicommon.Config{
   478  		ProtocolList:   []string{"https"},
   479  		IgnorePrefixes: []string{},
   480  		Info: &spec.Info{
   481  			InfoProps: spec.InfoProps{
   482  				Title: "Generic API Server",
   483  			},
   484  		},
   485  		DefaultResponse: &spec.Response{
   486  			ResponseProps: spec.ResponseProps{
   487  				Description: "Default Response.",
   488  			},
   489  		},
   490  		GetOperationIDAndTags: apiopenapi.GetOperationIDAndTags,
   491  		GetDefinitionName:     defNamer.GetDefinitionName,
   492  		GetDefinitions:        getDefinitions,
   493  	}
   494  }
   495  
   496  // DefaultOpenAPIV3Config provides the default OpenAPIV3Config used to build the OpenAPI V3 spec
   497  func DefaultOpenAPIV3Config(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.OpenAPIV3Config {
   498  	defaultConfig := &openapicommon.OpenAPIV3Config{
   499  		IgnorePrefixes: []string{},
   500  		Info: &spec.Info{
   501  			InfoProps: spec.InfoProps{
   502  				Title: "Generic API Server",
   503  			},
   504  		},
   505  		DefaultResponse: &spec3.Response{
   506  			ResponseProps: spec3.ResponseProps{
   507  				Description: "Default Response.",
   508  			},
   509  		},
   510  		GetOperationIDAndTags: apiopenapi.GetOperationIDAndTags,
   511  		GetDefinitionName:     defNamer.GetDefinitionName,
   512  		GetDefinitions:        getDefinitions,
   513  	}
   514  	defaultConfig.Definitions = getDefinitions(func(name string) spec.Ref {
   515  		defName, _ := defaultConfig.GetDefinitionName(name)
   516  		return spec.MustCreateRef("#/components/schemas/" + openapicommon.EscapeJsonPointer(defName))
   517  	})
   518  
   519  	return defaultConfig
   520  }
   521  
   522  func (c *AuthenticationInfo) ApplyClientCert(clientCA dynamiccertificates.CAContentProvider, servingInfo *SecureServingInfo) error {
   523  	if servingInfo == nil {
   524  		return nil
   525  	}
   526  	if clientCA == nil {
   527  		return nil
   528  	}
   529  	if servingInfo.ClientCA == nil {
   530  		servingInfo.ClientCA = clientCA
   531  		return nil
   532  	}
   533  
   534  	servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCA)
   535  	return nil
   536  }
   537  
   538  type completedConfig struct {
   539  	*Config
   540  
   541  	//===========================================================================
   542  	// values below here are filled in during completion
   543  	//===========================================================================
   544  
   545  	// SharedInformerFactory provides shared informers for resources
   546  	SharedInformerFactory informers.SharedInformerFactory
   547  }
   548  
   549  type CompletedConfig struct {
   550  	// Embed a private pointer that cannot be instantiated outside of this package.
   551  	*completedConfig
   552  }
   553  
   554  // AddHealthChecks adds a health check to our config to be exposed by the health endpoints
   555  // of our configured apiserver. We should prefer this to adding healthChecks directly to
   556  // the config unless we explicitly want to add a healthcheck only to a specific health endpoint.
   557  func (c *Config) AddHealthChecks(healthChecks ...healthz.HealthChecker) {
   558  	c.HealthzChecks = append(c.HealthzChecks, healthChecks...)
   559  	c.LivezChecks = append(c.LivezChecks, healthChecks...)
   560  	c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...)
   561  }
   562  
   563  // AddReadyzChecks adds a health check to our config to be exposed by the readyz endpoint
   564  // of our configured apiserver.
   565  func (c *Config) AddReadyzChecks(healthChecks ...healthz.HealthChecker) {
   566  	c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...)
   567  }
   568  
   569  // AddPostStartHook allows you to add a PostStartHook that will later be added to the server itself in a New call.
   570  // Name conflicts will cause an error.
   571  func (c *Config) AddPostStartHook(name string, hook PostStartHookFunc) error {
   572  	if len(name) == 0 {
   573  		return fmt.Errorf("missing name")
   574  	}
   575  	if hook == nil {
   576  		return fmt.Errorf("hook func may not be nil: %q", name)
   577  	}
   578  	if c.DisabledPostStartHooks.Has(name) {
   579  		klog.V(1).Infof("skipping %q because it was explicitly disabled", name)
   580  		return nil
   581  	}
   582  
   583  	if postStartHook, exists := c.PostStartHooks[name]; exists {
   584  		// this is programmer error, but it can be hard to debug
   585  		return fmt.Errorf("unable to add %q because it was already registered by: %s", name, postStartHook.originatingStack)
   586  	}
   587  	c.PostStartHooks[name] = PostStartHookConfigEntry{hook: hook, originatingStack: string(debug.Stack())}
   588  
   589  	return nil
   590  }
   591  
   592  // AddPostStartHookOrDie allows you to add a PostStartHook, but dies on failure.
   593  func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) {
   594  	if err := c.AddPostStartHook(name, hook); err != nil {
   595  		klog.Fatalf("Error registering PostStartHook %q: %v", name, err)
   596  	}
   597  }
   598  
   599  func completeOpenAPI(config *openapicommon.Config, version *version.Version) {
   600  	if config == nil {
   601  		return
   602  	}
   603  	if config.SecurityDefinitions != nil {
   604  		// Setup OpenAPI security: all APIs will have the same authentication for now.
   605  		config.DefaultSecurity = []map[string][]string{}
   606  		keys := []string{}
   607  		for k := range *config.SecurityDefinitions {
   608  			keys = append(keys, k)
   609  		}
   610  		sort.Strings(keys)
   611  		for _, k := range keys {
   612  			config.DefaultSecurity = append(config.DefaultSecurity, map[string][]string{k: {}})
   613  		}
   614  		if config.CommonResponses == nil {
   615  			config.CommonResponses = map[int]spec.Response{}
   616  		}
   617  		if _, exists := config.CommonResponses[http.StatusUnauthorized]; !exists {
   618  			config.CommonResponses[http.StatusUnauthorized] = spec.Response{
   619  				ResponseProps: spec.ResponseProps{
   620  					Description: "Unauthorized",
   621  				},
   622  			}
   623  		}
   624  	}
   625  	// make sure we populate info, and info.version, if not manually set
   626  	if config.Info == nil {
   627  		config.Info = &spec.Info{}
   628  	}
   629  	if config.Info.Version == "" {
   630  		if version != nil {
   631  			config.Info.Version = strings.Split(version.String(), "-")[0]
   632  		} else {
   633  			config.Info.Version = "unversioned"
   634  		}
   635  	}
   636  }
   637  
   638  func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) {
   639  	if config == nil {
   640  		return
   641  	}
   642  	if config.SecuritySchemes != nil {
   643  		// Setup OpenAPI security: all APIs will have the same authentication for now.
   644  		config.DefaultSecurity = []map[string][]string{}
   645  		keys := []string{}
   646  		for k := range config.SecuritySchemes {
   647  			keys = append(keys, k)
   648  		}
   649  		sort.Strings(keys)
   650  		for _, k := range keys {
   651  			config.DefaultSecurity = append(config.DefaultSecurity, map[string][]string{k: {}})
   652  		}
   653  		if config.CommonResponses == nil {
   654  			config.CommonResponses = map[int]*spec3.Response{}
   655  		}
   656  		if _, exists := config.CommonResponses[http.StatusUnauthorized]; !exists {
   657  			config.CommonResponses[http.StatusUnauthorized] = &spec3.Response{
   658  				ResponseProps: spec3.ResponseProps{
   659  					Description: "Unauthorized",
   660  				},
   661  			}
   662  		}
   663  	}
   664  	// make sure we populate info, and info.version, if not manually set
   665  	if config.Info == nil {
   666  		config.Info = &spec.Info{}
   667  	}
   668  	if config.Info.Version == "" {
   669  		if version != nil {
   670  			config.Info.Version = strings.Split(version.String(), "-")[0]
   671  		} else {
   672  			config.Info.Version = "unversioned"
   673  		}
   674  	}
   675  }
   676  
   677  // DrainedNotify returns a lifecycle signal of genericapiserver already drained while shutting down.
   678  func (c *Config) DrainedNotify() <-chan struct{} {
   679  	return c.lifecycleSignals.InFlightRequestsDrained.Signaled()
   680  }
   681  
   682  // ShutdownInitiated returns a lifecycle signal of apiserver shutdown having been initiated.
   683  func (c *Config) ShutdownInitiatedNotify() <-chan struct{} {
   684  	return c.lifecycleSignals.ShutdownInitiated.Signaled()
   685  }
   686  
   687  // Complete fills in any fields not set that are required to have valid data and can be derived
   688  // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver.
   689  func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig {
   690  	if c.FeatureGate == nil {
   691  		c.FeatureGate = utilfeature.DefaultFeatureGate
   692  	}
   693  	if len(c.ExternalAddress) == 0 && c.PublicAddress != nil {
   694  		c.ExternalAddress = c.PublicAddress.String()
   695  	}
   696  
   697  	// if there is no port, and we listen on one securely, use that one
   698  	if _, _, err := net.SplitHostPort(c.ExternalAddress); err != nil {
   699  		if c.SecureServing == nil {
   700  			klog.Fatalf("cannot derive external address port without listening on a secure port.")
   701  		}
   702  		_, port, err := c.SecureServing.HostPort()
   703  		if err != nil {
   704  			klog.Fatalf("cannot derive external address from the secure port: %v", err)
   705  		}
   706  		c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port))
   707  	}
   708  	completeOpenAPI(c.OpenAPIConfig, c.EffectiveVersion.EmulationVersion())
   709  	completeOpenAPIV3(c.OpenAPIV3Config, c.EffectiveVersion.EmulationVersion())
   710  
   711  	if c.DiscoveryAddresses == nil {
   712  		c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress}
   713  	}
   714  
   715  	AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization)
   716  
   717  	if c.RequestInfoResolver == nil {
   718  		c.RequestInfoResolver = NewRequestInfoResolver(c)
   719  	}
   720  
   721  	if c.EquivalentResourceRegistry == nil {
   722  		if c.RESTOptionsGetter == nil {
   723  			c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistry()
   724  		} else {
   725  			c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string {
   726  				// use the storage prefix as the key if possible
   727  				if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil {
   728  					return opts.ResourcePrefix
   729  				}
   730  				// otherwise return "" to use the default key (parent GV name)
   731  				return ""
   732  			})
   733  		}
   734  	}
   735  
   736  	return CompletedConfig{&completedConfig{c, informers}}
   737  }
   738  
   739  // Complete fills in any fields not set that are required to have valid data and can be derived
   740  // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver.
   741  func (c *RecommendedConfig) Complete() CompletedConfig {
   742  	return c.Config.Complete(c.SharedInformerFactory)
   743  }
   744  
   745  var allowedMediaTypes = []string{
   746  	runtime.ContentTypeJSON,
   747  	runtime.ContentTypeYAML,
   748  	runtime.ContentTypeProtobuf,
   749  }
   750  
   751  // New creates a new server which logically combines the handling chain with the passed server.
   752  // name is used to differentiate for logging. The handler chain in particular can be difficult as it starts delegating.
   753  // delegationTarget may not be nil.
   754  func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) {
   755  	if c.Serializer == nil {
   756  		return nil, fmt.Errorf("Genericapiserver.New() called with config.Serializer == nil")
   757  	}
   758  	for _, info := range c.Serializer.SupportedMediaTypes() {
   759  		var ok bool
   760  		for _, mt := range allowedMediaTypes {
   761  			if info.MediaType == mt {
   762  				ok = true
   763  				break
   764  			}
   765  		}
   766  		if !ok {
   767  			return nil, fmt.Errorf("refusing to create new apiserver %q with support for media type %q (allowed media types are: %s)", name, info.MediaType, strings.Join(allowedMediaTypes, ", "))
   768  		}
   769  	}
   770  	if c.LoopbackClientConfig == nil {
   771  		return nil, fmt.Errorf("Genericapiserver.New() called with config.LoopbackClientConfig == nil")
   772  	}
   773  	if c.EquivalentResourceRegistry == nil {
   774  		return nil, fmt.Errorf("Genericapiserver.New() called with config.EquivalentResourceRegistry == nil")
   775  	}
   776  
   777  	handlerChainBuilder := func(handler http.Handler) http.Handler {
   778  		return c.BuildHandlerChainFunc(handler, c.Config)
   779  	}
   780  
   781  	var debugSocket *routes.DebugSocket
   782  	if c.DebugSocketPath != "" {
   783  		debugSocket = routes.NewDebugSocket(c.DebugSocketPath)
   784  	}
   785  
   786  	apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())
   787  
   788  	s := &GenericAPIServer{
   789  		discoveryAddresses:             c.DiscoveryAddresses,
   790  		LoopbackClientConfig:           c.LoopbackClientConfig,
   791  		legacyAPIGroupPrefixes:         c.LegacyAPIGroupPrefixes,
   792  		admissionControl:               c.AdmissionControl,
   793  		Serializer:                     c.Serializer,
   794  		AuditBackend:                   c.AuditBackend,
   795  		Authorizer:                     c.Authorization.Authorizer,
   796  		delegationTarget:               delegationTarget,
   797  		EquivalentResourceRegistry:     c.EquivalentResourceRegistry,
   798  		NonLongRunningRequestWaitGroup: c.NonLongRunningRequestWaitGroup,
   799  		WatchRequestWaitGroup:          c.WatchRequestWaitGroup,
   800  		Handler:                        apiServerHandler,
   801  		UnprotectedDebugSocket:         debugSocket,
   802  
   803  		listedPathProvider: apiServerHandler,
   804  
   805  		minRequestTimeout:                   time.Duration(c.MinRequestTimeout) * time.Second,
   806  		ShutdownTimeout:                     c.RequestTimeout,
   807  		ShutdownDelayDuration:               c.ShutdownDelayDuration,
   808  		ShutdownWatchTerminationGracePeriod: c.ShutdownWatchTerminationGracePeriod,
   809  		SecureServingInfo:                   c.SecureServing,
   810  		ExternalAddress:                     c.ExternalAddress,
   811  
   812  		openAPIConfig:           c.OpenAPIConfig,
   813  		openAPIV3Config:         c.OpenAPIV3Config,
   814  		skipOpenAPIInstallation: c.SkipOpenAPIInstallation,
   815  
   816  		postStartHooks:         map[string]postStartHookEntry{},
   817  		preShutdownHooks:       map[string]preShutdownHookEntry{},
   818  		disabledPostStartHooks: c.DisabledPostStartHooks,
   819  
   820  		healthzRegistry:  healthCheckRegistry{path: "/healthz", checks: c.HealthzChecks},
   821  		livezRegistry:    healthCheckRegistry{path: "/livez", checks: c.LivezChecks, clock: clock.RealClock{}},
   822  		readyzRegistry:   healthCheckRegistry{path: "/readyz", checks: c.ReadyzChecks},
   823  		livezGracePeriod: c.LivezGracePeriod,
   824  
   825  		DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),
   826  
   827  		maxRequestBodyBytes: c.MaxRequestBodyBytes,
   828  
   829  		lifecycleSignals:       c.lifecycleSignals,
   830  		ShutdownSendRetryAfter: c.ShutdownSendRetryAfter,
   831  
   832  		APIServerID:           c.APIServerID,
   833  		StorageReadinessHook:  NewStorageReadinessHook(c.StorageInitializationTimeout),
   834  		StorageVersionManager: c.StorageVersionManager,
   835  
   836  		EffectiveVersion: c.EffectiveVersion,
   837  		FeatureGate:      c.FeatureGate,
   838  
   839  		muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
   840  	}
   841  
   842  	if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
   843  		manager := c.AggregatedDiscoveryGroupManager
   844  		if manager == nil {
   845  			manager = discoveryendpoint.NewResourceManager("apis")
   846  		}
   847  		s.AggregatedDiscoveryGroupManager = manager
   848  		s.AggregatedLegacyDiscoveryGroupManager = discoveryendpoint.NewResourceManager("api")
   849  	}
   850  	for {
   851  		if c.JSONPatchMaxCopyBytes <= 0 {
   852  			break
   853  		}
   854  		existing := atomic.LoadInt64(&jsonpatch.AccumulatedCopySizeLimit)
   855  		if existing > 0 && existing < c.JSONPatchMaxCopyBytes {
   856  			break
   857  		}
   858  		if atomic.CompareAndSwapInt64(&jsonpatch.AccumulatedCopySizeLimit, existing, c.JSONPatchMaxCopyBytes) {
   859  			break
   860  		}
   861  	}
   862  
   863  	// first add poststarthooks from delegated targets
   864  	for k, v := range delegationTarget.PostStartHooks() {
   865  		s.postStartHooks[k] = v
   866  	}
   867  
   868  	for k, v := range delegationTarget.PreShutdownHooks() {
   869  		s.preShutdownHooks[k] = v
   870  	}
   871  
   872  	// add poststarthooks that were preconfigured.  Using the add method will give us an error if the same name has already been registered.
   873  	for name, preconfiguredPostStartHook := range c.PostStartHooks {
   874  		if err := s.AddPostStartHook(name, preconfiguredPostStartHook.hook); err != nil {
   875  			return nil, err
   876  		}
   877  	}
   878  
   879  	// register mux signals from the delegated server
   880  	for k, v := range delegationTarget.MuxAndDiscoveryCompleteSignals() {
   881  		if err := s.RegisterMuxAndDiscoveryCompleteSignal(k, v); err != nil {
   882  			return nil, err
   883  		}
   884  	}
   885  
   886  	genericApiServerHookName := "generic-apiserver-start-informers"
   887  	if c.SharedInformerFactory != nil {
   888  		if !s.isPostStartHookRegistered(genericApiServerHookName) {
   889  			err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
   890  				c.SharedInformerFactory.Start(context.StopCh)
   891  				return nil
   892  			})
   893  			if err != nil {
   894  				return nil, err
   895  			}
   896  		}
   897  		// TODO: Once we get rid of /healthz consider changing this to post-start-hook.
   898  		err := s.AddReadyzChecks(healthz.NewInformerSyncHealthz(c.SharedInformerFactory))
   899  		if err != nil {
   900  			return nil, err
   901  		}
   902  	}
   903  
   904  	const priorityAndFairnessConfigConsumerHookName = "priority-and-fairness-config-consumer"
   905  	if s.isPostStartHookRegistered(priorityAndFairnessConfigConsumerHookName) {
   906  	} else if c.FlowControl != nil {
   907  		err := s.AddPostStartHook(priorityAndFairnessConfigConsumerHookName, func(context PostStartHookContext) error {
   908  			go c.FlowControl.Run(context.StopCh)
   909  			return nil
   910  		})
   911  		if err != nil {
   912  			return nil, err
   913  		}
   914  		// TODO(yue9944882): plumb pre-shutdown-hook for request-management system?
   915  	} else {
   916  		klog.V(3).Infof("Not requested to run hook %s", priorityAndFairnessConfigConsumerHookName)
   917  	}
   918  
   919  	// Add PostStartHooks for maintaining the watermarks for the Priority-and-Fairness and the Max-in-Flight filters.
   920  	if c.FlowControl != nil {
   921  		const priorityAndFairnessFilterHookName = "priority-and-fairness-filter"
   922  		if !s.isPostStartHookRegistered(priorityAndFairnessFilterHookName) {
   923  			err := s.AddPostStartHook(priorityAndFairnessFilterHookName, func(context PostStartHookContext) error {
   924  				genericfilters.StartPriorityAndFairnessWatermarkMaintenance(context.StopCh)
   925  				return nil
   926  			})
   927  			if err != nil {
   928  				return nil, err
   929  			}
   930  		}
   931  	} else {
   932  		const maxInFlightFilterHookName = "max-in-flight-filter"
   933  		if !s.isPostStartHookRegistered(maxInFlightFilterHookName) {
   934  			err := s.AddPostStartHook(maxInFlightFilterHookName, func(context PostStartHookContext) error {
   935  				genericfilters.StartMaxInFlightWatermarkMaintenance(context.StopCh)
   936  				return nil
   937  			})
   938  			if err != nil {
   939  				return nil, err
   940  			}
   941  		}
   942  	}
   943  
   944  	// Add PostStartHook for maintenaing the object count tracker.
   945  	if c.StorageObjectCountTracker != nil {
   946  		const storageObjectCountTrackerHookName = "storage-object-count-tracker-hook"
   947  		if !s.isPostStartHookRegistered(storageObjectCountTrackerHookName) {
   948  			if err := s.AddPostStartHook(storageObjectCountTrackerHookName, func(context PostStartHookContext) error {
   949  				go c.StorageObjectCountTracker.RunUntil(context.StopCh)
   950  				return nil
   951  			}); err != nil {
   952  				return nil, err
   953  			}
   954  		}
   955  	}
   956  
   957  	for _, delegateCheck := range delegationTarget.HealthzChecks() {
   958  		skip := false
   959  		for _, existingCheck := range c.HealthzChecks {
   960  			if existingCheck.Name() == delegateCheck.Name() {
   961  				skip = true
   962  				break
   963  			}
   964  		}
   965  		if skip {
   966  			continue
   967  		}
   968  		s.AddHealthChecks(delegateCheck)
   969  	}
   970  	s.RegisterDestroyFunc(func() {
   971  		if err := c.Config.TracerProvider.Shutdown(context.Background()); err != nil {
   972  			klog.Errorf("failed to shut down tracer provider: %v", err)
   973  		}
   974  	})
   975  
   976  	s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget}
   977  
   978  	installAPI(s, c.Config)
   979  
   980  	// use the UnprotectedHandler from the delegation target to ensure that we don't attempt to double authenticator, authorize,
   981  	// or some other part of the filter chain in delegation cases.
   982  	if delegationTarget.UnprotectedHandler() == nil && c.EnableIndex {
   983  		s.Handler.NonGoRestfulMux.NotFoundHandler(routes.IndexLister{
   984  			StatusCode:   http.StatusNotFound,
   985  			PathProvider: s.listedPathProvider,
   986  		})
   987  	}
   988  
   989  	return s, nil
   990  }
   991  
   992  func BuildHandlerChainWithStorageVersionPrecondition(apiHandler http.Handler, c *Config) http.Handler {
   993  	// WithStorageVersionPrecondition needs the WithRequestInfo to run first
   994  	handler := genericapifilters.WithStorageVersionPrecondition(apiHandler, c.StorageVersionManager, c.Serializer)
   995  	return DefaultBuildHandlerChain(handler, c)
   996  }
   997  
   998  func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
   999  	handler := apiHandler
  1000  
  1001  	handler = filterlatency.TrackCompleted(handler)
  1002  	handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
  1003  	handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authorization")
  1004  
  1005  	if c.FlowControl != nil {
  1006  		workEstimatorCfg := flowcontrolrequest.DefaultWorkEstimatorConfig()
  1007  		requestWorkEstimator := flowcontrolrequest.NewWorkEstimator(
  1008  			c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg, c.FlowControl.GetMaxSeats)
  1009  		handler = filterlatency.TrackCompleted(handler)
  1010  		handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator, c.RequestTimeout/4)
  1011  		handler = filterlatency.TrackStarted(handler, c.TracerProvider, "priorityandfairness")
  1012  	} else {
  1013  		handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
  1014  	}
  1015  
  1016  	handler = filterlatency.TrackCompleted(handler)
  1017  	handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
  1018  	handler = filterlatency.TrackStarted(handler, c.TracerProvider, "impersonation")
  1019  
  1020  	handler = filterlatency.TrackCompleted(handler)
  1021  	handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator, c.LongRunningFunc)
  1022  	handler = filterlatency.TrackStarted(handler, c.TracerProvider, "audit")
  1023  
  1024  	failedHandler := genericapifilters.Unauthorized(c.Serializer)
  1025  	failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyRuleEvaluator)
  1026  
  1027  	failedHandler = filterlatency.TrackCompleted(failedHandler)
  1028  	handler = filterlatency.TrackCompleted(handler)
  1029  	handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences, c.Authentication.RequestHeaderConfig)
  1030  	handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authentication")
  1031  
  1032  	handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")
  1033  
  1034  	// WithWarningRecorder must be wrapped by the timeout handler
  1035  	// to make the addition of warning headers threadsafe
  1036  	handler = genericapifilters.WithWarningRecorder(handler)
  1037  
  1038  	// WithTimeoutForNonLongRunningRequests will call the rest of the request handling in a go-routine with the
  1039  	// context with deadline. The go-routine can keep running, while the timeout logic will return a timeout to the client.
  1040  	handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc)
  1041  
  1042  	handler = genericapifilters.WithRequestDeadline(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator,
  1043  		c.LongRunningFunc, c.Serializer, c.RequestTimeout)
  1044  	handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.NonLongRunningRequestWaitGroup)
  1045  	if c.ShutdownWatchTerminationGracePeriod > 0 {
  1046  		handler = genericfilters.WithWatchTerminationDuringShutdown(handler, c.lifecycleSignals, c.WatchRequestWaitGroup)
  1047  	}
  1048  	if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 {
  1049  		handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance)
  1050  	}
  1051  	handler = genericapifilters.WithCacheControl(handler)
  1052  	handler = genericfilters.WithHSTS(handler, c.HSTSDirectives)
  1053  	if c.ShutdownSendRetryAfter {
  1054  		handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
  1055  	}
  1056  	handler = genericfilters.WithHTTPLogging(handler)
  1057  	if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) {
  1058  		handler = genericapifilters.WithTracing(handler, c.TracerProvider)
  1059  	}
  1060  	handler = genericapifilters.WithLatencyTrackers(handler)
  1061  	// WithRoutine will execute future handlers in a separate goroutine and serving
  1062  	// handler in current goroutine to minimize the stack memory usage. It must be
  1063  	// after WithPanicRecover() to be protected from panics.
  1064  	if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
  1065  		handler = routine.WithRoutine(handler, c.LongRunningFunc)
  1066  	}
  1067  	handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
  1068  	handler = genericapifilters.WithRequestReceivedTimestamp(handler)
  1069  	handler = genericapifilters.WithMuxAndDiscoveryComplete(handler, c.lifecycleSignals.MuxAndDiscoveryComplete.Signaled())
  1070  	handler = genericfilters.WithPanicRecovery(handler, c.RequestInfoResolver)
  1071  	handler = genericapifilters.WithAuditInit(handler)
  1072  	return handler
  1073  }
  1074  
  1075  func installAPI(s *GenericAPIServer, c *Config) {
  1076  	if c.EnableIndex {
  1077  		routes.Index{}.Install(s.listedPathProvider, s.Handler.NonGoRestfulMux)
  1078  	}
  1079  	if c.EnableProfiling {
  1080  		routes.Profiling{}.Install(s.Handler.NonGoRestfulMux)
  1081  		if c.EnableContentionProfiling {
  1082  			goruntime.SetBlockProfileRate(1)
  1083  		}
  1084  		// so far, only logging related endpoints are considered valid to add for these debug flags.
  1085  		routes.DebugFlags{}.Install(s.Handler.NonGoRestfulMux, "v", routes.StringFlagPutHandler(logs.GlogSetter))
  1086  	}
  1087  	if s.UnprotectedDebugSocket != nil {
  1088  		s.UnprotectedDebugSocket.InstallProfiling()
  1089  		s.UnprotectedDebugSocket.InstallDebugFlag("v", routes.StringFlagPutHandler(logs.GlogSetter))
  1090  		if c.EnableContentionProfiling {
  1091  			goruntime.SetBlockProfileRate(1)
  1092  		}
  1093  	}
  1094  
  1095  	if c.EnableMetrics {
  1096  		if c.EnableProfiling {
  1097  			routes.MetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
  1098  			slis.SLIMetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
  1099  		} else {
  1100  			routes.DefaultMetrics{}.Install(s.Handler.NonGoRestfulMux)
  1101  			slis.SLIMetrics{}.Install(s.Handler.NonGoRestfulMux)
  1102  		}
  1103  	}
  1104  
  1105  	routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer)
  1106  
  1107  	if c.EnableDiscovery {
  1108  		if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
  1109  			wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager)
  1110  			s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{}))
  1111  		} else {
  1112  			s.Handler.GoRestfulContainer.Add(s.DiscoveryGroupManager.WebService())
  1113  		}
  1114  	}
  1115  	if c.FlowControl != nil {
  1116  		c.FlowControl.Install(s.Handler.NonGoRestfulMux)
  1117  	}
  1118  }
  1119  
  1120  func NewRequestInfoResolver(c *Config) *apirequest.RequestInfoFactory {
  1121  	apiPrefixes := sets.NewString(strings.Trim(APIGroupPrefix, "/")) // all possible API prefixes
  1122  	legacyAPIPrefixes := sets.String{}                               // APIPrefixes that won't have groups (legacy)
  1123  	for legacyAPIPrefix := range c.LegacyAPIGroupPrefixes {
  1124  		apiPrefixes.Insert(strings.Trim(legacyAPIPrefix, "/"))
  1125  		legacyAPIPrefixes.Insert(strings.Trim(legacyAPIPrefix, "/"))
  1126  	}
  1127  
  1128  	return &apirequest.RequestInfoFactory{
  1129  		APIPrefixes:          apiPrefixes,
  1130  		GrouplessAPIPrefixes: legacyAPIPrefixes,
  1131  	}
  1132  }
  1133  
  1134  func (s *SecureServingInfo) HostPort() (string, int, error) {
  1135  	if s == nil || s.Listener == nil {
  1136  		return "", 0, fmt.Errorf("no listener found")
  1137  	}
  1138  	addr := s.Listener.Addr().String()
  1139  	host, portStr, err := net.SplitHostPort(addr)
  1140  	if err != nil {
  1141  		return "", 0, fmt.Errorf("failed to get port from listener address %q: %v", addr, err)
  1142  	}
  1143  	port, err := utilsnet.ParsePort(portStr, true)
  1144  	if err != nil {
  1145  		return "", 0, fmt.Errorf("invalid non-numeric port %q", portStr)
  1146  	}
  1147  	return host, port, nil
  1148  }
  1149  
  1150  // AuthorizeClientBearerToken wraps the authenticator and authorizer in loopback authentication logic
  1151  // if the loopback client config is specified AND it has a bearer token. Note that if either authn or
  1152  // authz is nil, this function won't add a token authenticator or authorizer.
  1153  func AuthorizeClientBearerToken(loopback *restclient.Config, authn *AuthenticationInfo, authz *AuthorizationInfo) {
  1154  	if loopback == nil || len(loopback.BearerToken) == 0 {
  1155  		return
  1156  	}
  1157  	if authn == nil || authz == nil {
  1158  		// prevent nil pointer panic
  1159  		return
  1160  	}
  1161  	if authn.Authenticator == nil || authz.Authorizer == nil {
  1162  		// authenticator or authorizer might be nil if we want to bypass authz/authn
  1163  		// and we also do nothing in this case.
  1164  		return
  1165  	}
  1166  
  1167  	privilegedLoopbackToken := loopback.BearerToken
  1168  	var uid = uuid.New().String()
  1169  	tokens := make(map[string]*user.DefaultInfo)
  1170  	tokens[privilegedLoopbackToken] = &user.DefaultInfo{
  1171  		Name:   user.APIServerUser,
  1172  		UID:    uid,
  1173  		Groups: []string{user.SystemPrivilegedGroup},
  1174  	}
  1175  
  1176  	tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens, authn.APIAudiences)
  1177  	authn.Authenticator = authenticatorunion.New(tokenAuthenticator, authn.Authenticator)
  1178  }
  1179  
  1180  // For testing purpose only
  1181  func SetHostnameFuncForTests(name string) {
  1182  	hostnameFunc = func() (host string, err error) {
  1183  		host = name
  1184  		err = nil
  1185  		return
  1186  	}
  1187  }