k8s.io/apiserver@v0.31.1/pkg/server/options/serving.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package options
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"net"
    23  	"path"
    24  	"strconv"
    25  	"strings"
    26  	"syscall"
    27  
    28  	"github.com/spf13/pflag"
    29  	"k8s.io/klog/v2"
    30  	netutils "k8s.io/utils/net"
    31  
    32  	utilnet "k8s.io/apimachinery/pkg/util/net"
    33  	"k8s.io/apiserver/pkg/server"
    34  	"k8s.io/apiserver/pkg/server/dynamiccertificates"
    35  	certutil "k8s.io/client-go/util/cert"
    36  	"k8s.io/client-go/util/keyutil"
    37  	cliflag "k8s.io/component-base/cli/flag"
    38  )
    39  
    40  type SecureServingOptions struct {
    41  	BindAddress net.IP
    42  	// BindPort is ignored when Listener is set, will serve https even with 0.
    43  	BindPort int
    44  	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
    45  	// "tcp4", and "tcp6".
    46  	BindNetwork string
    47  	// DisableHTTP2Serving indicates that http2 serving should not be enabled.
    48  	DisableHTTP2Serving bool
    49  	// Required set to true means that BindPort cannot be zero.
    50  	Required bool
    51  	// ExternalAddress is the address advertised, even if BindAddress is a loopback. By default this
    52  	// is set to BindAddress if the later no loopback, or to the first host interface address.
    53  	ExternalAddress net.IP
    54  
    55  	// Listener is the secure server network listener.
    56  	// either Listener or BindAddress/BindPort/BindNetwork is set,
    57  	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
    58  	Listener net.Listener
    59  
    60  	// ServerCert is the TLS cert info for serving secure traffic
    61  	ServerCert GeneratableKeyCert
    62  	// SNICertKeys are named CertKeys for serving secure traffic with SNI support.
    63  	SNICertKeys []cliflag.NamedCertKey
    64  	// CipherSuites is the list of allowed cipher suites for the server.
    65  	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
    66  	CipherSuites []string
    67  	// MinTLSVersion is the minimum TLS version supported.
    68  	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
    69  	MinTLSVersion string
    70  
    71  	// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
    72  	// A value of zero means to use the default provided by golang's HTTP/2 support.
    73  	HTTP2MaxStreamsPerConnection int
    74  
    75  	// PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows
    76  	// more than one instance to bind on the same address and port.
    77  	PermitPortSharing bool
    78  
    79  	// PermitAddressSharing controls if SO_REUSEADDR is used when binding the port.
    80  	PermitAddressSharing bool
    81  }
    82  
    83  type CertKey struct {
    84  	// CertFile is a file containing a PEM-encoded certificate, and possibly the complete certificate chain
    85  	CertFile string
    86  	// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
    87  	KeyFile string
    88  }
    89  
    90  type GeneratableKeyCert struct {
    91  	// CertKey allows setting an explicit cert/key file to use.
    92  	CertKey CertKey
    93  
    94  	// CertDirectory specifies a directory to write generated certificates to if CertFile/KeyFile aren't explicitly set.
    95  	// PairName is used to determine the filenames within CertDirectory.
    96  	// If CertDirectory and PairName are not set, an in-memory certificate will be generated.
    97  	CertDirectory string
    98  	// PairName is the name which will be used with CertDirectory to make a cert and key filenames.
    99  	// It becomes CertDirectory/PairName.crt and CertDirectory/PairName.key
   100  	PairName string
   101  
   102  	// GeneratedCert holds an in-memory generated certificate if CertFile/KeyFile aren't explicitly set, and CertDirectory/PairName are not set.
   103  	GeneratedCert dynamiccertificates.CertKeyContentProvider
   104  
   105  	// FixtureDirectory is a directory that contains test fixture used to avoid regeneration of certs during tests.
   106  	// The format is:
   107  	// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.crt
   108  	// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.key
   109  	FixtureDirectory string
   110  }
   111  
   112  func NewSecureServingOptions() *SecureServingOptions {
   113  	return &SecureServingOptions{
   114  		BindAddress: netutils.ParseIPSloppy("0.0.0.0"),
   115  		BindPort:    443,
   116  		ServerCert: GeneratableKeyCert{
   117  			PairName:      "apiserver",
   118  			CertDirectory: "apiserver.local.config/certificates",
   119  		},
   120  	}
   121  }
   122  
   123  func (s *SecureServingOptions) DefaultExternalAddress() (net.IP, error) {
   124  	if s.ExternalAddress != nil && !s.ExternalAddress.IsUnspecified() {
   125  		return s.ExternalAddress, nil
   126  	}
   127  	return utilnet.ResolveBindAddress(s.BindAddress)
   128  }
   129  
   130  func (s *SecureServingOptions) Validate() []error {
   131  	if s == nil {
   132  		return nil
   133  	}
   134  
   135  	errors := []error{}
   136  
   137  	if s.Required && s.BindPort < 1 || s.BindPort > 65535 {
   138  		errors = append(errors, fmt.Errorf("--secure-port %v must be between 1 and 65535, inclusive. It cannot be turned off with 0", s.BindPort))
   139  	} else if s.BindPort < 0 || s.BindPort > 65535 {
   140  		errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port", s.BindPort))
   141  	}
   142  
   143  	if (len(s.ServerCert.CertKey.CertFile) != 0 || len(s.ServerCert.CertKey.KeyFile) != 0) && s.ServerCert.GeneratedCert != nil {
   144  		errors = append(errors, fmt.Errorf("cert/key file and in-memory certificate cannot both be set"))
   145  	}
   146  
   147  	return errors
   148  }
   149  
   150  func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
   151  	if s == nil {
   152  		return
   153  	}
   154  
   155  	fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+
   156  		"The IP address on which to listen for the --secure-port port. The "+
   157  		"associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+
   158  		"clients. If blank or an unspecified address (0.0.0.0 or ::), all interfaces and IP address families will be used.")
   159  
   160  	desc := "The port on which to serve HTTPS with authentication and authorization."
   161  	if s.Required {
   162  		desc += " It cannot be switched off with 0."
   163  	} else {
   164  		desc += " If 0, don't serve HTTPS at all."
   165  	}
   166  	fs.IntVar(&s.BindPort, "secure-port", s.BindPort, desc)
   167  
   168  	fs.BoolVar(&s.DisableHTTP2Serving, "disable-http2-serving", s.DisableHTTP2Serving,
   169  		"If true, HTTP2 serving will be disabled [default=false]")
   170  
   171  	fs.StringVar(&s.ServerCert.CertDirectory, "cert-dir", s.ServerCert.CertDirectory, ""+
   172  		"The directory where the TLS certs are located. "+
   173  		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
   174  
   175  	fs.StringVar(&s.ServerCert.CertKey.CertFile, "tls-cert-file", s.ServerCert.CertKey.CertFile, ""+
   176  		"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
   177  		"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+
   178  		"--tls-private-key-file are not provided, a self-signed certificate and key "+
   179  		"are generated for the public address and saved to the directory specified by --cert-dir.")
   180  
   181  	fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile,
   182  		"File containing the default x509 private key matching --tls-cert-file.")
   183  
   184  	tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames()
   185  	tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames()
   186  	fs.StringSliceVar(&s.CipherSuites, "tls-cipher-suites", s.CipherSuites,
   187  		"Comma-separated list of cipher suites for the server. "+
   188  			"If omitted, the default Go cipher suites will be used. \n"+
   189  			"Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+
   190  			"Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".")
   191  
   192  	tlsPossibleVersions := cliflag.TLSPossibleVersions()
   193  	fs.StringVar(&s.MinTLSVersion, "tls-min-version", s.MinTLSVersion,
   194  		"Minimum TLS version supported. "+
   195  			"Possible values: "+strings.Join(tlsPossibleVersions, ", "))
   196  
   197  	fs.Var(cliflag.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
   198  		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
   199  		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
   200  		"segments. The domain patterns also allow IP addresses, but IPs should only be used if "+
   201  		"the apiserver has visibility to the IP address requested by a client. "+
   202  		"If no domain patterns are provided, the names of the certificate are "+
   203  		"extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+
   204  		"trump over extracted names. For multiple key/certificate pairs, use the "+
   205  		"--tls-sni-cert-key multiple times. "+
   206  		"Examples: \"example.crt,example.key\" or \"foo.crt,foo.key:*.foo.com,foo.com\".")
   207  
   208  	fs.IntVar(&s.HTTP2MaxStreamsPerConnection, "http2-max-streams-per-connection", s.HTTP2MaxStreamsPerConnection, ""+
   209  		"The limit that the server gives to clients for "+
   210  		"the maximum number of streams in an HTTP/2 connection. "+
   211  		"Zero means to use golang's default.")
   212  
   213  	fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing,
   214  		"If true, SO_REUSEPORT will be used when binding the port, which allows "+
   215  			"more than one instance to bind on the same address and port. [default=false]")
   216  
   217  	fs.BoolVar(&s.PermitAddressSharing, "permit-address-sharing", s.PermitAddressSharing,
   218  		"If true, SO_REUSEADDR will be used when binding the port. This allows binding "+
   219  			"to wildcard IPs like 0.0.0.0 and specific IPs in parallel, and it avoids waiting "+
   220  			"for the kernel to release sockets in TIME_WAIT state. [default=false]")
   221  }
   222  
   223  // ApplyTo fills up serving information in the server configuration.
   224  func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error {
   225  	if s == nil {
   226  		return nil
   227  	}
   228  	if s.BindPort <= 0 && s.Listener == nil {
   229  		return nil
   230  	}
   231  
   232  	if s.Listener == nil {
   233  		var err error
   234  		addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
   235  
   236  		c := net.ListenConfig{}
   237  
   238  		ctls := multipleControls{}
   239  		if s.PermitPortSharing {
   240  			ctls = append(ctls, permitPortReuse)
   241  		}
   242  		if s.PermitAddressSharing {
   243  			ctls = append(ctls, permitAddressReuse)
   244  		}
   245  		if len(ctls) > 0 {
   246  			c.Control = ctls.Control
   247  		}
   248  
   249  		s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c)
   250  		if err != nil {
   251  			return fmt.Errorf("failed to create listener: %v", err)
   252  		}
   253  	} else {
   254  		if _, ok := s.Listener.Addr().(*net.TCPAddr); !ok {
   255  			return fmt.Errorf("failed to parse ip and port from listener")
   256  		}
   257  		s.BindPort = s.Listener.Addr().(*net.TCPAddr).Port
   258  		s.BindAddress = s.Listener.Addr().(*net.TCPAddr).IP
   259  	}
   260  
   261  	*config = &server.SecureServingInfo{
   262  		Listener:                     s.Listener,
   263  		HTTP2MaxStreamsPerConnection: s.HTTP2MaxStreamsPerConnection,
   264  		DisableHTTP2:                 s.DisableHTTP2Serving,
   265  	}
   266  	c := *config
   267  
   268  	serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
   269  	// load main cert *original description until 2023-08-18*
   270  
   271  	/*
   272  		kubernetes mutual (2-way) x509 between client and apiserver:
   273  
   274  			>1. apiserver sending its apiserver certificate along with its publickey to client
   275  			2. client verifies the apiserver certificate sent against its cluster certificate authority data
   276  			3. client sending its client certificate along with its public key to the apiserver
   277  			4. apiserver verifies the client certificate sent against its cluster certificate authority data
   278  
   279  			description:
   280  				here, with this block,
   281  				apiserver certificate and pub key data (along with priv key)get loaded into server.SecureServingInfo
   282  				for client to later in the step 2 verify the apiserver certificate during the handshake
   283  				when making a request
   284  
   285  			normal args related to this stage:
   286  				--tls-cert-file string  File containing the default x509 Certificate for HTTPS.
   287  					(CA cert, if any, concatenated after server cert). If HTTPS serving is enabled, and
   288  					--tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate
   289  					and key are generated for the public address and saved to the directory specified by
   290  					--cert-dir
   291  				--tls-private-key-file string  File containing the default x509 private key matching --tls-cert-file.
   292  
   293  				(retrievable from "kube-apiserver --help" command)
   294  				(suggested by @deads2k)
   295  
   296  			see also:
   297  				- for the step 2, see: staging/src/k8s.io/client-go/transport/transport.go
   298  				- for the step 3, see: staging/src/k8s.io/client-go/transport/transport.go
   299  				- for the step 4, see: staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go
   300  	*/
   301  
   302  	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
   303  		var err error
   304  		c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile)
   305  		if err != nil {
   306  			return err
   307  		}
   308  	} else if s.ServerCert.GeneratedCert != nil {
   309  		c.Cert = s.ServerCert.GeneratedCert
   310  	}
   311  
   312  	if len(s.CipherSuites) != 0 {
   313  		cipherSuites, err := cliflag.TLSCipherSuites(s.CipherSuites)
   314  		if err != nil {
   315  			return err
   316  		}
   317  		c.CipherSuites = cipherSuites
   318  	}
   319  
   320  	var err error
   321  	c.MinTLSVersion, err = cliflag.TLSVersion(s.MinTLSVersion)
   322  	if err != nil {
   323  		return err
   324  	}
   325  
   326  	// load SNI certs
   327  	namedTLSCerts := make([]dynamiccertificates.SNICertKeyContentProvider, 0, len(s.SNICertKeys))
   328  	for _, nck := range s.SNICertKeys {
   329  		tlsCert, err := dynamiccertificates.NewDynamicSNIContentFromFiles("sni-serving-cert", nck.CertFile, nck.KeyFile, nck.Names...)
   330  		namedTLSCerts = append(namedTLSCerts, tlsCert)
   331  		if err != nil {
   332  			return fmt.Errorf("failed to load SNI cert and key: %v", err)
   333  		}
   334  	}
   335  	c.SNICerts = namedTLSCerts
   336  
   337  	return nil
   338  }
   339  
   340  func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress string, alternateDNS []string, alternateIPs []net.IP) error {
   341  	if s == nil || (s.BindPort == 0 && s.Listener == nil) {
   342  		return nil
   343  	}
   344  	keyCert := &s.ServerCert.CertKey
   345  	if len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 {
   346  		return nil
   347  	}
   348  
   349  	canReadCertAndKey := false
   350  	if len(s.ServerCert.CertDirectory) > 0 {
   351  		if len(s.ServerCert.PairName) == 0 {
   352  			return fmt.Errorf("PairName is required if CertDirectory is set")
   353  		}
   354  		keyCert.CertFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".crt")
   355  		keyCert.KeyFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".key")
   356  		if canRead, err := certutil.CanReadCertAndKey(keyCert.CertFile, keyCert.KeyFile); err != nil {
   357  			return err
   358  		} else {
   359  			canReadCertAndKey = canRead
   360  		}
   361  	}
   362  
   363  	if !canReadCertAndKey {
   364  		// add either the bind address or localhost to the valid alternates
   365  		if s.BindAddress.IsUnspecified() {
   366  			alternateDNS = append(alternateDNS, "localhost")
   367  		} else {
   368  			alternateIPs = append(alternateIPs, s.BindAddress)
   369  		}
   370  
   371  		if cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures(publicAddress, alternateIPs, alternateDNS, s.ServerCert.FixtureDirectory); err != nil {
   372  			return fmt.Errorf("unable to generate self signed cert: %v", err)
   373  		} else if len(keyCert.CertFile) > 0 && len(keyCert.KeyFile) > 0 {
   374  			if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil {
   375  				return err
   376  			}
   377  			if err := keyutil.WriteKey(keyCert.KeyFile, key); err != nil {
   378  				return err
   379  			}
   380  			klog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile)
   381  		} else {
   382  			s.ServerCert.GeneratedCert, err = dynamiccertificates.NewStaticCertKeyContent("Generated self signed cert", cert, key)
   383  			if err != nil {
   384  				return err
   385  			}
   386  			klog.Infof("Generated self-signed cert in-memory")
   387  		}
   388  	}
   389  
   390  	return nil
   391  }
   392  
   393  func CreateListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) {
   394  	if len(network) == 0 {
   395  		network = "tcp"
   396  	}
   397  
   398  	ln, err := config.Listen(context.TODO(), network, addr)
   399  	if err != nil {
   400  		return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
   401  	}
   402  
   403  	// get port
   404  	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
   405  	if !ok {
   406  		ln.Close()
   407  		return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
   408  	}
   409  
   410  	return ln, tcpAddr.Port, nil
   411  }
   412  
   413  type multipleControls []func(network, addr string, conn syscall.RawConn) error
   414  
   415  func (mcs multipleControls) Control(network, addr string, conn syscall.RawConn) error {
   416  	for _, c := range mcs {
   417  		if err := c(network, addr, conn); err != nil {
   418  			return err
   419  		}
   420  	}
   421  	return nil
   422  }