k8s.io/client-go@v0.31.1/plugin/pkg/client/auth/exec/exec.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package exec
    18  
    19  import (
    20  	"bytes"
    21  	"crypto/tls"
    22  	"crypto/x509"
    23  	"errors"
    24  	"fmt"
    25  	"io"
    26  	"net"
    27  	"net/http"
    28  	"os"
    29  	"os/exec"
    30  	"reflect"
    31  	"strings"
    32  	"sync"
    33  	"time"
    34  
    35  	"golang.org/x/term"
    36  
    37  	"k8s.io/apimachinery/pkg/runtime"
    38  	"k8s.io/apimachinery/pkg/runtime/schema"
    39  	"k8s.io/apimachinery/pkg/runtime/serializer"
    40  	"k8s.io/apimachinery/pkg/util/dump"
    41  	utilnet "k8s.io/apimachinery/pkg/util/net"
    42  	"k8s.io/client-go/pkg/apis/clientauthentication"
    43  	"k8s.io/client-go/pkg/apis/clientauthentication/install"
    44  	clientauthenticationv1 "k8s.io/client-go/pkg/apis/clientauthentication/v1"
    45  	clientauthenticationv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1"
    46  	"k8s.io/client-go/tools/clientcmd/api"
    47  	"k8s.io/client-go/tools/metrics"
    48  	"k8s.io/client-go/transport"
    49  	"k8s.io/client-go/util/connrotation"
    50  	"k8s.io/klog/v2"
    51  	"k8s.io/utils/clock"
    52  )
    53  
    54  const execInfoEnv = "KUBERNETES_EXEC_INFO"
    55  const installHintVerboseHelp = `
    56  
    57  It looks like you are trying to use a client-go credential plugin that is not installed.
    58  
    59  To learn more about this feature, consult the documentation available at:
    60        https://kubernetes.io/docs/reference/access-authn-authz/authentication/#client-go-credential-plugins`
    61  
    62  var scheme = runtime.NewScheme()
    63  var codecs = serializer.NewCodecFactory(scheme)
    64  
    65  func init() {
    66  	install.Install(scheme)
    67  }
    68  
    69  var (
    70  	// Since transports can be constantly re-initialized by programs like kubectl,
    71  	// keep a cache of initialized authenticators keyed by a hash of their config.
    72  	globalCache = newCache()
    73  	// The list of API versions we accept.
    74  	apiVersions = map[string]schema.GroupVersion{
    75  		clientauthenticationv1beta1.SchemeGroupVersion.String(): clientauthenticationv1beta1.SchemeGroupVersion,
    76  		clientauthenticationv1.SchemeGroupVersion.String():      clientauthenticationv1.SchemeGroupVersion,
    77  	}
    78  )
    79  
    80  func newCache() *cache {
    81  	return &cache{m: make(map[string]*Authenticator)}
    82  }
    83  
    84  func cacheKey(conf *api.ExecConfig, cluster *clientauthentication.Cluster) string {
    85  	key := struct {
    86  		conf    *api.ExecConfig
    87  		cluster *clientauthentication.Cluster
    88  	}{
    89  		conf:    conf,
    90  		cluster: cluster,
    91  	}
    92  	return dump.Pretty(key)
    93  }
    94  
    95  type cache struct {
    96  	mu sync.Mutex
    97  	m  map[string]*Authenticator
    98  }
    99  
   100  func (c *cache) get(s string) (*Authenticator, bool) {
   101  	c.mu.Lock()
   102  	defer c.mu.Unlock()
   103  	a, ok := c.m[s]
   104  	return a, ok
   105  }
   106  
   107  // put inserts an authenticator into the cache. If an authenticator is already
   108  // associated with the key, the first one is returned instead.
   109  func (c *cache) put(s string, a *Authenticator) *Authenticator {
   110  	c.mu.Lock()
   111  	defer c.mu.Unlock()
   112  	existing, ok := c.m[s]
   113  	if ok {
   114  		return existing
   115  	}
   116  	c.m[s] = a
   117  	return a
   118  }
   119  
   120  // sometimes rate limits how often a function f() is called. Specifically, Do()
   121  // will run the provided function f() up to threshold times every interval
   122  // duration.
   123  type sometimes struct {
   124  	threshold int
   125  	interval  time.Duration
   126  
   127  	clock clock.Clock
   128  	mu    sync.Mutex
   129  
   130  	count  int       // times we have called f() in this window
   131  	window time.Time // beginning of current window of length interval
   132  }
   133  
   134  func (s *sometimes) Do(f func()) {
   135  	s.mu.Lock()
   136  	defer s.mu.Unlock()
   137  
   138  	now := s.clock.Now()
   139  	if s.window.IsZero() {
   140  		s.window = now
   141  	}
   142  
   143  	// If we are no longer in our saved time window, then we get to reset our run
   144  	// count back to 0 and start increasing towards the threshold again.
   145  	if inWindow := now.Sub(s.window) < s.interval; !inWindow {
   146  		s.window = now
   147  		s.count = 0
   148  	}
   149  
   150  	// If we have not run the function more than threshold times in this current
   151  	// time window, we get to run it now!
   152  	if underThreshold := s.count < s.threshold; underThreshold {
   153  		s.count++
   154  		f()
   155  	}
   156  }
   157  
   158  // GetAuthenticator returns an exec-based plugin for providing client credentials.
   159  func GetAuthenticator(config *api.ExecConfig, cluster *clientauthentication.Cluster) (*Authenticator, error) {
   160  	return newAuthenticator(globalCache, term.IsTerminal, config, cluster)
   161  }
   162  
   163  func newAuthenticator(c *cache, isTerminalFunc func(int) bool, config *api.ExecConfig, cluster *clientauthentication.Cluster) (*Authenticator, error) {
   164  	key := cacheKey(config, cluster)
   165  	if a, ok := c.get(key); ok {
   166  		return a, nil
   167  	}
   168  
   169  	gv, ok := apiVersions[config.APIVersion]
   170  	if !ok {
   171  		return nil, fmt.Errorf("exec plugin: invalid apiVersion %q", config.APIVersion)
   172  	}
   173  
   174  	connTracker := connrotation.NewConnectionTracker()
   175  	defaultDialer := connrotation.NewDialerWithTracker(
   176  		(&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
   177  		connTracker,
   178  	)
   179  
   180  	a := &Authenticator{
   181  		cmd:                config.Command,
   182  		args:               config.Args,
   183  		group:              gv,
   184  		cluster:            cluster,
   185  		provideClusterInfo: config.ProvideClusterInfo,
   186  
   187  		installHint: config.InstallHint,
   188  		sometimes: &sometimes{
   189  			threshold: 10,
   190  			interval:  time.Hour,
   191  			clock:     clock.RealClock{},
   192  		},
   193  
   194  		stdin:           os.Stdin,
   195  		stderr:          os.Stderr,
   196  		interactiveFunc: func() (bool, error) { return isInteractive(isTerminalFunc, config) },
   197  		now:             time.Now,
   198  		environ:         os.Environ,
   199  
   200  		connTracker: connTracker,
   201  	}
   202  
   203  	for _, env := range config.Env {
   204  		a.env = append(a.env, env.Name+"="+env.Value)
   205  	}
   206  
   207  	// these functions are made comparable and stored in the cache so that repeated clientset
   208  	// construction with the same rest.Config results in a single TLS cache and Authenticator
   209  	a.getCert = &transport.GetCertHolder{GetCert: a.cert}
   210  	a.dial = &transport.DialHolder{Dial: defaultDialer.DialContext}
   211  
   212  	return c.put(key, a), nil
   213  }
   214  
   215  func isInteractive(isTerminalFunc func(int) bool, config *api.ExecConfig) (bool, error) {
   216  	var shouldBeInteractive bool
   217  	switch config.InteractiveMode {
   218  	case api.NeverExecInteractiveMode:
   219  		shouldBeInteractive = false
   220  	case api.IfAvailableExecInteractiveMode:
   221  		shouldBeInteractive = !config.StdinUnavailable && isTerminalFunc(int(os.Stdin.Fd()))
   222  	case api.AlwaysExecInteractiveMode:
   223  		if !isTerminalFunc(int(os.Stdin.Fd())) {
   224  			return false, errors.New("standard input is not a terminal")
   225  		}
   226  		if config.StdinUnavailable {
   227  			suffix := ""
   228  			if len(config.StdinUnavailableMessage) > 0 {
   229  				// only print extra ": <message>" if the user actually specified a message
   230  				suffix = fmt.Sprintf(": %s", config.StdinUnavailableMessage)
   231  			}
   232  			return false, fmt.Errorf("standard input is unavailable%s", suffix)
   233  		}
   234  		shouldBeInteractive = true
   235  	default:
   236  		return false, fmt.Errorf("unknown interactiveMode: %q", config.InteractiveMode)
   237  	}
   238  
   239  	return shouldBeInteractive, nil
   240  }
   241  
   242  // Authenticator is a client credential provider that rotates credentials by executing a plugin.
   243  // The plugin input and output are defined by the API group client.authentication.k8s.io.
   244  type Authenticator struct {
   245  	// Set by the config
   246  	cmd                string
   247  	args               []string
   248  	group              schema.GroupVersion
   249  	env                []string
   250  	cluster            *clientauthentication.Cluster
   251  	provideClusterInfo bool
   252  
   253  	// Used to avoid log spew by rate limiting install hint printing. We didn't do
   254  	// this by interval based rate limiting alone since that way may have prevented
   255  	// the install hint from showing up for kubectl users.
   256  	sometimes   *sometimes
   257  	installHint string
   258  
   259  	// Stubbable for testing
   260  	stdin           io.Reader
   261  	stderr          io.Writer
   262  	interactiveFunc func() (bool, error)
   263  	now             func() time.Time
   264  	environ         func() []string
   265  
   266  	// connTracker tracks all connections opened that we need to close when rotating a client certificate
   267  	connTracker *connrotation.ConnectionTracker
   268  
   269  	// Cached results.
   270  	//
   271  	// The mutex also guards calling the plugin. Since the plugin could be
   272  	// interactive we want to make sure it's only called once.
   273  	mu          sync.Mutex
   274  	cachedCreds *credentials
   275  	exp         time.Time
   276  
   277  	// getCert makes Authenticator.cert comparable to support TLS config caching
   278  	getCert *transport.GetCertHolder
   279  	// dial is used for clients which do not specify a custom dialer
   280  	// it is comparable to support TLS config caching
   281  	dial *transport.DialHolder
   282  }
   283  
   284  type credentials struct {
   285  	token string           `datapolicy:"token"`
   286  	cert  *tls.Certificate `datapolicy:"secret-key"`
   287  }
   288  
   289  // UpdateTransportConfig updates the transport.Config to use credentials
   290  // returned by the plugin.
   291  func (a *Authenticator) UpdateTransportConfig(c *transport.Config) error {
   292  	// If a bearer token is present in the request - avoid the GetCert callback when
   293  	// setting up the transport, as that triggers the exec action if the server is
   294  	// also configured to allow client certificates for authentication. For requests
   295  	// like "kubectl get --token (token) pods" we should assume the intention is to
   296  	// use the provided token for authentication. The same can be said for when the
   297  	// user specifies basic auth or cert auth.
   298  	if c.HasTokenAuth() || c.HasBasicAuth() || c.HasCertAuth() {
   299  		return nil
   300  	}
   301  
   302  	c.Wrap(func(rt http.RoundTripper) http.RoundTripper {
   303  		return &roundTripper{a, rt}
   304  	})
   305  
   306  	if c.HasCertCallback() {
   307  		return errors.New("can't add TLS certificate callback: transport.Config.TLS.GetCert already set")
   308  	}
   309  	c.TLS.GetCertHolder = a.getCert // comparable for TLS config caching
   310  
   311  	if c.DialHolder != nil {
   312  		if c.DialHolder.Dial == nil {
   313  			return errors.New("invalid transport.Config.DialHolder: wrapped Dial function is nil")
   314  		}
   315  
   316  		// if c has a custom dialer, we have to wrap it
   317  		// TLS config caching is not supported for this config
   318  		d := connrotation.NewDialerWithTracker(c.DialHolder.Dial, a.connTracker)
   319  		c.DialHolder = &transport.DialHolder{Dial: d.DialContext}
   320  	} else {
   321  		c.DialHolder = a.dial // comparable for TLS config caching
   322  	}
   323  
   324  	return nil
   325  }
   326  
   327  var _ utilnet.RoundTripperWrapper = &roundTripper{}
   328  
   329  type roundTripper struct {
   330  	a    *Authenticator
   331  	base http.RoundTripper
   332  }
   333  
   334  func (r *roundTripper) WrappedRoundTripper() http.RoundTripper {
   335  	return r.base
   336  }
   337  
   338  func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
   339  	// If a user has already set credentials, use that. This makes commands like
   340  	// "kubectl get --token (token) pods" work.
   341  	if req.Header.Get("Authorization") != "" {
   342  		return r.base.RoundTrip(req)
   343  	}
   344  
   345  	creds, err := r.a.getCreds()
   346  	if err != nil {
   347  		return nil, fmt.Errorf("getting credentials: %v", err)
   348  	}
   349  	if creds.token != "" {
   350  		req.Header.Set("Authorization", "Bearer "+creds.token)
   351  	}
   352  
   353  	res, err := r.base.RoundTrip(req)
   354  	if err != nil {
   355  		return nil, err
   356  	}
   357  	if res.StatusCode == http.StatusUnauthorized {
   358  		if err := r.a.maybeRefreshCreds(creds); err != nil {
   359  			klog.Errorf("refreshing credentials: %v", err)
   360  		}
   361  	}
   362  	return res, nil
   363  }
   364  
   365  func (a *Authenticator) credsExpired() bool {
   366  	if a.exp.IsZero() {
   367  		return false
   368  	}
   369  	return a.now().After(a.exp)
   370  }
   371  
   372  func (a *Authenticator) cert() (*tls.Certificate, error) {
   373  	creds, err := a.getCreds()
   374  	if err != nil {
   375  		return nil, err
   376  	}
   377  	return creds.cert, nil
   378  }
   379  
   380  func (a *Authenticator) getCreds() (*credentials, error) {
   381  	a.mu.Lock()
   382  	defer a.mu.Unlock()
   383  
   384  	if a.cachedCreds != nil && !a.credsExpired() {
   385  		return a.cachedCreds, nil
   386  	}
   387  
   388  	if err := a.refreshCredsLocked(); err != nil {
   389  		return nil, err
   390  	}
   391  
   392  	return a.cachedCreds, nil
   393  }
   394  
   395  // maybeRefreshCreds executes the plugin to force a rotation of the
   396  // credentials, unless they were rotated already.
   397  func (a *Authenticator) maybeRefreshCreds(creds *credentials) error {
   398  	a.mu.Lock()
   399  	defer a.mu.Unlock()
   400  
   401  	// Since we're not making a new pointer to a.cachedCreds in getCreds, no
   402  	// need to do deep comparison.
   403  	if creds != a.cachedCreds {
   404  		// Credentials already rotated.
   405  		return nil
   406  	}
   407  
   408  	return a.refreshCredsLocked()
   409  }
   410  
   411  // refreshCredsLocked executes the plugin and reads the credentials from
   412  // stdout. It must be called while holding the Authenticator's mutex.
   413  func (a *Authenticator) refreshCredsLocked() error {
   414  	interactive, err := a.interactiveFunc()
   415  	if err != nil {
   416  		return fmt.Errorf("exec plugin cannot support interactive mode: %w", err)
   417  	}
   418  
   419  	cred := &clientauthentication.ExecCredential{
   420  		Spec: clientauthentication.ExecCredentialSpec{
   421  			Interactive: interactive,
   422  		},
   423  	}
   424  	if a.provideClusterInfo {
   425  		cred.Spec.Cluster = a.cluster
   426  	}
   427  
   428  	env := append(a.environ(), a.env...)
   429  	data, err := runtime.Encode(codecs.LegacyCodec(a.group), cred)
   430  	if err != nil {
   431  		return fmt.Errorf("encode ExecCredentials: %v", err)
   432  	}
   433  	env = append(env, fmt.Sprintf("%s=%s", execInfoEnv, data))
   434  
   435  	stdout := &bytes.Buffer{}
   436  	cmd := exec.Command(a.cmd, a.args...)
   437  	cmd.Env = env
   438  	cmd.Stderr = a.stderr
   439  	cmd.Stdout = stdout
   440  	if interactive {
   441  		cmd.Stdin = a.stdin
   442  	}
   443  
   444  	err = cmd.Run()
   445  	incrementCallsMetric(err)
   446  	if err != nil {
   447  		return a.wrapCmdRunErrorLocked(err)
   448  	}
   449  
   450  	_, gvk, err := codecs.UniversalDecoder(a.group).Decode(stdout.Bytes(), nil, cred)
   451  	if err != nil {
   452  		return fmt.Errorf("decoding stdout: %v", err)
   453  	}
   454  	if gvk.Group != a.group.Group || gvk.Version != a.group.Version {
   455  		return fmt.Errorf("exec plugin is configured to use API version %s, plugin returned version %s",
   456  			a.group, schema.GroupVersion{Group: gvk.Group, Version: gvk.Version})
   457  	}
   458  
   459  	if cred.Status == nil {
   460  		return fmt.Errorf("exec plugin didn't return a status field")
   461  	}
   462  	if cred.Status.Token == "" && cred.Status.ClientCertificateData == "" && cred.Status.ClientKeyData == "" {
   463  		return fmt.Errorf("exec plugin didn't return a token or cert/key pair")
   464  	}
   465  	if (cred.Status.ClientCertificateData == "") != (cred.Status.ClientKeyData == "") {
   466  		return fmt.Errorf("exec plugin returned only certificate or key, not both")
   467  	}
   468  
   469  	if cred.Status.ExpirationTimestamp != nil {
   470  		a.exp = cred.Status.ExpirationTimestamp.Time
   471  	} else {
   472  		a.exp = time.Time{}
   473  	}
   474  
   475  	newCreds := &credentials{
   476  		token: cred.Status.Token,
   477  	}
   478  	if cred.Status.ClientKeyData != "" && cred.Status.ClientCertificateData != "" {
   479  		cert, err := tls.X509KeyPair([]byte(cred.Status.ClientCertificateData), []byte(cred.Status.ClientKeyData))
   480  		if err != nil {
   481  			return fmt.Errorf("failed parsing client key/certificate: %v", err)
   482  		}
   483  
   484  		// Leaf is initialized to be nil:
   485  		//  https://golang.org/pkg/crypto/tls/#X509KeyPair
   486  		// Leaf certificate is the first certificate:
   487  		//  https://golang.org/pkg/crypto/tls/#Certificate
   488  		// Populating leaf is useful for quickly accessing the underlying x509
   489  		// certificate values.
   490  		cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
   491  		if err != nil {
   492  			return fmt.Errorf("failed parsing client leaf certificate: %v", err)
   493  		}
   494  		newCreds.cert = &cert
   495  	}
   496  
   497  	oldCreds := a.cachedCreds
   498  	a.cachedCreds = newCreds
   499  	// Only close all connections when TLS cert rotates. Token rotation doesn't
   500  	// need the extra noise.
   501  	if oldCreds != nil && !reflect.DeepEqual(oldCreds.cert, a.cachedCreds.cert) {
   502  		// Can be nil if the exec auth plugin only returned token auth.
   503  		if oldCreds.cert != nil && oldCreds.cert.Leaf != nil {
   504  			metrics.ClientCertRotationAge.Observe(time.Since(oldCreds.cert.Leaf.NotBefore))
   505  		}
   506  		a.connTracker.CloseAll()
   507  	}
   508  
   509  	expiry := time.Time{}
   510  	if a.cachedCreds.cert != nil && a.cachedCreds.cert.Leaf != nil {
   511  		expiry = a.cachedCreds.cert.Leaf.NotAfter
   512  	}
   513  	expirationMetrics.set(a, expiry)
   514  	return nil
   515  }
   516  
   517  // wrapCmdRunErrorLocked pulls out the code to construct a helpful error message
   518  // for when the exec plugin's binary fails to Run().
   519  //
   520  // It must be called while holding the Authenticator's mutex.
   521  func (a *Authenticator) wrapCmdRunErrorLocked(err error) error {
   522  	switch err.(type) {
   523  	case *exec.Error: // Binary does not exist (see exec.Error).
   524  		builder := strings.Builder{}
   525  		fmt.Fprintf(&builder, "exec: executable %s not found", a.cmd)
   526  
   527  		a.sometimes.Do(func() {
   528  			fmt.Fprint(&builder, installHintVerboseHelp)
   529  			if a.installHint != "" {
   530  				fmt.Fprintf(&builder, "\n\n%s", a.installHint)
   531  			}
   532  		})
   533  
   534  		return errors.New(builder.String())
   535  
   536  	case *exec.ExitError: // Binary execution failed (see exec.Cmd.Run()).
   537  		e := err.(*exec.ExitError)
   538  		return fmt.Errorf(
   539  			"exec: executable %s failed with exit code %d",
   540  			a.cmd,
   541  			e.ProcessState.ExitCode(),
   542  		)
   543  
   544  	default:
   545  		return fmt.Errorf("exec: %v", err)
   546  	}
   547  }