k8s.io/client-go@v0.31.1/plugin/pkg/client/auth/exec/exec.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package exec 18 19 import ( 20 "bytes" 21 "crypto/tls" 22 "crypto/x509" 23 "errors" 24 "fmt" 25 "io" 26 "net" 27 "net/http" 28 "os" 29 "os/exec" 30 "reflect" 31 "strings" 32 "sync" 33 "time" 34 35 "golang.org/x/term" 36 37 "k8s.io/apimachinery/pkg/runtime" 38 "k8s.io/apimachinery/pkg/runtime/schema" 39 "k8s.io/apimachinery/pkg/runtime/serializer" 40 "k8s.io/apimachinery/pkg/util/dump" 41 utilnet "k8s.io/apimachinery/pkg/util/net" 42 "k8s.io/client-go/pkg/apis/clientauthentication" 43 "k8s.io/client-go/pkg/apis/clientauthentication/install" 44 clientauthenticationv1 "k8s.io/client-go/pkg/apis/clientauthentication/v1" 45 clientauthenticationv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" 46 "k8s.io/client-go/tools/clientcmd/api" 47 "k8s.io/client-go/tools/metrics" 48 "k8s.io/client-go/transport" 49 "k8s.io/client-go/util/connrotation" 50 "k8s.io/klog/v2" 51 "k8s.io/utils/clock" 52 ) 53 54 const execInfoEnv = "KUBERNETES_EXEC_INFO" 55 const installHintVerboseHelp = ` 56 57 It looks like you are trying to use a client-go credential plugin that is not installed. 58 59 To learn more about this feature, consult the documentation available at: 60 https://kubernetes.io/docs/reference/access-authn-authz/authentication/#client-go-credential-plugins` 61 62 var scheme = runtime.NewScheme() 63 var codecs = serializer.NewCodecFactory(scheme) 64 65 func init() { 66 install.Install(scheme) 67 } 68 69 var ( 70 // Since transports can be constantly re-initialized by programs like kubectl, 71 // keep a cache of initialized authenticators keyed by a hash of their config. 72 globalCache = newCache() 73 // The list of API versions we accept. 74 apiVersions = map[string]schema.GroupVersion{ 75 clientauthenticationv1beta1.SchemeGroupVersion.String(): clientauthenticationv1beta1.SchemeGroupVersion, 76 clientauthenticationv1.SchemeGroupVersion.String(): clientauthenticationv1.SchemeGroupVersion, 77 } 78 ) 79 80 func newCache() *cache { 81 return &cache{m: make(map[string]*Authenticator)} 82 } 83 84 func cacheKey(conf *api.ExecConfig, cluster *clientauthentication.Cluster) string { 85 key := struct { 86 conf *api.ExecConfig 87 cluster *clientauthentication.Cluster 88 }{ 89 conf: conf, 90 cluster: cluster, 91 } 92 return dump.Pretty(key) 93 } 94 95 type cache struct { 96 mu sync.Mutex 97 m map[string]*Authenticator 98 } 99 100 func (c *cache) get(s string) (*Authenticator, bool) { 101 c.mu.Lock() 102 defer c.mu.Unlock() 103 a, ok := c.m[s] 104 return a, ok 105 } 106 107 // put inserts an authenticator into the cache. If an authenticator is already 108 // associated with the key, the first one is returned instead. 109 func (c *cache) put(s string, a *Authenticator) *Authenticator { 110 c.mu.Lock() 111 defer c.mu.Unlock() 112 existing, ok := c.m[s] 113 if ok { 114 return existing 115 } 116 c.m[s] = a 117 return a 118 } 119 120 // sometimes rate limits how often a function f() is called. Specifically, Do() 121 // will run the provided function f() up to threshold times every interval 122 // duration. 123 type sometimes struct { 124 threshold int 125 interval time.Duration 126 127 clock clock.Clock 128 mu sync.Mutex 129 130 count int // times we have called f() in this window 131 window time.Time // beginning of current window of length interval 132 } 133 134 func (s *sometimes) Do(f func()) { 135 s.mu.Lock() 136 defer s.mu.Unlock() 137 138 now := s.clock.Now() 139 if s.window.IsZero() { 140 s.window = now 141 } 142 143 // If we are no longer in our saved time window, then we get to reset our run 144 // count back to 0 and start increasing towards the threshold again. 145 if inWindow := now.Sub(s.window) < s.interval; !inWindow { 146 s.window = now 147 s.count = 0 148 } 149 150 // If we have not run the function more than threshold times in this current 151 // time window, we get to run it now! 152 if underThreshold := s.count < s.threshold; underThreshold { 153 s.count++ 154 f() 155 } 156 } 157 158 // GetAuthenticator returns an exec-based plugin for providing client credentials. 159 func GetAuthenticator(config *api.ExecConfig, cluster *clientauthentication.Cluster) (*Authenticator, error) { 160 return newAuthenticator(globalCache, term.IsTerminal, config, cluster) 161 } 162 163 func newAuthenticator(c *cache, isTerminalFunc func(int) bool, config *api.ExecConfig, cluster *clientauthentication.Cluster) (*Authenticator, error) { 164 key := cacheKey(config, cluster) 165 if a, ok := c.get(key); ok { 166 return a, nil 167 } 168 169 gv, ok := apiVersions[config.APIVersion] 170 if !ok { 171 return nil, fmt.Errorf("exec plugin: invalid apiVersion %q", config.APIVersion) 172 } 173 174 connTracker := connrotation.NewConnectionTracker() 175 defaultDialer := connrotation.NewDialerWithTracker( 176 (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, 177 connTracker, 178 ) 179 180 a := &Authenticator{ 181 cmd: config.Command, 182 args: config.Args, 183 group: gv, 184 cluster: cluster, 185 provideClusterInfo: config.ProvideClusterInfo, 186 187 installHint: config.InstallHint, 188 sometimes: &sometimes{ 189 threshold: 10, 190 interval: time.Hour, 191 clock: clock.RealClock{}, 192 }, 193 194 stdin: os.Stdin, 195 stderr: os.Stderr, 196 interactiveFunc: func() (bool, error) { return isInteractive(isTerminalFunc, config) }, 197 now: time.Now, 198 environ: os.Environ, 199 200 connTracker: connTracker, 201 } 202 203 for _, env := range config.Env { 204 a.env = append(a.env, env.Name+"="+env.Value) 205 } 206 207 // these functions are made comparable and stored in the cache so that repeated clientset 208 // construction with the same rest.Config results in a single TLS cache and Authenticator 209 a.getCert = &transport.GetCertHolder{GetCert: a.cert} 210 a.dial = &transport.DialHolder{Dial: defaultDialer.DialContext} 211 212 return c.put(key, a), nil 213 } 214 215 func isInteractive(isTerminalFunc func(int) bool, config *api.ExecConfig) (bool, error) { 216 var shouldBeInteractive bool 217 switch config.InteractiveMode { 218 case api.NeverExecInteractiveMode: 219 shouldBeInteractive = false 220 case api.IfAvailableExecInteractiveMode: 221 shouldBeInteractive = !config.StdinUnavailable && isTerminalFunc(int(os.Stdin.Fd())) 222 case api.AlwaysExecInteractiveMode: 223 if !isTerminalFunc(int(os.Stdin.Fd())) { 224 return false, errors.New("standard input is not a terminal") 225 } 226 if config.StdinUnavailable { 227 suffix := "" 228 if len(config.StdinUnavailableMessage) > 0 { 229 // only print extra ": <message>" if the user actually specified a message 230 suffix = fmt.Sprintf(": %s", config.StdinUnavailableMessage) 231 } 232 return false, fmt.Errorf("standard input is unavailable%s", suffix) 233 } 234 shouldBeInteractive = true 235 default: 236 return false, fmt.Errorf("unknown interactiveMode: %q", config.InteractiveMode) 237 } 238 239 return shouldBeInteractive, nil 240 } 241 242 // Authenticator is a client credential provider that rotates credentials by executing a plugin. 243 // The plugin input and output are defined by the API group client.authentication.k8s.io. 244 type Authenticator struct { 245 // Set by the config 246 cmd string 247 args []string 248 group schema.GroupVersion 249 env []string 250 cluster *clientauthentication.Cluster 251 provideClusterInfo bool 252 253 // Used to avoid log spew by rate limiting install hint printing. We didn't do 254 // this by interval based rate limiting alone since that way may have prevented 255 // the install hint from showing up for kubectl users. 256 sometimes *sometimes 257 installHint string 258 259 // Stubbable for testing 260 stdin io.Reader 261 stderr io.Writer 262 interactiveFunc func() (bool, error) 263 now func() time.Time 264 environ func() []string 265 266 // connTracker tracks all connections opened that we need to close when rotating a client certificate 267 connTracker *connrotation.ConnectionTracker 268 269 // Cached results. 270 // 271 // The mutex also guards calling the plugin. Since the plugin could be 272 // interactive we want to make sure it's only called once. 273 mu sync.Mutex 274 cachedCreds *credentials 275 exp time.Time 276 277 // getCert makes Authenticator.cert comparable to support TLS config caching 278 getCert *transport.GetCertHolder 279 // dial is used for clients which do not specify a custom dialer 280 // it is comparable to support TLS config caching 281 dial *transport.DialHolder 282 } 283 284 type credentials struct { 285 token string `datapolicy:"token"` 286 cert *tls.Certificate `datapolicy:"secret-key"` 287 } 288 289 // UpdateTransportConfig updates the transport.Config to use credentials 290 // returned by the plugin. 291 func (a *Authenticator) UpdateTransportConfig(c *transport.Config) error { 292 // If a bearer token is present in the request - avoid the GetCert callback when 293 // setting up the transport, as that triggers the exec action if the server is 294 // also configured to allow client certificates for authentication. For requests 295 // like "kubectl get --token (token) pods" we should assume the intention is to 296 // use the provided token for authentication. The same can be said for when the 297 // user specifies basic auth or cert auth. 298 if c.HasTokenAuth() || c.HasBasicAuth() || c.HasCertAuth() { 299 return nil 300 } 301 302 c.Wrap(func(rt http.RoundTripper) http.RoundTripper { 303 return &roundTripper{a, rt} 304 }) 305 306 if c.HasCertCallback() { 307 return errors.New("can't add TLS certificate callback: transport.Config.TLS.GetCert already set") 308 } 309 c.TLS.GetCertHolder = a.getCert // comparable for TLS config caching 310 311 if c.DialHolder != nil { 312 if c.DialHolder.Dial == nil { 313 return errors.New("invalid transport.Config.DialHolder: wrapped Dial function is nil") 314 } 315 316 // if c has a custom dialer, we have to wrap it 317 // TLS config caching is not supported for this config 318 d := connrotation.NewDialerWithTracker(c.DialHolder.Dial, a.connTracker) 319 c.DialHolder = &transport.DialHolder{Dial: d.DialContext} 320 } else { 321 c.DialHolder = a.dial // comparable for TLS config caching 322 } 323 324 return nil 325 } 326 327 var _ utilnet.RoundTripperWrapper = &roundTripper{} 328 329 type roundTripper struct { 330 a *Authenticator 331 base http.RoundTripper 332 } 333 334 func (r *roundTripper) WrappedRoundTripper() http.RoundTripper { 335 return r.base 336 } 337 338 func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 339 // If a user has already set credentials, use that. This makes commands like 340 // "kubectl get --token (token) pods" work. 341 if req.Header.Get("Authorization") != "" { 342 return r.base.RoundTrip(req) 343 } 344 345 creds, err := r.a.getCreds() 346 if err != nil { 347 return nil, fmt.Errorf("getting credentials: %v", err) 348 } 349 if creds.token != "" { 350 req.Header.Set("Authorization", "Bearer "+creds.token) 351 } 352 353 res, err := r.base.RoundTrip(req) 354 if err != nil { 355 return nil, err 356 } 357 if res.StatusCode == http.StatusUnauthorized { 358 if err := r.a.maybeRefreshCreds(creds); err != nil { 359 klog.Errorf("refreshing credentials: %v", err) 360 } 361 } 362 return res, nil 363 } 364 365 func (a *Authenticator) credsExpired() bool { 366 if a.exp.IsZero() { 367 return false 368 } 369 return a.now().After(a.exp) 370 } 371 372 func (a *Authenticator) cert() (*tls.Certificate, error) { 373 creds, err := a.getCreds() 374 if err != nil { 375 return nil, err 376 } 377 return creds.cert, nil 378 } 379 380 func (a *Authenticator) getCreds() (*credentials, error) { 381 a.mu.Lock() 382 defer a.mu.Unlock() 383 384 if a.cachedCreds != nil && !a.credsExpired() { 385 return a.cachedCreds, nil 386 } 387 388 if err := a.refreshCredsLocked(); err != nil { 389 return nil, err 390 } 391 392 return a.cachedCreds, nil 393 } 394 395 // maybeRefreshCreds executes the plugin to force a rotation of the 396 // credentials, unless they were rotated already. 397 func (a *Authenticator) maybeRefreshCreds(creds *credentials) error { 398 a.mu.Lock() 399 defer a.mu.Unlock() 400 401 // Since we're not making a new pointer to a.cachedCreds in getCreds, no 402 // need to do deep comparison. 403 if creds != a.cachedCreds { 404 // Credentials already rotated. 405 return nil 406 } 407 408 return a.refreshCredsLocked() 409 } 410 411 // refreshCredsLocked executes the plugin and reads the credentials from 412 // stdout. It must be called while holding the Authenticator's mutex. 413 func (a *Authenticator) refreshCredsLocked() error { 414 interactive, err := a.interactiveFunc() 415 if err != nil { 416 return fmt.Errorf("exec plugin cannot support interactive mode: %w", err) 417 } 418 419 cred := &clientauthentication.ExecCredential{ 420 Spec: clientauthentication.ExecCredentialSpec{ 421 Interactive: interactive, 422 }, 423 } 424 if a.provideClusterInfo { 425 cred.Spec.Cluster = a.cluster 426 } 427 428 env := append(a.environ(), a.env...) 429 data, err := runtime.Encode(codecs.LegacyCodec(a.group), cred) 430 if err != nil { 431 return fmt.Errorf("encode ExecCredentials: %v", err) 432 } 433 env = append(env, fmt.Sprintf("%s=%s", execInfoEnv, data)) 434 435 stdout := &bytes.Buffer{} 436 cmd := exec.Command(a.cmd, a.args...) 437 cmd.Env = env 438 cmd.Stderr = a.stderr 439 cmd.Stdout = stdout 440 if interactive { 441 cmd.Stdin = a.stdin 442 } 443 444 err = cmd.Run() 445 incrementCallsMetric(err) 446 if err != nil { 447 return a.wrapCmdRunErrorLocked(err) 448 } 449 450 _, gvk, err := codecs.UniversalDecoder(a.group).Decode(stdout.Bytes(), nil, cred) 451 if err != nil { 452 return fmt.Errorf("decoding stdout: %v", err) 453 } 454 if gvk.Group != a.group.Group || gvk.Version != a.group.Version { 455 return fmt.Errorf("exec plugin is configured to use API version %s, plugin returned version %s", 456 a.group, schema.GroupVersion{Group: gvk.Group, Version: gvk.Version}) 457 } 458 459 if cred.Status == nil { 460 return fmt.Errorf("exec plugin didn't return a status field") 461 } 462 if cred.Status.Token == "" && cred.Status.ClientCertificateData == "" && cred.Status.ClientKeyData == "" { 463 return fmt.Errorf("exec plugin didn't return a token or cert/key pair") 464 } 465 if (cred.Status.ClientCertificateData == "") != (cred.Status.ClientKeyData == "") { 466 return fmt.Errorf("exec plugin returned only certificate or key, not both") 467 } 468 469 if cred.Status.ExpirationTimestamp != nil { 470 a.exp = cred.Status.ExpirationTimestamp.Time 471 } else { 472 a.exp = time.Time{} 473 } 474 475 newCreds := &credentials{ 476 token: cred.Status.Token, 477 } 478 if cred.Status.ClientKeyData != "" && cred.Status.ClientCertificateData != "" { 479 cert, err := tls.X509KeyPair([]byte(cred.Status.ClientCertificateData), []byte(cred.Status.ClientKeyData)) 480 if err != nil { 481 return fmt.Errorf("failed parsing client key/certificate: %v", err) 482 } 483 484 // Leaf is initialized to be nil: 485 // https://golang.org/pkg/crypto/tls/#X509KeyPair 486 // Leaf certificate is the first certificate: 487 // https://golang.org/pkg/crypto/tls/#Certificate 488 // Populating leaf is useful for quickly accessing the underlying x509 489 // certificate values. 490 cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) 491 if err != nil { 492 return fmt.Errorf("failed parsing client leaf certificate: %v", err) 493 } 494 newCreds.cert = &cert 495 } 496 497 oldCreds := a.cachedCreds 498 a.cachedCreds = newCreds 499 // Only close all connections when TLS cert rotates. Token rotation doesn't 500 // need the extra noise. 501 if oldCreds != nil && !reflect.DeepEqual(oldCreds.cert, a.cachedCreds.cert) { 502 // Can be nil if the exec auth plugin only returned token auth. 503 if oldCreds.cert != nil && oldCreds.cert.Leaf != nil { 504 metrics.ClientCertRotationAge.Observe(time.Since(oldCreds.cert.Leaf.NotBefore)) 505 } 506 a.connTracker.CloseAll() 507 } 508 509 expiry := time.Time{} 510 if a.cachedCreds.cert != nil && a.cachedCreds.cert.Leaf != nil { 511 expiry = a.cachedCreds.cert.Leaf.NotAfter 512 } 513 expirationMetrics.set(a, expiry) 514 return nil 515 } 516 517 // wrapCmdRunErrorLocked pulls out the code to construct a helpful error message 518 // for when the exec plugin's binary fails to Run(). 519 // 520 // It must be called while holding the Authenticator's mutex. 521 func (a *Authenticator) wrapCmdRunErrorLocked(err error) error { 522 switch err.(type) { 523 case *exec.Error: // Binary does not exist (see exec.Error). 524 builder := strings.Builder{} 525 fmt.Fprintf(&builder, "exec: executable %s not found", a.cmd) 526 527 a.sometimes.Do(func() { 528 fmt.Fprint(&builder, installHintVerboseHelp) 529 if a.installHint != "" { 530 fmt.Fprintf(&builder, "\n\n%s", a.installHint) 531 } 532 }) 533 534 return errors.New(builder.String()) 535 536 case *exec.ExitError: // Binary execution failed (see exec.Cmd.Run()). 537 e := err.(*exec.ExitError) 538 return fmt.Errorf( 539 "exec: executable %s failed with exit code %d", 540 a.cmd, 541 e.ProcessState.ExitCode(), 542 ) 543 544 default: 545 return fmt.Errorf("exec: %v", err) 546 } 547 }