github.com/adityamillind98/nomad@v0.11.8/nomad/vault.go (about) 1 package nomad 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "math/rand" 8 "strconv" 9 "strings" 10 "sync" 11 "sync/atomic" 12 "time" 13 14 tomb "gopkg.in/tomb.v2" 15 16 metrics "github.com/armon/go-metrics" 17 log "github.com/hashicorp/go-hclog" 18 multierror "github.com/hashicorp/go-multierror" 19 "github.com/hashicorp/nomad/nomad/structs" 20 "github.com/hashicorp/nomad/nomad/structs/config" 21 vapi "github.com/hashicorp/vault/api" 22 "github.com/mitchellh/mapstructure" 23 24 "golang.org/x/sync/errgroup" 25 "golang.org/x/time/rate" 26 ) 27 28 const ( 29 // vaultTokenCreateTTL is the duration the wrapped token for the client is 30 // valid for. The units are in seconds. 31 vaultTokenCreateTTL = "60s" 32 33 // minimumTokenTTL is the minimum Token TTL allowed for child tokens. 34 minimumTokenTTL = 5 * time.Minute 35 36 // defaultTokenTTL is the default Token TTL used when the passed token is a 37 // root token such that child tokens aren't being created against a role 38 // that has defined a TTL 39 defaultTokenTTL = "72h" 40 41 // requestRateLimit is the maximum number of requests per second Nomad will 42 // make against Vault 43 requestRateLimit rate.Limit = 500.0 44 45 // maxParallelRevokes is the maximum number of parallel Vault 46 // token revocation requests 47 maxParallelRevokes = 64 48 49 // vaultRevocationIntv is the interval at which Vault tokens that failed 50 // initial revocation are retried 51 vaultRevocationIntv = 5 * time.Minute 52 53 // vaultCapabilitiesLookupPath is the path to lookup the capabilities of 54 // ones token. 55 vaultCapabilitiesLookupPath = "sys/capabilities-self" 56 57 // vaultTokenRenewPath is the path used to renew our token 58 vaultTokenRenewPath = "auth/token/renew-self" 59 60 // vaultTokenLookupPath is the path used to lookup a token 61 vaultTokenLookupPath = "auth/token/lookup" 62 63 // vaultTokenRevokePath is the path used to revoke a token 64 vaultTokenRevokePath = "auth/token/revoke-accessor" 65 66 // vaultRoleLookupPath is the path to lookup a role 67 vaultRoleLookupPath = "auth/token/roles/%s" 68 69 // vaultRoleCreatePath is the path to create a token from a role 70 vaultTokenRoleCreatePath = "auth/token/create/%s" 71 ) 72 73 var ( 74 // vaultCapabilitiesCapability is the expected capability of Nomad's Vault 75 // token on the the path. The token must have at least one of the 76 // capabilities. 77 vaultCapabilitiesCapability = []string{"update", "root"} 78 79 // vaultTokenRenewCapability is the expected capability Nomad's 80 // Vault token should have on the path. The token must have at least one of 81 // the capabilities. 82 vaultTokenRenewCapability = []string{"update", "root"} 83 84 // vaultTokenLookupCapability is the expected capability Nomad's 85 // Vault token should have on the path. The token must have at least one of 86 // the capabilities. 87 vaultTokenLookupCapability = []string{"update", "root"} 88 89 // vaultTokenRevokeCapability is the expected capability Nomad's 90 // Vault token should have on the path. The token must have at least one of 91 // the capabilities. 92 vaultTokenRevokeCapability = []string{"update", "root"} 93 94 // vaultRoleLookupCapability is the the expected capability Nomad's Vault 95 // token should have on the path. The token must have at least one of the 96 // capabilities. 97 vaultRoleLookupCapability = []string{"read", "root"} 98 99 // vaultTokenRoleCreateCapability is the the expected capability Nomad's Vault 100 // token should have on the path. The token must have at least one of the 101 // capabilities. 102 vaultTokenRoleCreateCapability = []string{"update", "root"} 103 ) 104 105 // VaultClient is the Servers interface for interfacing with Vault 106 type VaultClient interface { 107 // SetActive activates or de-activates the Vault client. When active, token 108 // creation/lookup/revocation operation are allowed. 109 SetActive(active bool) 110 111 // SetConfig updates the config used by the Vault client 112 SetConfig(config *config.VaultConfig) error 113 114 // CreateToken takes an allocation and task and returns an appropriate Vault 115 // Secret 116 CreateToken(ctx context.Context, a *structs.Allocation, task string) (*vapi.Secret, error) 117 118 // LookupToken takes a token string and returns its capabilities. 119 LookupToken(ctx context.Context, token string) (*vapi.Secret, error) 120 121 // RevokeTokens takes a set of tokens accessor and revokes the tokens 122 RevokeTokens(ctx context.Context, accessors []*structs.VaultAccessor, committed bool) error 123 124 // MarkForRevocation revokes the tokens in background 125 MarkForRevocation(accessors []*structs.VaultAccessor) error 126 127 // Stop is used to stop token renewal 128 Stop() 129 130 // Running returns whether the Vault client is running 131 Running() bool 132 133 // Stats returns the Vault clients statistics 134 Stats() map[string]string 135 136 // EmitStats emits that clients statistics at the given period until stopCh 137 // is called. 138 EmitStats(period time.Duration, stopCh <-chan struct{}) 139 } 140 141 // VaultStats returns all the stats about Vault tokens created and managed by 142 // Nomad. 143 type VaultStats struct { 144 // TrackedForRevoke is the count of tokens that are being tracked to be 145 // revoked since they could not be immediately revoked. 146 TrackedForRevoke int 147 148 // TokenTTL is the time-to-live duration for the current token 149 TokenTTL time.Duration 150 151 // TokenExpiry is the recorded expiry time of the current token 152 TokenExpiry time.Time 153 } 154 155 // PurgeVaultAccessorFn is called to remove VaultAccessors from the system. If 156 // the function returns an error, the token will still be tracked and revocation 157 // will retry till there is a success 158 type PurgeVaultAccessorFn func(accessors []*structs.VaultAccessor) error 159 160 // tokenData holds the relevant information about the Vault token passed to the 161 // client. 162 type tokenData struct { 163 CreationTTL int `mapstructure:"creation_ttl"` 164 TTL int `mapstructure:"ttl"` 165 Renewable bool `mapstructure:"renewable"` 166 Policies []string `mapstructure:"policies"` 167 Role string `mapstructure:"role"` 168 Root bool 169 } 170 171 // vaultClient is the Servers implementation of the VaultClient interface. The 172 // client renews the PeriodicToken given in the Vault configuration and provides 173 // the Server with the ability to create child tokens and lookup the permissions 174 // of tokens. 175 type vaultClient struct { 176 // limiter is used to rate limit requests to Vault 177 limiter *rate.Limiter 178 179 // client is the Vault API client used for Namespace-relative integrations 180 // with the Vault API (anything except `/v1/sys`). If this server is not 181 // configured to reference a Vault namespace, this will point to the same 182 // client as clientSys 183 client *vapi.Client 184 185 // clientSys is the Vault API client used for non-Namespace-relative integrations 186 // with the Vault API (anything involving `/v1/sys`). This client is never configured 187 // with a Vault namespace, because these endpoints may return errors if a namespace 188 // header is provided 189 clientSys *vapi.Client 190 191 // auth is the Vault token auth API client 192 auth *vapi.TokenAuth 193 194 // config is the user passed Vault config 195 config *config.VaultConfig 196 197 // connEstablished marks whether we have an established connection to Vault. 198 connEstablished bool 199 200 // connEstablishedErr marks an error that can occur when establishing a 201 // connection 202 connEstablishedErr error 203 204 // token is the raw token used by the client 205 token string 206 207 // tokenData is the data of the passed Vault token 208 tokenData *tokenData 209 210 // revoking tracks the VaultAccessors that must be revoked 211 revoking map[*structs.VaultAccessor]time.Time 212 purgeFn PurgeVaultAccessorFn 213 revLock sync.Mutex 214 215 // active indicates whether the vaultClient is active. It should be 216 // accessed using a helper and updated atomically 217 active int32 218 219 // running indicates whether the vault client is started. 220 running bool 221 222 // renewLoopActive indicates whether the renewal goroutine is running 223 // It should be accessed and updated atomically 224 // used for testing purposes only 225 renewLoopActive int32 226 227 // childTTL is the TTL for child tokens. 228 childTTL string 229 230 // currentExpiration is the time the current token lease expires 231 currentExpiration time.Time 232 currentExpirationLock sync.Mutex 233 234 tomb *tomb.Tomb 235 logger log.Logger 236 237 // l is used to lock the configuration aspects of the client such that 238 // multiple callers can't cause conflicting config updates 239 l sync.Mutex 240 241 // setConfigLock serializes access to the SetConfig method 242 setConfigLock sync.Mutex 243 244 // consts as struct fields for overriding in tests 245 maxRevokeBatchSize int 246 revocationIntv time.Duration 247 } 248 249 // NewVaultClient returns a Vault client from the given config. If the client 250 // couldn't be made an error is returned. 251 func NewVaultClient(c *config.VaultConfig, logger log.Logger, purgeFn PurgeVaultAccessorFn) (*vaultClient, error) { 252 if c == nil { 253 return nil, fmt.Errorf("must pass valid VaultConfig") 254 } 255 256 if logger == nil { 257 return nil, fmt.Errorf("must pass valid logger") 258 } 259 if purgeFn == nil { 260 purgeFn = func(accessors []*structs.VaultAccessor) error { return nil } 261 } 262 263 v := &vaultClient{ 264 config: c, 265 logger: logger.Named("vault"), 266 limiter: rate.NewLimiter(requestRateLimit, int(requestRateLimit)), 267 revoking: make(map[*structs.VaultAccessor]time.Time), 268 purgeFn: purgeFn, 269 tomb: &tomb.Tomb{}, 270 maxRevokeBatchSize: maxVaultRevokeBatchSize, 271 revocationIntv: vaultRevocationIntv, 272 } 273 274 if v.config.IsEnabled() { 275 if err := v.buildClient(); err != nil { 276 return nil, err 277 } 278 279 // Launch the required goroutines 280 v.tomb.Go(wrapNilError(v.establishConnection)) 281 v.tomb.Go(wrapNilError(v.revokeDaemon)) 282 283 v.running = true 284 } 285 286 return v, nil 287 } 288 289 func (v *vaultClient) Stop() { 290 v.l.Lock() 291 running := v.running 292 v.running = false 293 v.l.Unlock() 294 295 if running { 296 v.tomb.Kill(nil) 297 v.tomb.Wait() 298 v.flush() 299 } 300 } 301 302 func (v *vaultClient) Running() bool { 303 v.l.Lock() 304 defer v.l.Unlock() 305 return v.running 306 } 307 308 // SetActive activates or de-activates the Vault client. When active, token 309 // creation/lookup/revocation operation are allowed. All queued revocations are 310 // cancelled if set un-active as it is assumed another instances is taking over 311 func (v *vaultClient) SetActive(active bool) { 312 if active { 313 atomic.StoreInt32(&v.active, 1) 314 } else { 315 atomic.StoreInt32(&v.active, 0) 316 } 317 318 // Clear out the revoking tokens 319 v.revLock.Lock() 320 v.revoking = make(map[*structs.VaultAccessor]time.Time) 321 v.revLock.Unlock() 322 323 return 324 } 325 326 // flush is used to reset the state of the vault client 327 func (v *vaultClient) flush() { 328 v.l.Lock() 329 defer v.l.Unlock() 330 v.revLock.Lock() 331 defer v.revLock.Unlock() 332 333 v.client = nil 334 v.clientSys = nil 335 v.auth = nil 336 v.connEstablished = false 337 v.connEstablishedErr = nil 338 v.token = "" 339 v.tokenData = nil 340 v.revoking = make(map[*structs.VaultAccessor]time.Time) 341 v.childTTL = "" 342 v.tomb = &tomb.Tomb{} 343 } 344 345 // SetConfig is used to update the Vault config being used. A temporary outage 346 // may occur after calling as it re-establishes a connection to Vault 347 func (v *vaultClient) SetConfig(config *config.VaultConfig) error { 348 if config == nil { 349 return fmt.Errorf("must pass valid VaultConfig") 350 } 351 v.setConfigLock.Lock() 352 defer v.setConfigLock.Unlock() 353 354 v.l.Lock() 355 defer v.l.Unlock() 356 357 // If reloading the same config, no-op 358 if v.config.IsEqual(config) { 359 return nil 360 } 361 362 // Kill any background routines 363 if v.running { 364 // Kill any background routine 365 v.tomb.Kill(nil) 366 367 // Locking around tomb.Wait can deadlock with 368 // establishConnection exiting, so we must unlock here. 369 v.l.Unlock() 370 v.tomb.Wait() 371 v.l.Lock() 372 373 // Stop accepting any new requests 374 v.connEstablished = false 375 v.tomb = &tomb.Tomb{} 376 v.running = false 377 } 378 379 // Store the new config 380 v.config = config 381 382 // Check if we should relaunch 383 if v.config.IsEnabled() { 384 // Rebuild the client 385 if err := v.buildClient(); err != nil { 386 return err 387 } 388 389 // Launch the required goroutines 390 v.tomb.Go(wrapNilError(v.establishConnection)) 391 v.tomb.Go(wrapNilError(v.revokeDaemon)) 392 v.running = true 393 } 394 395 return nil 396 } 397 398 // buildClient is used to build a Vault client based on the stored Vault config 399 func (v *vaultClient) buildClient() error { 400 // Validate we have the required fields. 401 if v.config.Token == "" { 402 return errors.New("Vault token must be set") 403 } else if v.config.Addr == "" { 404 return errors.New("Vault address must be set") 405 } 406 407 // Parse the TTL if it is set 408 if v.config.TaskTokenTTL != "" { 409 d, err := time.ParseDuration(v.config.TaskTokenTTL) 410 if err != nil { 411 return fmt.Errorf("failed to parse TaskTokenTTL %q: %v", v.config.TaskTokenTTL, err) 412 } 413 414 if d.Nanoseconds() < minimumTokenTTL.Nanoseconds() { 415 return fmt.Errorf("ChildTokenTTL is less than minimum allowed of %v", minimumTokenTTL) 416 } 417 418 v.childTTL = v.config.TaskTokenTTL 419 } else { 420 // Default the TaskTokenTTL 421 v.childTTL = defaultTokenTTL 422 } 423 424 // Get the Vault API configuration 425 apiConf, err := v.config.ApiConfig() 426 if err != nil { 427 return fmt.Errorf("Failed to create Vault API config: %v", err) 428 } 429 430 // Create the Vault API client 431 client, err := vapi.NewClient(apiConf) 432 if err != nil { 433 v.logger.Error("failed to create Vault client and not retrying", "error", err) 434 return err 435 } 436 437 // Store the client, create/assign the /sys client 438 v.client = client 439 if v.config.Namespace != "" { 440 v.logger.Debug("configuring Vault namespace", "namespace", v.config.Namespace) 441 v.clientSys, err = vapi.NewClient(apiConf) 442 if err != nil { 443 v.logger.Error("failed to create Vault sys client and not retrying", "error", err) 444 return err 445 } 446 client.SetNamespace(v.config.Namespace) 447 } else { 448 v.clientSys = client 449 } 450 451 // Set the token 452 v.token = v.config.Token 453 client.SetToken(v.token) 454 v.auth = client.Auth().Token() 455 456 return nil 457 } 458 459 // establishConnection is used to make first contact with Vault. This should be 460 // called in a go-routine since the connection is retried until the Vault Client 461 // is stopped or the connection is successfully made at which point the renew 462 // loop is started. 463 func (v *vaultClient) establishConnection() { 464 // Create the retry timer and set initial duration to zero so it fires 465 // immediately 466 retryTimer := time.NewTimer(0) 467 initStatus := false 468 OUTER: 469 for { 470 select { 471 case <-v.tomb.Dying(): 472 return 473 case <-retryTimer.C: 474 // Ensure the API is reachable 475 if !initStatus { 476 if _, err := v.clientSys.Sys().InitStatus(); err != nil { 477 v.logger.Warn("failed to contact Vault API", "retry", v.config.ConnectionRetryIntv, "error", err) 478 retryTimer.Reset(v.config.ConnectionRetryIntv) 479 continue OUTER 480 } 481 initStatus = true 482 } 483 // Retry validating the token till success 484 if err := v.parseSelfToken(); err != nil { 485 v.logger.Error("failed to validate self token/role", "retry", v.config.ConnectionRetryIntv, "error", err) 486 retryTimer.Reset(v.config.ConnectionRetryIntv) 487 v.l.Lock() 488 v.connEstablished = true 489 v.connEstablishedErr = fmt.Errorf("failed to establish connection to Vault: %v", err) 490 v.l.Unlock() 491 continue OUTER 492 } 493 break OUTER 494 } 495 } 496 497 // Set the wrapping function such that token creation is wrapped now 498 // that we know our role 499 v.client.SetWrappingLookupFunc(v.getWrappingFn()) 500 501 // If we are given a non-root token, start renewing it 502 if v.tokenData.Root && v.tokenData.CreationTTL == 0 { 503 v.logger.Debug("not renewing token as it is root") 504 } else { 505 v.logger.Debug("starting renewal loop", "creation_ttl", time.Duration(v.tokenData.CreationTTL)*time.Second) 506 v.tomb.Go(wrapNilError(v.renewalLoop)) 507 } 508 509 v.l.Lock() 510 v.connEstablished = true 511 v.connEstablishedErr = nil 512 v.l.Unlock() 513 } 514 515 func (v *vaultClient) isRenewLoopActive() bool { 516 return atomic.LoadInt32(&v.renewLoopActive) == 1 517 } 518 519 // renewalLoop runs the renew loop. This should only be called if we are given a 520 // non-root token. 521 func (v *vaultClient) renewalLoop() { 522 atomic.StoreInt32(&v.renewLoopActive, 1) 523 defer atomic.StoreInt32(&v.renewLoopActive, 0) 524 525 // Create the renewal timer and set initial duration to zero so it fires 526 // immediately 527 authRenewTimer := time.NewTimer(0) 528 529 // Backoff is to reduce the rate we try to renew with Vault under error 530 // situations 531 backoff := 0.0 532 533 for { 534 select { 535 case <-v.tomb.Dying(): 536 return 537 case <-authRenewTimer.C: 538 // Renew the token and determine the new expiration 539 recoverable, err := v.renew() 540 v.currentExpirationLock.Lock() 541 currentExpiration := v.currentExpiration 542 v.currentExpirationLock.Unlock() 543 544 // Successfully renewed 545 if err == nil { 546 // Attempt to renew the token at half the expiration time 547 durationUntilRenew := currentExpiration.Sub(time.Now()) / 2 548 549 v.logger.Info("successfully renewed token", "next_renewal", durationUntilRenew) 550 authRenewTimer.Reset(durationUntilRenew) 551 552 // Reset any backoff 553 backoff = 0 554 break 555 } 556 557 metrics.IncrCounter([]string{"nomad", "vault", "renew_failed"}, 1) 558 v.logger.Warn("got error or bad auth, so backing off", "error", err, "recoverable", recoverable) 559 560 if !recoverable { 561 return 562 } 563 564 backoff = nextBackoff(backoff, currentExpiration) 565 if backoff < 0 { 566 // We have failed to renew the token past its expiration. Stop 567 // renewing with Vault. 568 v.logger.Error("failed to renew Vault token before lease expiration. Shutting down Vault client", 569 "error", err) 570 v.l.Lock() 571 v.connEstablished = false 572 v.connEstablishedErr = err 573 v.l.Unlock() 574 return 575 } 576 577 durationUntilRetry := time.Duration(backoff) * time.Second 578 v.logger.Info("backing off renewal", "retry", durationUntilRetry) 579 580 authRenewTimer.Reset(durationUntilRetry) 581 } 582 } 583 } 584 585 // nextBackoff returns the delay for the next auto renew interval, in seconds. 586 // Returns negative value if past expiration 587 // 588 // It should increase the amount of backoff each time, with the following rules: 589 // 590 // * If token expired already despite earlier renewal attempts, 591 // back off for 1 minute + jitter 592 // * If we have an existing authentication that is going to expire, 593 // never back off more than half of the amount of time remaining 594 // until expiration (with 5s floor) 595 // * Never back off more than 30 seconds multiplied by a random 596 // value between 1 and 2 597 // * Use randomness so that many clients won't keep hitting Vault 598 // at the same time 599 func nextBackoff(backoff float64, expiry time.Time) float64 { 600 maxBackoff := time.Until(expiry) / 2 601 602 if maxBackoff < 0 { 603 // expiry passed 604 return 60 * (1.0 + rand.Float64()) 605 } 606 607 switch { 608 case backoff >= 24: 609 backoff = 30 610 default: 611 backoff = backoff * 1.25 612 } 613 614 // Add randomness 615 backoff = backoff * (1.0 + rand.Float64()) 616 617 if backoff > maxBackoff.Seconds() { 618 backoff = maxBackoff.Seconds() 619 } 620 621 if backoff < 5 { 622 backoff = 5 623 } 624 625 return backoff 626 } 627 628 // renew attempts to renew our Vault token. If the renewal fails, an error is 629 // returned. The boolean indicates whether it's safe to attempt to renew again. 630 // This method updates the currentExpiration time 631 func (v *vaultClient) renew() (bool, error) { 632 // Track how long the request takes 633 defer metrics.MeasureSince([]string{"nomad", "vault", "renew"}, time.Now()) 634 635 // Attempt to renew the token 636 secret, err := v.auth.RenewSelf(v.tokenData.CreationTTL) 637 if err != nil { 638 // Check if there is a permission denied 639 recoverable := !structs.VaultUnrecoverableError.MatchString(err.Error()) 640 return recoverable, fmt.Errorf("failed to renew the vault token: %v", err) 641 } 642 643 if secret == nil { 644 // It's possible for RenewSelf to return (nil, nil) if the 645 // response body from Vault is empty. 646 return true, fmt.Errorf("renewal failed: empty response from vault") 647 } 648 649 // these treated as transient errors, where can keep renewing 650 auth := secret.Auth 651 if auth == nil { 652 return true, fmt.Errorf("renewal successful but not auth information returned") 653 } else if auth.LeaseDuration == 0 { 654 return true, fmt.Errorf("renewal successful but no lease duration returned") 655 } 656 657 v.extendExpiration(auth.LeaseDuration) 658 659 v.logger.Debug("successfully renewed server token") 660 return true, nil 661 } 662 663 // getWrappingFn returns an appropriate wrapping function for Nomad Servers 664 func (v *vaultClient) getWrappingFn() func(operation, path string) string { 665 createPath := "auth/token/create" 666 role := v.getRole() 667 if role != "" { 668 createPath = fmt.Sprintf("auth/token/create/%s", role) 669 } 670 671 return func(operation, path string) string { 672 // Only wrap the token create operation 673 if operation != "POST" || path != createPath { 674 return "" 675 } 676 677 return vaultTokenCreateTTL 678 } 679 } 680 681 // parseSelfToken looks up the Vault token in Vault and parses its data storing 682 // it in the client. If the token is not valid for Nomads purposes an error is 683 // returned. 684 func (v *vaultClient) parseSelfToken() error { 685 // Try looking up the token using the self endpoint 686 secret, err := v.lookupSelf() 687 if err != nil { 688 return err 689 } 690 691 // Read and parse the fields 692 var data tokenData 693 if err := mapstructure.WeakDecode(secret.Data, &data); err != nil { 694 return fmt.Errorf("failed to parse Vault token's data block: %v", err) 695 } 696 root := false 697 for _, p := range data.Policies { 698 if p == "root" { 699 root = true 700 break 701 } 702 } 703 data.Root = root 704 v.tokenData = &data 705 v.extendExpiration(data.TTL) 706 707 // The criteria that must be met for the token to be valid are as follows: 708 // 1) If token is non-root or is but has a creation ttl 709 // a) The token must be renewable 710 // b) Token must have a non-zero TTL 711 // 2) Must have update capability for "auth/token/lookup/" (used to verify incoming tokens) 712 // 3) Must have update capability for "/auth/token/revoke-accessor/" (used to revoke unneeded tokens) 713 // 4) If configured to create tokens against a role: 714 // a) Must have read capability for "auth/token/roles/<role_name" (Can just attempt a read) 715 // b) Must have update capability for path "auth/token/create/<role_name>" 716 // c) Role must: 717 // 1) Must allow tokens to be renewed 718 // 2) Must not have an explicit max TTL 719 // 3) Must have non-zero period 720 // 5) If not configured against a role, the token must be root 721 722 var mErr multierror.Error 723 role := v.getRole() 724 if !data.Root { 725 // All non-root tokens must be renewable 726 if !data.Renewable { 727 multierror.Append(&mErr, fmt.Errorf("Vault token is not renewable or root")) 728 } 729 730 // All non-root tokens must have a lease duration 731 if data.CreationTTL == 0 { 732 multierror.Append(&mErr, fmt.Errorf("invalid lease duration of zero")) 733 } 734 735 // The lease duration can not be expired 736 if data.TTL == 0 { 737 multierror.Append(&mErr, fmt.Errorf("token TTL is zero")) 738 } 739 740 // There must be a valid role since we aren't root 741 if role == "" { 742 multierror.Append(&mErr, fmt.Errorf("token role name must be set when not using a root token")) 743 } 744 745 } else if data.CreationTTL != 0 { 746 // If the root token has a TTL it must be renewable 747 if !data.Renewable { 748 multierror.Append(&mErr, fmt.Errorf("Vault token has a TTL but is not renewable")) 749 } else if data.TTL == 0 { 750 // If the token has a TTL make sure it has not expired 751 multierror.Append(&mErr, fmt.Errorf("token TTL is zero")) 752 } 753 } 754 755 // Check we have the correct capabilities 756 if err := v.validateCapabilities(role, data.Root); err != nil { 757 multierror.Append(&mErr, err) 758 } 759 760 // If given a role validate it 761 if role != "" { 762 if err := v.validateRole(role); err != nil { 763 multierror.Append(&mErr, err) 764 } 765 } 766 767 return mErr.ErrorOrNil() 768 } 769 770 // lookupSelf is a helper function that looks up latest self lease info. 771 func (v *vaultClient) lookupSelf() (*vapi.Secret, error) { 772 // Get the initial lease duration 773 auth := v.client.Auth().Token() 774 775 secret, err := auth.LookupSelf() 776 if err == nil && secret != nil && secret.Data != nil { 777 return secret, nil 778 } 779 780 // Try looking up our token directly, even when we get an empty response, 781 // in case of an unexpected event - a true failure would occur in this lookup again 782 secret, err = auth.Lookup(v.client.Token()) 783 switch { 784 case err != nil: 785 return nil, fmt.Errorf("failed to lookup Vault periodic token: %v", err) 786 case secret == nil || secret.Data == nil: 787 return nil, fmt.Errorf("failed to lookup Vault periodic token: got empty response") 788 default: 789 return secret, nil 790 } 791 } 792 793 // getRole returns the role name to be used when creating tokens 794 func (v *vaultClient) getRole() string { 795 if v.config.Role != "" { 796 return v.config.Role 797 } 798 799 return v.tokenData.Role 800 } 801 802 // validateCapabilities checks that Nomad's Vault token has the correct 803 // capabilities. 804 func (v *vaultClient) validateCapabilities(role string, root bool) error { 805 // Check if the token can lookup capabilities. 806 var mErr multierror.Error 807 _, _, err := v.hasCapability(vaultCapabilitiesLookupPath, vaultCapabilitiesCapability) 808 if err != nil { 809 // Check if there is a permission denied 810 if structs.VaultUnrecoverableError.MatchString(err.Error()) { 811 // Since we can't read permissions, we just log a warning that we 812 // can't tell if the Vault token will work 813 msg := fmt.Sprintf("can not lookup token capabilities. "+ 814 "As such certain operations may fail in the future. "+ 815 "Please give Nomad a Vault token with one of the following "+ 816 "capabilities %q on %q so that the required capabilities can be verified", 817 vaultCapabilitiesCapability, vaultCapabilitiesLookupPath) 818 v.logger.Warn(msg) 819 return nil 820 } else { 821 multierror.Append(&mErr, err) 822 } 823 } 824 825 // verify is a helper function that verifies the token has one of the 826 // capabilities on the given path and adds an issue to the error 827 verify := func(path string, requiredCaps []string) { 828 ok, caps, err := v.hasCapability(path, requiredCaps) 829 if err != nil { 830 multierror.Append(&mErr, err) 831 } else if !ok { 832 multierror.Append(&mErr, 833 fmt.Errorf("token must have one of the following capabilities %q on %q; has %v", requiredCaps, path, caps)) 834 } 835 } 836 837 // Check if we are verifying incoming tokens 838 if !v.config.AllowsUnauthenticated() { 839 verify(vaultTokenLookupPath, vaultTokenLookupCapability) 840 } 841 842 // Verify we can renew our selves tokens 843 verify(vaultTokenRenewPath, vaultTokenRenewCapability) 844 845 // Verify we can revoke tokens 846 verify(vaultTokenRevokePath, vaultTokenRevokeCapability) 847 848 // If we are using a role verify the capability 849 if role != "" { 850 // Verify we can read the role 851 verify(fmt.Sprintf(vaultRoleLookupPath, role), vaultRoleLookupCapability) 852 853 // Verify we can create from the role 854 verify(fmt.Sprintf(vaultTokenRoleCreatePath, role), vaultTokenRoleCreateCapability) 855 } 856 857 return mErr.ErrorOrNil() 858 } 859 860 // hasCapability takes a path and returns whether the token has at least one of 861 // the required capabilities on the given path. It also returns the set of 862 // capabilities the token does have as well as any error that occurred. 863 func (v *vaultClient) hasCapability(path string, required []string) (bool, []string, error) { 864 caps, err := v.client.Sys().CapabilitiesSelf(path) 865 if err != nil { 866 return false, nil, err 867 } 868 for _, c := range caps { 869 for _, r := range required { 870 if c == r { 871 return true, caps, nil 872 } 873 } 874 } 875 return false, caps, nil 876 } 877 878 // validateRole contacts Vault and checks that the given Vault role is valid for 879 // the purposes of being used by Nomad 880 func (v *vaultClient) validateRole(role string) error { 881 if role == "" { 882 return fmt.Errorf("Invalid empty role name") 883 } 884 885 // Validate the role 886 rsecret, err := v.client.Logical().Read(fmt.Sprintf("auth/token/roles/%s", role)) 887 if err != nil { 888 return fmt.Errorf("failed to lookup role %q: %v", role, err) 889 } 890 if rsecret == nil { 891 return fmt.Errorf("Role %q does not exist", role) 892 } 893 894 // Read and parse the fields 895 var data struct { 896 ExplicitMaxTtl int `mapstructure:"explicit_max_ttl"` 897 TokenExplicitMaxTtl int `mapstructure:"token_explicit_max_ttl"` 898 Orphan bool 899 Period int 900 TokenPeriod int `mapstructure:"token_period"` 901 Renewable bool 902 } 903 if err := mapstructure.WeakDecode(rsecret.Data, &data); err != nil { 904 return fmt.Errorf("failed to parse Vault role's data block: %v", err) 905 } 906 907 // Validate the role is acceptable 908 var mErr multierror.Error 909 if !data.Renewable { 910 multierror.Append(&mErr, fmt.Errorf("Role must allow tokens to be renewed")) 911 } 912 913 if data.ExplicitMaxTtl != 0 || data.TokenExplicitMaxTtl != 0 { 914 multierror.Append(&mErr, fmt.Errorf("Role can not use an explicit max ttl. Token must be periodic.")) 915 } 916 917 if data.Period == 0 && data.TokenPeriod == 0 { 918 multierror.Append(&mErr, fmt.Errorf("Role must have a non-zero period to make tokens periodic.")) 919 } 920 921 return mErr.ErrorOrNil() 922 } 923 924 // ConnectionEstablished returns whether a connection to Vault has been 925 // established and any error that potentially caused it to be false 926 func (v *vaultClient) ConnectionEstablished() (bool, error) { 927 v.l.Lock() 928 defer v.l.Unlock() 929 return v.connEstablished, v.connEstablishedErr 930 } 931 932 // Enabled returns whether the client is active 933 func (v *vaultClient) Enabled() bool { 934 v.l.Lock() 935 defer v.l.Unlock() 936 return v.config.IsEnabled() 937 } 938 939 // Active returns whether the client is active 940 func (v *vaultClient) Active() bool { 941 return atomic.LoadInt32(&v.active) == 1 942 } 943 944 // CreateToken takes the allocation and task and returns an appropriate Vault 945 // token. The call is rate limited and may be canceled with the passed policy. 946 // When the error is recoverable, it will be of type RecoverableError 947 func (v *vaultClient) CreateToken(ctx context.Context, a *structs.Allocation, task string) (*vapi.Secret, error) { 948 if !v.Enabled() { 949 return nil, fmt.Errorf("Vault integration disabled") 950 } 951 if !v.Active() { 952 return nil, structs.NewRecoverableError(fmt.Errorf("Vault client not active"), true) 953 } 954 955 // Check if we have established a connection with Vault 956 if established, err := v.ConnectionEstablished(); !established && err == nil { 957 return nil, structs.NewRecoverableError(fmt.Errorf("Connection to Vault has not been established"), true) 958 } else if err != nil { 959 return nil, err 960 } 961 962 // Track how long the request takes 963 defer metrics.MeasureSince([]string{"nomad", "vault", "create_token"}, time.Now()) 964 965 // Retrieve the Vault block for the task 966 policies := a.Job.VaultPolicies() 967 if policies == nil { 968 return nil, fmt.Errorf("Job doesn't require Vault policies") 969 } 970 tg, ok := policies[a.TaskGroup] 971 if !ok { 972 return nil, fmt.Errorf("Task group does not require Vault policies") 973 } 974 taskVault, ok := tg[task] 975 if !ok { 976 return nil, fmt.Errorf("Task does not require Vault policies") 977 } 978 979 // Build the creation request 980 req := &vapi.TokenCreateRequest{ 981 Policies: taskVault.Policies, 982 Metadata: map[string]string{ 983 "AllocationID": a.ID, 984 "Task": task, 985 "NodeID": a.NodeID, 986 }, 987 TTL: v.childTTL, 988 DisplayName: fmt.Sprintf("%s-%s", a.ID, task), 989 } 990 991 // Ensure we are under our rate limit 992 if err := v.limiter.Wait(ctx); err != nil { 993 return nil, err 994 } 995 996 // Make the request and switch depending on whether we are using a root 997 // token or a role based token 998 var secret *vapi.Secret 999 var err error 1000 role := v.getRole() 1001 if v.tokenData.Root && role == "" { 1002 req.Period = v.childTTL 1003 secret, err = v.auth.Create(req) 1004 } else { 1005 // Make the token using the role 1006 secret, err = v.auth.CreateWithRole(req, v.getRole()) 1007 } 1008 1009 // Determine whether it is unrecoverable 1010 if err != nil { 1011 err = fmt.Errorf("failed to create an alloc vault token: %v", err) 1012 if structs.VaultUnrecoverableError.MatchString(err.Error()) { 1013 return secret, err 1014 } 1015 1016 // The error is recoverable 1017 return nil, structs.NewRecoverableError(err, true) 1018 } 1019 1020 // Validate the response 1021 var validationErr error 1022 if secret == nil { 1023 validationErr = fmt.Errorf("Vault returned nil Secret") 1024 } else if secret.WrapInfo == nil { 1025 validationErr = fmt.Errorf("Vault returned Secret with nil WrapInfo. Secret warnings: %v", secret.Warnings) 1026 } else if secret.WrapInfo.WrappedAccessor == "" { 1027 validationErr = fmt.Errorf("Vault returned WrapInfo without WrappedAccessor. Secret warnings: %v", secret.Warnings) 1028 } 1029 if validationErr != nil { 1030 v.logger.Warn("failed to CreateToken", "error", validationErr) 1031 return nil, structs.NewRecoverableError(validationErr, true) 1032 } 1033 1034 // Got a valid response 1035 return secret, nil 1036 } 1037 1038 // LookupToken takes a Vault token and does a lookup against Vault. The call is 1039 // rate limited and may be canceled with passed context. 1040 func (v *vaultClient) LookupToken(ctx context.Context, token string) (*vapi.Secret, error) { 1041 if !v.Enabled() { 1042 return nil, fmt.Errorf("Vault integration disabled") 1043 } 1044 1045 if !v.Active() { 1046 return nil, fmt.Errorf("Vault client not active") 1047 } 1048 1049 // Check if we have established a connection with Vault 1050 if established, err := v.ConnectionEstablished(); !established && err == nil { 1051 return nil, structs.NewRecoverableError(fmt.Errorf("Connection to Vault has not been established"), true) 1052 } else if err != nil { 1053 return nil, err 1054 } 1055 1056 // Track how long the request takes 1057 defer metrics.MeasureSince([]string{"nomad", "vault", "lookup_token"}, time.Now()) 1058 1059 // Ensure we are under our rate limit 1060 if err := v.limiter.Wait(ctx); err != nil { 1061 return nil, err 1062 } 1063 1064 // Lookup the token 1065 return v.auth.Lookup(token) 1066 } 1067 1068 // PoliciesFrom parses the set of policies returned by a token lookup. 1069 func PoliciesFrom(s *vapi.Secret) ([]string, error) { 1070 if s == nil { 1071 return nil, fmt.Errorf("cannot parse nil Vault secret") 1072 } 1073 var data tokenData 1074 if err := mapstructure.WeakDecode(s.Data, &data); err != nil { 1075 return nil, fmt.Errorf("failed to parse Vault token's data block: %v", err) 1076 } 1077 1078 return data.Policies, nil 1079 } 1080 1081 // RevokeTokens revokes the passed set of accessors. If committed is set, the 1082 // purge function passed to the client is called. If there is an error purging 1083 // either because of Vault failures or because of the purge function, the 1084 // revocation is retried until the tokens TTL. 1085 func (v *vaultClient) RevokeTokens(ctx context.Context, accessors []*structs.VaultAccessor, committed bool) error { 1086 if !v.Enabled() { 1087 return nil 1088 } 1089 1090 if !v.Active() { 1091 return fmt.Errorf("Vault client not active") 1092 } 1093 1094 // Track how long the request takes 1095 defer metrics.MeasureSince([]string{"nomad", "vault", "revoke_tokens"}, time.Now()) 1096 1097 // Check if we have established a connection with Vault. If not just add it 1098 // to the queue 1099 if established, err := v.ConnectionEstablished(); !established && err == nil { 1100 // Only bother tracking it for later revocation if the accessor was 1101 // committed 1102 if committed { 1103 v.storeForRevocation(accessors) 1104 } 1105 1106 // Track that we are abandoning these accessors. 1107 metrics.IncrCounter([]string{"nomad", "vault", "undistributed_tokens_abandoned"}, float32(len(accessors))) 1108 return nil 1109 } 1110 1111 // Attempt to revoke immediately and if it fails, add it to the revoke queue 1112 err := v.parallelRevoke(ctx, accessors) 1113 if err != nil { 1114 // If it is uncommitted, it is a best effort revoke as it will shortly 1115 // TTL within the cubbyhole and has not been leaked to any outside 1116 // system 1117 if !committed { 1118 metrics.IncrCounter([]string{"nomad", "vault", "undistributed_tokens_abandoned"}, float32(len(accessors))) 1119 return nil 1120 } 1121 1122 v.logger.Warn("failed to revoke tokens. Will reattempt until TTL", "error", err) 1123 v.storeForRevocation(accessors) 1124 return nil 1125 } else if !committed { 1126 // Mark that it was revoked but there is nothing to purge so exit 1127 metrics.IncrCounter([]string{"nomad", "vault", "undistributed_tokens_revoked"}, float32(len(accessors))) 1128 return nil 1129 } 1130 1131 if err := v.purgeFn(accessors); err != nil { 1132 v.logger.Error("failed to purge Vault accessors", "error", err) 1133 v.storeForRevocation(accessors) 1134 return nil 1135 } 1136 1137 // Track that it was revoked successfully 1138 metrics.IncrCounter([]string{"nomad", "vault", "distributed_tokens_revoked"}, float32(len(accessors))) 1139 1140 return nil 1141 } 1142 1143 func (v *vaultClient) MarkForRevocation(accessors []*structs.VaultAccessor) error { 1144 if !v.Enabled() { 1145 return nil 1146 } 1147 1148 if !v.Active() { 1149 return fmt.Errorf("Vault client not active") 1150 } 1151 1152 v.storeForRevocation(accessors) 1153 return nil 1154 } 1155 1156 // storeForRevocation stores the passed set of accessors for revocation. It 1157 // captures their effective TTL by storing their create TTL plus the current 1158 // time. 1159 func (v *vaultClient) storeForRevocation(accessors []*structs.VaultAccessor) { 1160 v.revLock.Lock() 1161 1162 now := time.Now() 1163 for _, a := range accessors { 1164 if _, ok := v.revoking[a]; !ok { 1165 v.revoking[a] = now.Add(time.Duration(a.CreationTTL) * time.Second) 1166 } 1167 } 1168 v.revLock.Unlock() 1169 } 1170 1171 // parallelRevoke revokes the passed VaultAccessors in parallel. 1172 func (v *vaultClient) parallelRevoke(ctx context.Context, accessors []*structs.VaultAccessor) error { 1173 if !v.Enabled() { 1174 return fmt.Errorf("Vault integration disabled") 1175 } 1176 1177 if !v.Active() { 1178 return fmt.Errorf("Vault client not active") 1179 } 1180 1181 // Check if we have established a connection with Vault 1182 if established, err := v.ConnectionEstablished(); !established && err == nil { 1183 return structs.NewRecoverableError(fmt.Errorf("Connection to Vault has not been established"), true) 1184 } else if err != nil { 1185 return err 1186 } 1187 1188 g, pCtx := errgroup.WithContext(ctx) 1189 1190 // Cap the handlers 1191 handlers := len(accessors) 1192 if handlers > maxParallelRevokes { 1193 handlers = maxParallelRevokes 1194 } 1195 1196 // Revoke the Vault Token Accessors 1197 input := make(chan *structs.VaultAccessor, handlers) 1198 for i := 0; i < handlers; i++ { 1199 g.Go(func() error { 1200 for { 1201 select { 1202 case va, ok := <-input: 1203 if !ok { 1204 return nil 1205 } 1206 1207 err := v.auth.RevokeAccessor(va.Accessor) 1208 if err != nil && !strings.Contains(err.Error(), "invalid accessor") { 1209 return fmt.Errorf("failed to revoke token (alloc: %q, node: %q, task: %q): %v", va.AllocID, va.NodeID, va.Task, err) 1210 } 1211 case <-pCtx.Done(): 1212 return nil 1213 } 1214 } 1215 }) 1216 } 1217 1218 // Send the input 1219 go func() { 1220 defer close(input) 1221 for _, va := range accessors { 1222 select { 1223 case <-pCtx.Done(): 1224 return 1225 case input <- va: 1226 } 1227 } 1228 1229 }() 1230 1231 // Wait for everything to complete 1232 return g.Wait() 1233 } 1234 1235 // maxVaultRevokeBatchSize is the maximum tokens a revokeDaemon should revoke 1236 // and purge at any given time. 1237 // 1238 // Limiting the revocation batch size is beneficial for few reasons: 1239 // * A single revocation failure of any entry in batch result into retrying the whole batch; 1240 // the larger the batch is the higher likelihood of such failure 1241 // * Smaller batch sizes result into more co-operativeness: provides hooks for 1242 // reconsidering token TTL and leadership steps down. 1243 // * Batches limit the size of the Raft message purging tokens. Due to bugs 1244 // pre-0.11.3, expired tokens were not properly purged, so users upgrading from 1245 // older versions may have huge numbers (millions) of expired tokens to purge. 1246 const maxVaultRevokeBatchSize = 1000 1247 1248 // revokeDaemon should be called in a goroutine and is used to periodically 1249 // revoke Vault accessors that failed the original revocation 1250 func (v *vaultClient) revokeDaemon() { 1251 ticker := time.NewTicker(v.revocationIntv) 1252 defer ticker.Stop() 1253 1254 for { 1255 select { 1256 case <-v.tomb.Dying(): 1257 return 1258 case now := <-ticker.C: 1259 if established, _ := v.ConnectionEstablished(); !established { 1260 continue 1261 } 1262 1263 v.revLock.Lock() 1264 1265 // Fast path 1266 if len(v.revoking) == 0 { 1267 v.revLock.Unlock() 1268 continue 1269 } 1270 1271 // Build the list of accessors that need to be revoked while pruning any TTL'd checks 1272 toRevoke := len(v.revoking) 1273 if toRevoke > v.maxRevokeBatchSize { 1274 v.logger.Info("batching tokens to be revoked", 1275 "to_revoke", toRevoke, "batch_size", v.maxRevokeBatchSize, 1276 "batch_interval", v.revocationIntv) 1277 toRevoke = v.maxRevokeBatchSize 1278 } 1279 revoking := make([]*structs.VaultAccessor, 0, toRevoke) 1280 ttlExpired := []*structs.VaultAccessor{} 1281 for va, ttl := range v.revoking { 1282 if now.After(ttl) { 1283 ttlExpired = append(ttlExpired, va) 1284 } else { 1285 revoking = append(revoking, va) 1286 } 1287 1288 // Batches should consider tokens to be revoked 1289 // as well as expired tokens to ensure the Raft 1290 // message is reasonably sized. 1291 if len(revoking)+len(ttlExpired) >= toRevoke { 1292 break 1293 } 1294 } 1295 1296 if err := v.parallelRevoke(context.Background(), revoking); err != nil { 1297 v.logger.Warn("background token revocation errored", "error", err) 1298 v.revLock.Unlock() 1299 continue 1300 } 1301 1302 // Unlock before a potentially expensive operation 1303 v.revLock.Unlock() 1304 1305 // purge all explicitly revoked as well as ttl expired tokens 1306 // and only remove them locally on purge success 1307 revoking = append(revoking, ttlExpired...) 1308 1309 // Call the passed in token revocation function 1310 if err := v.purgeFn(revoking); err != nil { 1311 // Can continue since revocation is idempotent 1312 v.logger.Error("token revocation errored", "error", err) 1313 continue 1314 } 1315 1316 // Track that tokens were revoked successfully 1317 metrics.IncrCounter([]string{"nomad", "vault", "distributed_tokens_revoked"}, float32(len(revoking))) 1318 1319 // Can delete from the tracked list now that we have purged 1320 v.revLock.Lock() 1321 for _, va := range revoking { 1322 delete(v.revoking, va) 1323 } 1324 v.revLock.Unlock() 1325 1326 } 1327 } 1328 } 1329 1330 // purgeVaultAccessors creates a Raft transaction to remove the passed Vault 1331 // Accessors 1332 func (s *Server) purgeVaultAccessors(accessors []*structs.VaultAccessor) error { 1333 // Commit this update via Raft 1334 req := structs.VaultAccessorsRequest{Accessors: accessors} 1335 _, _, err := s.raftApply(structs.VaultAccessorDeregisterRequestType, req) 1336 return err 1337 } 1338 1339 // wrapNilError is a helper that returns a wrapped function that returns a nil 1340 // error 1341 func wrapNilError(f func()) func() error { 1342 return func() error { 1343 f() 1344 return nil 1345 } 1346 } 1347 1348 // setLimit is used to update the rate limit 1349 func (v *vaultClient) setLimit(l rate.Limit) { 1350 v.l.Lock() 1351 defer v.l.Unlock() 1352 v.limiter = rate.NewLimiter(l, int(l)) 1353 } 1354 1355 func (v *vaultClient) Stats() map[string]string { 1356 stat := v.stats() 1357 1358 expireTimeStr := "" 1359 1360 if !stat.TokenExpiry.IsZero() { 1361 expireTimeStr = stat.TokenExpiry.Format(time.RFC3339) 1362 } 1363 1364 return map[string]string{ 1365 "tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke), 1366 "token_ttl": stat.TokenTTL.Round(time.Second).String(), 1367 "token_expire_time": expireTimeStr, 1368 } 1369 } 1370 1371 func (v *vaultClient) stats() *VaultStats { 1372 // Allocate a new stats struct 1373 stats := new(VaultStats) 1374 1375 v.revLock.Lock() 1376 stats.TrackedForRevoke = len(v.revoking) 1377 v.revLock.Unlock() 1378 1379 v.currentExpirationLock.Lock() 1380 stats.TokenExpiry = v.currentExpiration 1381 v.currentExpirationLock.Unlock() 1382 1383 if !stats.TokenExpiry.IsZero() { 1384 stats.TokenTTL = time.Until(stats.TokenExpiry) 1385 } 1386 1387 return stats 1388 } 1389 1390 // EmitStats is used to export metrics about the blocked eval tracker while enabled 1391 func (v *vaultClient) EmitStats(period time.Duration, stopCh <-chan struct{}) { 1392 for { 1393 select { 1394 case <-time.After(period): 1395 stats := v.stats() 1396 metrics.SetGauge([]string{"nomad", "vault", "distributed_tokens_revoking"}, float32(stats.TrackedForRevoke)) 1397 metrics.SetGauge([]string{"nomad", "vault", "token_ttl"}, float32(stats.TokenTTL/time.Millisecond)) 1398 1399 case <-stopCh: 1400 return 1401 } 1402 } 1403 } 1404 1405 // extendExpiration sets the current auth token expiration record to ttLSeconds seconds from now 1406 func (v *vaultClient) extendExpiration(ttlSeconds int) { 1407 v.currentExpirationLock.Lock() 1408 v.currentExpiration = time.Now().Add(time.Duration(ttlSeconds) * time.Second) 1409 v.currentExpirationLock.Unlock() 1410 }