github.com/mattyr/nomad@v0.3.3-0.20160919021406-3485a065154a/nomad/vault.go (about) 1 package nomad 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "log" 8 "math/rand" 9 "sync" 10 "sync/atomic" 11 "time" 12 13 "gopkg.in/tomb.v2" 14 15 "github.com/hashicorp/nomad/nomad/structs" 16 "github.com/hashicorp/nomad/nomad/structs/config" 17 vapi "github.com/hashicorp/vault/api" 18 "github.com/mitchellh/mapstructure" 19 20 "golang.org/x/sync/errgroup" 21 "golang.org/x/time/rate" 22 ) 23 24 const ( 25 // vaultTokenCreateTTL is the duration the wrapped token for the client is 26 // valid for. The units are in seconds. 27 vaultTokenCreateTTL = "60s" 28 29 // minimumTokenTTL is the minimum Token TTL allowed for child tokens. 30 minimumTokenTTL = 5 * time.Minute 31 32 // defaultTokenTTL is the default Token TTL used when the passed token is a 33 // root token such that child tokens aren't being created against a role 34 // that has defined a TTL 35 defaultTokenTTL = "72h" 36 37 // requestRateLimit is the maximum number of requests per second Nomad will 38 // make against Vault 39 requestRateLimit rate.Limit = 500.0 40 41 // maxParallelRevokes is the maximum number of parallel Vault 42 // token revocation requests 43 maxParallelRevokes = 64 44 45 // vaultRevocationIntv is the interval at which Vault tokens that failed 46 // initial revocation are retried 47 vaultRevocationIntv = 5 * time.Minute 48 ) 49 50 // VaultClient is the Servers interface for interfacing with Vault 51 type VaultClient interface { 52 // SetActive activates or de-activates the Vault client. When active, token 53 // creation/lookup/revocation operation are allowed. 54 SetActive(active bool) 55 56 // SetConfig updates the config used by the Vault client 57 SetConfig(config *config.VaultConfig) error 58 59 // CreateToken takes an allocation and task and returns an appropriate Vault 60 // Secret 61 CreateToken(ctx context.Context, a *structs.Allocation, task string) (*vapi.Secret, error) 62 63 // LookupToken takes a token string and returns its capabilities. 64 LookupToken(ctx context.Context, token string) (*vapi.Secret, error) 65 66 // RevokeTokens takes a set of tokens accessor and revokes the tokens 67 RevokeTokens(ctx context.Context, accessors []*structs.VaultAccessor, committed bool) error 68 69 // Stop is used to stop token renewal 70 Stop() 71 } 72 73 // PurgeVaultAccessor is called to remove VaultAccessors from the system. If 74 // the function returns an error, the token will still be tracked and revocation 75 // will retry till there is a success 76 type PurgeVaultAccessorFn func(accessors []*structs.VaultAccessor) error 77 78 // tokenData holds the relevant information about the Vault token passed to the 79 // client. 80 type tokenData struct { 81 CreationTTL int `mapstructure:"creation_ttl"` 82 TTL int `mapstructure:"ttl"` 83 Renewable bool `mapstructure:"renewable"` 84 Policies []string `mapstructure:"policies"` 85 Role string `mapstructure:"role"` 86 Root bool 87 } 88 89 // vaultClient is the Servers implementation of the VaultClient interface. The 90 // client renews the PeriodicToken given in the Vault configuration and provides 91 // the Server with the ability to create child tokens and lookup the permissions 92 // of tokens. 93 type vaultClient struct { 94 // limiter is used to rate limit requests to Vault 95 limiter *rate.Limiter 96 97 // client is the Vault API client 98 client *vapi.Client 99 100 // auth is the Vault token auth API client 101 auth *vapi.TokenAuth 102 103 // config is the user passed Vault config 104 config *config.VaultConfig 105 106 // connEstablished marks whether we have an established connection to Vault. 107 // It should be accessed using a helper and updated atomically 108 connEstablished int32 109 110 // token is the raw token used by the client 111 token string 112 113 // tokenData is the data of the passed Vault token 114 tokenData *tokenData 115 116 // revoking tracks the VaultAccessors that must be revoked 117 revoking map[*structs.VaultAccessor]time.Time 118 purgeFn PurgeVaultAccessorFn 119 revLock sync.Mutex 120 121 // active indicates whether the vaultClient is active. It should be 122 // accessed using a helper and updated atomically 123 active int32 124 125 // running indicates whether the vault client is started. 126 running bool 127 128 // childTTL is the TTL for child tokens. 129 childTTL string 130 131 // lastRenewed is the time the token was last renewed 132 lastRenewed time.Time 133 134 tomb *tomb.Tomb 135 logger *log.Logger 136 137 // l is used to lock the configuration aspects of the client such that 138 // multiple callers can't cause conflicting config updates 139 l sync.Mutex 140 } 141 142 // NewVaultClient returns a Vault client from the given config. If the client 143 // couldn't be made an error is returned. 144 func NewVaultClient(c *config.VaultConfig, logger *log.Logger, purgeFn PurgeVaultAccessorFn) (*vaultClient, error) { 145 if c == nil { 146 return nil, fmt.Errorf("must pass valid VaultConfig") 147 } 148 149 if logger == nil { 150 return nil, fmt.Errorf("must pass valid logger") 151 } 152 153 v := &vaultClient{ 154 config: c, 155 logger: logger, 156 limiter: rate.NewLimiter(requestRateLimit, int(requestRateLimit)), 157 revoking: make(map[*structs.VaultAccessor]time.Time), 158 purgeFn: purgeFn, 159 tomb: &tomb.Tomb{}, 160 } 161 162 if v.config.Enabled { 163 if err := v.buildClient(); err != nil { 164 return nil, err 165 } 166 167 // Launch the required goroutines 168 v.tomb.Go(wrapNilError(v.establishConnection)) 169 v.tomb.Go(wrapNilError(v.revokeDaemon)) 170 171 v.running = true 172 } 173 174 return v, nil 175 } 176 177 func (v *vaultClient) Stop() { 178 v.l.Lock() 179 running := v.running 180 v.running = false 181 v.l.Unlock() 182 183 if running { 184 v.tomb.Kill(nil) 185 v.tomb.Wait() 186 v.flush() 187 } 188 } 189 190 // SetActive activates or de-activates the Vault client. When active, token 191 // creation/lookup/revocation operation are allowed. All queued revocations are 192 // cancelled if set un-active as it is assumed another instances is taking over 193 func (v *vaultClient) SetActive(active bool) { 194 atomic.StoreInt32(&v.active, 1) 195 return 196 } 197 198 // flush is used to reset the state of the vault client 199 func (v *vaultClient) flush() { 200 v.l.Lock() 201 defer v.l.Unlock() 202 203 v.client = nil 204 v.auth = nil 205 v.connEstablished = 0 206 v.token = "" 207 v.tokenData = nil 208 v.revoking = make(map[*structs.VaultAccessor]time.Time) 209 v.childTTL = "" 210 v.tomb = &tomb.Tomb{} 211 } 212 213 // SetConfig is used to update the Vault config being used. A temporary outage 214 // may occur after calling as it re-establishes a connection to Vault 215 func (v *vaultClient) SetConfig(config *config.VaultConfig) error { 216 if config == nil { 217 return fmt.Errorf("must pass valid VaultConfig") 218 } 219 220 v.l.Lock() 221 defer v.l.Unlock() 222 223 // Store the new config 224 v.config = config 225 226 if v.config.Enabled { 227 // Stop accepting any new request 228 atomic.StoreInt32(&v.connEstablished, 0) 229 230 // Kill any background routine and create a new tomb 231 v.tomb.Kill(nil) 232 v.tomb.Wait() 233 v.tomb = &tomb.Tomb{} 234 235 // Rebuild the client 236 if err := v.buildClient(); err != nil { 237 v.l.Unlock() 238 return err 239 } 240 241 // Launch the required goroutines 242 v.tomb.Go(wrapNilError(v.establishConnection)) 243 v.tomb.Go(wrapNilError(v.revokeDaemon)) 244 } 245 246 return nil 247 } 248 249 // buildClient is used to build a Vault client based on the stored Vault config 250 func (v *vaultClient) buildClient() error { 251 // Validate we have the required fields. 252 if v.config.Token == "" { 253 return errors.New("Vault token must be set") 254 } else if v.config.Addr == "" { 255 return errors.New("Vault address must be set") 256 } 257 258 // Parse the TTL if it is set 259 if v.config.TaskTokenTTL != "" { 260 d, err := time.ParseDuration(v.config.TaskTokenTTL) 261 if err != nil { 262 return fmt.Errorf("failed to parse TaskTokenTTL %q: %v", v.config.TaskTokenTTL, err) 263 } 264 265 if d.Nanoseconds() < minimumTokenTTL.Nanoseconds() { 266 return fmt.Errorf("ChildTokenTTL is less than minimum allowed of %v", minimumTokenTTL) 267 } 268 269 v.childTTL = v.config.TaskTokenTTL 270 } else { 271 // Default the TaskTokenTTL 272 v.childTTL = defaultTokenTTL 273 } 274 275 // Get the Vault API configuration 276 apiConf, err := v.config.ApiConfig() 277 if err != nil { 278 return fmt.Errorf("Failed to create Vault API config: %v", err) 279 } 280 281 // Create the Vault API client 282 client, err := vapi.NewClient(apiConf) 283 if err != nil { 284 v.logger.Printf("[ERR] vault: failed to create Vault client. Not retrying: %v", err) 285 return err 286 } 287 288 // Set the token and store the client 289 v.token = v.config.Token 290 client.SetToken(v.token) 291 v.client = client 292 v.auth = client.Auth().Token() 293 return nil 294 } 295 296 // establishConnection is used to make first contact with Vault. This should be 297 // called in a go-routine since the connection is retried til the Vault Client 298 // is stopped or the connection is successfully made at which point the renew 299 // loop is started. 300 func (v *vaultClient) establishConnection() { 301 // Create the retry timer and set initial duration to zero so it fires 302 // immediately 303 retryTimer := time.NewTimer(0) 304 305 OUTER: 306 for { 307 select { 308 case <-v.tomb.Dying(): 309 return 310 case <-retryTimer.C: 311 // Ensure the API is reachable 312 if _, err := v.client.Sys().InitStatus(); err != nil { 313 v.logger.Printf("[WARN] vault: failed to contact Vault API. Retrying in %v", 314 v.config.ConnectionRetryIntv) 315 retryTimer.Reset(v.config.ConnectionRetryIntv) 316 continue OUTER 317 } 318 319 break OUTER 320 } 321 } 322 323 atomic.StoreInt32(&v.connEstablished, 1) 324 325 // Retrieve our token, validate it and parse the lease duration 326 if err := v.parseSelfToken(); err != nil { 327 v.logger.Printf("[ERR] vault: failed to lookup self token and not retrying: %v", err) 328 return 329 } 330 331 // Set the wrapping function such that token creation is wrapped now 332 // that we know our role 333 v.client.SetWrappingLookupFunc(v.getWrappingFn()) 334 335 // If we are given a non-root token, start renewing it 336 if v.tokenData.Root { 337 v.logger.Printf("[DEBUG] vault: not renewing token as it is root") 338 } else { 339 v.logger.Printf("[DEBUG] vault: token lease duration is %v", 340 time.Duration(v.tokenData.CreationTTL)*time.Second) 341 v.tomb.Go(wrapNilError(v.renewalLoop)) 342 } 343 } 344 345 // renewalLoop runs the renew loop. This should only be called if we are given a 346 // non-root token. 347 func (v *vaultClient) renewalLoop() { 348 // Create the renewal timer and set initial duration to zero so it fires 349 // immediately 350 authRenewTimer := time.NewTimer(0) 351 352 // Backoff is to reduce the rate we try to renew with Vault under error 353 // situations 354 backoff := 0.0 355 356 for { 357 select { 358 case <-v.tomb.Dying(): 359 return 360 case <-authRenewTimer.C: 361 // Renew the token and determine the new expiration 362 err := v.renew() 363 currentExpiration := v.lastRenewed.Add(time.Duration(v.tokenData.CreationTTL) * time.Second) 364 365 // Successfully renewed 366 if err == nil { 367 // If we take the expiration (lastRenewed + auth duration) and 368 // subtract the current time, we get a duration until expiry. 369 // Set the timer to poke us after half of that time is up. 370 durationUntilRenew := currentExpiration.Sub(time.Now()) / 2 371 372 v.logger.Printf("[INFO] vault: renewing token in %v", durationUntilRenew) 373 authRenewTimer.Reset(durationUntilRenew) 374 375 // Reset any backoff 376 backoff = 0 377 break 378 } 379 380 // Back off, increasing the amount of backoff each time. There are some rules: 381 // 382 // * If we have an existing authentication that is going to expire, 383 // never back off more than half of the amount of time remaining 384 // until expiration 385 // * Never back off more than 30 seconds multiplied by a random 386 // value between 1 and 2 387 // * Use randomness so that many clients won't keep hitting Vault 388 // at the same time 389 390 // Set base values and add some backoff 391 392 v.logger.Printf("[DEBUG] vault: got error or bad auth, so backing off: %v", err) 393 switch { 394 case backoff < 5: 395 backoff = 5 396 case backoff >= 24: 397 backoff = 30 398 default: 399 backoff = backoff * 1.25 400 } 401 402 // Add randomness 403 backoff = backoff * (1.0 + rand.Float64()) 404 405 maxBackoff := currentExpiration.Sub(time.Now()) / 2 406 if maxBackoff < 0 { 407 // We have failed to renew the token past its expiration. Stop 408 // renewing with Vault. 409 v.logger.Printf("[ERR] vault: failed to renew Vault token before lease expiration. Shutting down Vault client") 410 atomic.StoreInt32(&v.connEstablished, 0) 411 return 412 413 } else if backoff > maxBackoff.Seconds() { 414 backoff = maxBackoff.Seconds() 415 } 416 417 durationUntilRetry := time.Duration(backoff) * time.Second 418 v.logger.Printf("[INFO] vault: backing off for %v", durationUntilRetry) 419 420 authRenewTimer.Reset(durationUntilRetry) 421 } 422 } 423 } 424 425 // renew attempts to renew our Vault token. If the renewal fails, an error is 426 // returned. This method updates the lastRenewed time 427 func (v *vaultClient) renew() error { 428 // Attempt to renew the token 429 secret, err := v.auth.RenewSelf(v.tokenData.CreationTTL) 430 if err != nil { 431 return err 432 } 433 434 auth := secret.Auth 435 if auth == nil { 436 return fmt.Errorf("renewal successful but not auth information returned") 437 } else if auth.LeaseDuration == 0 { 438 return fmt.Errorf("renewal successful but no lease duration returned") 439 } 440 441 v.lastRenewed = time.Now() 442 v.logger.Printf("[DEBUG] vault: succesfully renewed server token") 443 return nil 444 } 445 446 // getWrappingFn returns an appropriate wrapping function for Nomad Servers 447 func (v *vaultClient) getWrappingFn() func(operation, path string) string { 448 createPath := "auth/token/create" 449 if !v.tokenData.Root { 450 createPath = fmt.Sprintf("auth/token/create/%s", v.tokenData.Role) 451 } 452 453 return func(operation, path string) string { 454 // Only wrap the token create operation 455 if operation != "POST" || path != createPath { 456 return "" 457 } 458 459 return vaultTokenCreateTTL 460 } 461 } 462 463 // parseSelfToken looks up the Vault token in Vault and parses its data storing 464 // it in the client. If the token is not valid for Nomads purposes an error is 465 // returned. 466 func (v *vaultClient) parseSelfToken() error { 467 // Get the initial lease duration 468 auth := v.client.Auth().Token() 469 self, err := auth.LookupSelf() 470 if err != nil { 471 return fmt.Errorf("failed to lookup Vault periodic token: %v", err) 472 } 473 474 // Read and parse the fields 475 var data tokenData 476 if err := mapstructure.WeakDecode(self.Data, &data); err != nil { 477 return fmt.Errorf("failed to parse Vault token's data block: %v", err) 478 } 479 480 root := false 481 for _, p := range data.Policies { 482 if p == "root" { 483 root = true 484 break 485 } 486 } 487 488 if !data.Renewable && !root { 489 return fmt.Errorf("Vault token is not renewable or root") 490 } 491 492 if data.CreationTTL == 0 && !root { 493 return fmt.Errorf("invalid lease duration of zero") 494 } 495 496 if data.TTL == 0 && !root { 497 return fmt.Errorf("token TTL is zero") 498 } 499 500 if !root && data.Role == "" { 501 return fmt.Errorf("token role name must be set when not using a root token") 502 } 503 504 data.Root = root 505 v.tokenData = &data 506 return nil 507 } 508 509 // ConnectionEstablished returns whether a connection to Vault has been 510 // established. 511 func (v *vaultClient) ConnectionEstablished() bool { 512 return atomic.LoadInt32(&v.connEstablished) == 1 513 } 514 515 func (v *vaultClient) Enabled() bool { 516 v.l.Lock() 517 defer v.l.Unlock() 518 return v.config.Enabled 519 } 520 521 // 522 func (v *vaultClient) Active() bool { 523 return atomic.LoadInt32(&v.active) == 1 524 } 525 526 // CreateToken takes the allocation and task and returns an appropriate Vault 527 // token. The call is rate limited and may be canceled with the passed policy 528 func (v *vaultClient) CreateToken(ctx context.Context, a *structs.Allocation, task string) (*vapi.Secret, error) { 529 if !v.Enabled() { 530 return nil, fmt.Errorf("Vault integration disabled") 531 } 532 533 if !v.Active() { 534 return nil, fmt.Errorf("Vault client not active") 535 } 536 537 // Check if we have established a connection with Vault 538 if !v.ConnectionEstablished() { 539 return nil, fmt.Errorf("Connection to Vault has not been established. Retry") 540 } 541 542 // Retrieve the Vault block for the task 543 policies := a.Job.VaultPolicies() 544 if policies == nil { 545 return nil, fmt.Errorf("Job doesn't require Vault policies") 546 } 547 tg, ok := policies[a.TaskGroup] 548 if !ok { 549 return nil, fmt.Errorf("Task group does not require Vault policies") 550 } 551 taskVault, ok := tg[task] 552 if !ok { 553 return nil, fmt.Errorf("Task does not require Vault policies") 554 } 555 556 // Build the creation request 557 req := &vapi.TokenCreateRequest{ 558 Policies: taskVault.Policies, 559 Metadata: map[string]string{ 560 "AllocationID": a.ID, 561 "Task": task, 562 "NodeID": a.NodeID, 563 }, 564 TTL: v.childTTL, 565 DisplayName: fmt.Sprintf("%s: %s", a.ID, task), 566 } 567 568 // Ensure we are under our rate limit 569 if err := v.limiter.Wait(ctx); err != nil { 570 return nil, err 571 } 572 573 // Make the request and switch depending on whether we are using a root 574 // token or a role based token 575 var secret *vapi.Secret 576 var err error 577 if v.tokenData.Root { 578 req.Period = v.childTTL 579 secret, err = v.auth.Create(req) 580 } else { 581 // Make the token using the role 582 secret, err = v.auth.CreateWithRole(req, v.tokenData.Role) 583 } 584 585 return secret, err 586 } 587 588 // LookupToken takes a Vault token and does a lookup against Vault. The call is 589 // rate limited and may be canceled with passed context. 590 func (v *vaultClient) LookupToken(ctx context.Context, token string) (*vapi.Secret, error) { 591 if !v.Enabled() { 592 return nil, fmt.Errorf("Vault integration disabled") 593 } 594 595 if !v.Active() { 596 return nil, fmt.Errorf("Vault client not active") 597 } 598 599 // Check if we have established a connection with Vault 600 if !v.ConnectionEstablished() { 601 return nil, fmt.Errorf("Connection to Vault has not been established. Retry") 602 } 603 604 // Ensure we are under our rate limit 605 if err := v.limiter.Wait(ctx); err != nil { 606 return nil, err 607 } 608 609 // Lookup the token 610 return v.auth.Lookup(token) 611 } 612 613 // PoliciesFrom parses the set of policies returned by a token lookup. 614 func PoliciesFrom(s *vapi.Secret) ([]string, error) { 615 if s == nil { 616 return nil, fmt.Errorf("cannot parse nil Vault secret") 617 } 618 var data tokenData 619 if err := mapstructure.WeakDecode(s.Data, &data); err != nil { 620 return nil, fmt.Errorf("failed to parse Vault token's data block: %v", err) 621 } 622 623 return data.Policies, nil 624 } 625 626 // RevokeTokens revokes the passed set of accessors. If committed is set, the 627 // purge function passed to the client is called. If there is an error purging 628 // either because of Vault failures or because of the purge function, the 629 // revocation is retried until the tokens TTL. 630 func (v *vaultClient) RevokeTokens(ctx context.Context, accessors []*structs.VaultAccessor, committed bool) error { 631 if !v.Enabled() { 632 return nil 633 } 634 635 if !v.Active() { 636 return fmt.Errorf("Vault client not active") 637 } 638 639 // Check if we have established a connection with Vault. If not just add it 640 // to the queue 641 if !v.ConnectionEstablished() { 642 // Only bother tracking it for later revocation if the accessor was 643 // committed 644 if committed { 645 v.storeForRevocation(accessors) 646 } 647 648 return nil 649 } 650 651 // Attempt to revoke immediately and if it fails, add it to the revoke queue 652 err := v.parallelRevoke(ctx, accessors) 653 if !committed { 654 // If it is uncommitted, it is a best effort revoke as it will shortly 655 // TTL within the cubbyhole and has not been leaked to any outside 656 // system 657 return nil 658 } 659 660 if err != nil { 661 v.logger.Printf("[WARN] vault: failed to revoke tokens. Will reattempt til TTL: %v", err) 662 v.storeForRevocation(accessors) 663 return nil 664 } 665 666 if err := v.purgeFn(accessors); err != nil { 667 v.logger.Printf("[ERR] vault: failed to purge Vault accessors: %v", err) 668 v.storeForRevocation(accessors) 669 return nil 670 } 671 672 return nil 673 } 674 675 // storeForRevocation stores the passed set of accessors for revocation. It 676 // captrues their effective TTL by storing their create TTL plus the current 677 // time. 678 func (v *vaultClient) storeForRevocation(accessors []*structs.VaultAccessor) { 679 v.revLock.Lock() 680 now := time.Now() 681 for _, a := range accessors { 682 v.revoking[a] = now.Add(time.Duration(a.CreationTTL) * time.Second) 683 } 684 v.revLock.Unlock() 685 } 686 687 // parallelRevoke revokes the passed VaultAccessors in parallel. 688 func (v *vaultClient) parallelRevoke(ctx context.Context, accessors []*structs.VaultAccessor) error { 689 if !v.Enabled() { 690 return fmt.Errorf("Vault integration disabled") 691 } 692 693 if !v.Active() { 694 return fmt.Errorf("Vault client not active") 695 } 696 697 // Check if we have established a connection with Vault 698 if !v.ConnectionEstablished() { 699 return fmt.Errorf("Connection to Vault has not been established. Retry") 700 } 701 702 g, pCtx := errgroup.WithContext(ctx) 703 704 // Cap the handlers 705 handlers := len(accessors) 706 if handlers > maxParallelRevokes { 707 handlers = maxParallelRevokes 708 } 709 710 // Create the Vault Tokens 711 input := make(chan *structs.VaultAccessor, handlers) 712 for i := 0; i < handlers; i++ { 713 g.Go(func() error { 714 for { 715 select { 716 case va, ok := <-input: 717 if !ok { 718 return nil 719 } 720 721 if err := v.auth.RevokeAccessor(va.Accessor); err != nil { 722 return fmt.Errorf("failed to revoke token (alloc: %q, node: %q, task: %q)", va.AllocID, va.NodeID, va.Task) 723 } 724 case <-pCtx.Done(): 725 return nil 726 } 727 } 728 }) 729 } 730 731 // Send the input 732 go func() { 733 defer close(input) 734 for _, va := range accessors { 735 select { 736 case <-pCtx.Done(): 737 return 738 case input <- va: 739 } 740 } 741 742 }() 743 744 // Wait for everything to complete 745 return g.Wait() 746 } 747 748 // revokeDaemon should be called in a goroutine and is used to periodically 749 // revoke Vault accessors that failed the original revocation 750 func (v *vaultClient) revokeDaemon() { 751 ticker := time.NewTicker(vaultRevocationIntv) 752 defer ticker.Stop() 753 754 for { 755 select { 756 case <-v.tomb.Dying(): 757 return 758 case now := <-ticker.C: 759 if !v.ConnectionEstablished() { 760 continue 761 } 762 763 v.revLock.Lock() 764 765 // Fast path 766 if len(v.revoking) == 0 { 767 v.revLock.Unlock() 768 continue 769 } 770 771 // Build the list of allocations that need to revoked while pruning any TTL'd checks 772 revoking := make([]*structs.VaultAccessor, 0, len(v.revoking)) 773 for va, ttl := range v.revoking { 774 if now.After(ttl) { 775 delete(v.revoking, va) 776 } else { 777 revoking = append(revoking, va) 778 } 779 } 780 781 if err := v.parallelRevoke(context.Background(), revoking); err != nil { 782 v.logger.Printf("[WARN] vault: background token revocation errored: %v", err) 783 v.revLock.Unlock() 784 continue 785 } 786 787 // Unlock before a potentially expensive operation 788 v.revLock.Unlock() 789 790 // Call the passed in token revocation function 791 if err := v.purgeFn(revoking); err != nil { 792 // Can continue since revocation is idempotent 793 v.logger.Printf("[ERR] vault: token revocation errored: %v", err) 794 continue 795 } 796 797 // Can delete from the tracked list now that we have purged 798 v.revLock.Lock() 799 for _, va := range revoking { 800 delete(v.revoking, va) 801 } 802 v.revLock.Unlock() 803 } 804 } 805 } 806 807 // purgeVaultAccessors creates a Raft transaction to remove the passed Vault 808 // Accessors 809 func (s *Server) purgeVaultAccessors(accessors []*structs.VaultAccessor) error { 810 // Commit this update via Raft 811 req := structs.VaultAccessorsRequest{Accessors: accessors} 812 _, _, err := s.raftApply(structs.VaultAccessorDegisterRequestType, req) 813 return err 814 } 815 816 // wrapNilError is a helper that returns a wrapped function that returns a nil 817 // error 818 func wrapNilError(f func()) func() error { 819 return func() error { 820 f() 821 return nil 822 } 823 } 824 825 // setLimit is used to update the rate limit 826 func (v *vaultClient) setLimit(l rate.Limit) { 827 v.l.Lock() 828 defer v.l.Unlock() 829 v.limiter = rate.NewLimiter(l, int(l)) 830 }