github.com/adityamillind98/nomad@v0.11.8/nomad/vault_test.go (about) 1 package nomad 2 3 import ( 4 "context" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "math/rand" 9 "reflect" 10 "strings" 11 "sync/atomic" 12 "testing" 13 "time" 14 15 "github.com/stretchr/testify/assert" 16 "github.com/stretchr/testify/require" 17 18 "golang.org/x/time/rate" 19 20 "github.com/hashicorp/nomad/helper" 21 "github.com/hashicorp/nomad/helper/testlog" 22 "github.com/hashicorp/nomad/helper/uuid" 23 "github.com/hashicorp/nomad/nomad/mock" 24 "github.com/hashicorp/nomad/nomad/structs" 25 "github.com/hashicorp/nomad/nomad/structs/config" 26 "github.com/hashicorp/nomad/testutil" 27 vapi "github.com/hashicorp/vault/api" 28 vaultconsts "github.com/hashicorp/vault/sdk/helper/consts" 29 ) 30 31 const ( 32 // nomadRoleManagementPolicy is a policy that allows nomad to manage tokens 33 nomadRoleManagementPolicy = ` 34 path "auth/token/renew-self" { 35 capabilities = ["update"] 36 } 37 38 path "auth/token/lookup" { 39 capabilities = ["update"] 40 } 41 42 path "auth/token/roles/test" { 43 capabilities = ["read"] 44 } 45 46 path "auth/token/revoke-accessor" { 47 capabilities = ["update"] 48 } 49 ` 50 51 // tokenLookupPolicy allows a token to be looked up 52 tokenLookupPolicy = ` 53 path "auth/token/lookup" { 54 capabilities = ["update"] 55 } 56 ` 57 58 // nomadRoleCreatePolicy gives the ability to create the role and derive tokens 59 // from the test role 60 nomadRoleCreatePolicy = ` 61 path "auth/token/create/test" { 62 capabilities = ["create", "update"] 63 } 64 ` 65 66 // secretPolicy gives access to the secret mount 67 secretPolicy = ` 68 path "secret/*" { 69 capabilities = ["create", "read", "update", "delete", "list"] 70 } 71 ` 72 ) 73 74 // defaultTestVaultWhitelistRoleAndToken creates a test Vault role and returns a token 75 // created in that role 76 func defaultTestVaultWhitelistRoleAndToken(v *testutil.TestVault, t *testing.T, rolePeriod int) string { 77 vaultPolicies := map[string]string{ 78 "nomad-role-create": nomadRoleCreatePolicy, 79 "nomad-role-management": nomadRoleManagementPolicy, 80 } 81 d := make(map[string]interface{}, 2) 82 d["allowed_policies"] = "nomad-role-create,nomad-role-management" 83 d["period"] = rolePeriod 84 return testVaultRoleAndToken(v, t, vaultPolicies, d, 85 []string{"nomad-role-create", "nomad-role-management"}) 86 } 87 88 // defaultTestVaultBlacklistRoleAndToken creates a test Vault role using 89 // disallowed_policies and returns a token created in that role 90 func defaultTestVaultBlacklistRoleAndToken(v *testutil.TestVault, t *testing.T, rolePeriod int) string { 91 vaultPolicies := map[string]string{ 92 "nomad-role-create": nomadRoleCreatePolicy, 93 "nomad-role-management": nomadRoleManagementPolicy, 94 "secrets": secretPolicy, 95 } 96 97 // Create the role 98 d := make(map[string]interface{}, 2) 99 d["disallowed_policies"] = "nomad-role-create" 100 d["period"] = rolePeriod 101 testVaultRoleAndToken(v, t, vaultPolicies, d, []string{"default"}) 102 103 // Create a token that can use the role 104 a := v.Client.Auth().Token() 105 req := &vapi.TokenCreateRequest{ 106 Policies: []string{"nomad-role-create", "nomad-role-management"}, 107 } 108 s, err := a.Create(req) 109 if err != nil { 110 t.Fatalf("failed to create child token: %v", err) 111 } 112 113 if s == nil || s.Auth == nil { 114 t.Fatalf("bad secret response: %+v", s) 115 } 116 117 return s.Auth.ClientToken 118 } 119 120 // testVaultRoleAndToken writes the vaultPolicies to vault and then creates a 121 // test role with the passed data. After that it derives a token from the role 122 // with the tokenPolicies 123 func testVaultRoleAndToken(v *testutil.TestVault, t *testing.T, vaultPolicies map[string]string, 124 data map[string]interface{}, tokenPolicies []string) string { 125 // Write the policies 126 sys := v.Client.Sys() 127 for p, data := range vaultPolicies { 128 if err := sys.PutPolicy(p, data); err != nil { 129 t.Fatalf("failed to create %q policy: %v", p, err) 130 } 131 } 132 133 // Build a role 134 l := v.Client.Logical() 135 l.Write("auth/token/roles/test", data) 136 137 // Create a new token with the role 138 a := v.Client.Auth().Token() 139 req := vapi.TokenCreateRequest{ 140 Policies: tokenPolicies, 141 } 142 s, err := a.CreateWithRole(&req, "test") 143 if err != nil { 144 t.Fatalf("failed to create child token: %v", err) 145 } 146 147 // Get the client token 148 if s == nil || s.Auth == nil { 149 t.Fatalf("bad secret response: %+v", s) 150 } 151 152 return s.Auth.ClientToken 153 } 154 155 func TestVaultClient_BadConfig(t *testing.T) { 156 t.Parallel() 157 conf := &config.VaultConfig{} 158 logger := testlog.HCLogger(t) 159 160 // Should be no error since Vault is not enabled 161 _, err := NewVaultClient(nil, logger, nil) 162 if err == nil || !strings.Contains(err.Error(), "valid") { 163 t.Fatalf("expected config error: %v", err) 164 } 165 166 tr := true 167 conf.Enabled = &tr 168 _, err = NewVaultClient(conf, logger, nil) 169 if err == nil || !strings.Contains(err.Error(), "token must be set") { 170 t.Fatalf("Expected token unset error: %v", err) 171 } 172 173 conf.Token = "123" 174 _, err = NewVaultClient(conf, logger, nil) 175 if err == nil || !strings.Contains(err.Error(), "address must be set") { 176 t.Fatalf("Expected address unset error: %v", err) 177 } 178 } 179 180 // TestVaultClient_WithNamespaceSupport tests that the Vault namespace config, if present, will result in the 181 // namespace header being set on the created Vault client. 182 func TestVaultClient_WithNamespaceSupport(t *testing.T) { 183 t.Parallel() 184 require := require.New(t) 185 tr := true 186 testNs := "test-namespace" 187 conf := &config.VaultConfig{ 188 Addr: "https://vault.service.consul:8200", 189 Enabled: &tr, 190 Token: "testvaulttoken", 191 Namespace: testNs, 192 } 193 logger := testlog.HCLogger(t) 194 195 // Should be no error since Vault is not enabled 196 c, err := NewVaultClient(conf, logger, nil) 197 if err != nil { 198 t.Fatalf("failed to build vault client: %v", err) 199 } 200 201 require.Equal(testNs, c.client.Headers().Get(vaultconsts.NamespaceHeaderName)) 202 require.Equal("", c.clientSys.Headers().Get(vaultconsts.NamespaceHeaderName)) 203 require.NotEqual(c.clientSys, c.client) 204 } 205 206 // TestVaultClient_WithoutNamespaceSupport tests that the Vault namespace config, if present, will result in the 207 // namespace header being set on the created Vault client. 208 func TestVaultClient_WithoutNamespaceSupport(t *testing.T) { 209 t.Parallel() 210 require := require.New(t) 211 tr := true 212 conf := &config.VaultConfig{ 213 Addr: "https://vault.service.consul:8200", 214 Enabled: &tr, 215 Token: "testvaulttoken", 216 Namespace: "", 217 } 218 logger := testlog.HCLogger(t) 219 220 // Should be no error since Vault is not enabled 221 c, err := NewVaultClient(conf, logger, nil) 222 if err != nil { 223 t.Fatalf("failed to build vault client: %v", err) 224 } 225 226 require.Equal("", c.client.Headers().Get(vaultconsts.NamespaceHeaderName)) 227 require.Equal("", c.clientSys.Headers().Get(vaultconsts.NamespaceHeaderName)) 228 require.Equal(c.clientSys, c.client) 229 } 230 231 // started separately. 232 // Test that the Vault Client can establish a connection even if it is started 233 // before Vault is available. 234 func TestVaultClient_EstablishConnection(t *testing.T) { 235 t.Parallel() 236 for i := 10; i >= 0; i-- { 237 v := testutil.NewTestVaultDelayed(t) 238 logger := testlog.HCLogger(t) 239 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 240 client, err := NewVaultClient(v.Config, logger, nil) 241 if err != nil { 242 t.Fatalf("failed to build vault client: %v", err) 243 } 244 245 // Sleep a little while and check that no connection has been established. 246 time.Sleep(100 * time.Duration(testutil.TestMultiplier()) * time.Millisecond) 247 if established, _ := client.ConnectionEstablished(); established { 248 t.Fatalf("ConnectionEstablished() returned true before Vault server started") 249 } 250 251 // Start Vault 252 if err := v.Start(); err != nil { 253 v.Stop() 254 client.Stop() 255 256 if i == 0 { 257 t.Fatalf("Failed to start vault: %v", err) 258 } 259 260 wait := time.Duration(rand.Int31n(2000)) * time.Millisecond 261 time.Sleep(wait) 262 continue 263 } 264 265 var waitErr error 266 testutil.WaitForResult(func() (bool, error) { 267 return client.ConnectionEstablished() 268 }, func(err error) { 269 waitErr = err 270 }) 271 272 v.Stop() 273 client.Stop() 274 if waitErr != nil { 275 if i == 0 { 276 t.Fatalf("Failed to start vault: %v", err) 277 } 278 279 wait := time.Duration(rand.Int31n(2000)) * time.Millisecond 280 time.Sleep(wait) 281 continue 282 } 283 284 break 285 } 286 } 287 288 func TestVaultClient_ValidateRole(t *testing.T) { 289 t.Parallel() 290 v := testutil.NewTestVault(t) 291 defer v.Stop() 292 293 // Set the configs token in a new test role 294 vaultPolicies := map[string]string{ 295 "nomad-role-create": nomadRoleCreatePolicy, 296 "nomad-role-management": nomadRoleManagementPolicy, 297 } 298 data := map[string]interface{}{ 299 "allowed_policies": "default,root", 300 "orphan": true, 301 "renewable": true, 302 "token_explicit_max_ttl": 10, 303 } 304 v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, nil) 305 306 logger := testlog.HCLogger(t) 307 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 308 client, err := NewVaultClient(v.Config, logger, nil) 309 require.NoError(t, err) 310 311 defer client.Stop() 312 313 // Wait for an error 314 var conn bool 315 var connErr error 316 testutil.WaitForResult(func() (bool, error) { 317 conn, connErr = client.ConnectionEstablished() 318 if !conn { 319 return false, fmt.Errorf("Should connect") 320 } 321 322 if connErr == nil { 323 return false, fmt.Errorf("expect an error") 324 } 325 326 return true, nil 327 }, func(err error) { 328 require.NoError(t, err) 329 }) 330 331 require.Contains(t, connErr.Error(), "explicit max ttl") 332 require.Contains(t, connErr.Error(), "non-zero period") 333 } 334 335 // TestVaultClient_ValidateRole_Success asserts that a valid token role 336 // gets marked as valid 337 func TestVaultClient_ValidateRole_Success(t *testing.T) { 338 t.Parallel() 339 v := testutil.NewTestVault(t) 340 defer v.Stop() 341 342 // Set the configs token in a new test role 343 vaultPolicies := map[string]string{ 344 "nomad-role-create": nomadRoleCreatePolicy, 345 "nomad-role-management": nomadRoleManagementPolicy, 346 } 347 data := map[string]interface{}{ 348 "allowed_policies": "default,root", 349 "orphan": true, 350 "renewable": true, 351 "token_period": 1000, 352 } 353 v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, nil) 354 355 logger := testlog.HCLogger(t) 356 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 357 client, err := NewVaultClient(v.Config, logger, nil) 358 require.NoError(t, err) 359 360 defer client.Stop() 361 362 // Wait for an error 363 var conn bool 364 var connErr error 365 testutil.WaitForResult(func() (bool, error) { 366 conn, connErr = client.ConnectionEstablished() 367 if !conn { 368 return false, fmt.Errorf("Should connect") 369 } 370 371 if connErr != nil { 372 return false, connErr 373 } 374 375 return true, nil 376 }, func(err error) { 377 require.NoError(t, err) 378 }) 379 } 380 381 // TestVaultClient_ValidateRole_Deprecated_Success asserts that a valid token 382 // role gets marked as valid, even if it uses deprecated field, period 383 func TestVaultClient_ValidateRole_Deprecated_Success(t *testing.T) { 384 t.Parallel() 385 v := testutil.NewTestVault(t) 386 defer v.Stop() 387 388 // Set the configs token in a new test role 389 vaultPolicies := map[string]string{ 390 "nomad-role-create": nomadRoleCreatePolicy, 391 "nomad-role-management": nomadRoleManagementPolicy, 392 } 393 data := map[string]interface{}{ 394 "allowed_policies": "default,root", 395 "orphan": true, 396 "renewable": true, 397 "period": 1000, 398 } 399 v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, nil) 400 401 logger := testlog.HCLogger(t) 402 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 403 client, err := NewVaultClient(v.Config, logger, nil) 404 require.NoError(t, err) 405 406 defer client.Stop() 407 408 // Wait for an error 409 var conn bool 410 var connErr error 411 testutil.WaitForResult(func() (bool, error) { 412 conn, connErr = client.ConnectionEstablished() 413 if !conn { 414 return false, fmt.Errorf("Should connect") 415 } 416 417 if connErr != nil { 418 return false, connErr 419 } 420 421 return true, nil 422 }, func(err error) { 423 require.NoError(t, err) 424 }) 425 } 426 427 func TestVaultClient_ValidateRole_NonExistent(t *testing.T) { 428 t.Parallel() 429 v := testutil.NewTestVault(t) 430 defer v.Stop() 431 432 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 433 v.Config.Token = v.RootToken 434 logger := testlog.HCLogger(t) 435 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 436 v.Config.Role = "test-nonexistent" 437 client, err := NewVaultClient(v.Config, logger, nil) 438 if err != nil { 439 t.Fatalf("failed to build vault client: %v", err) 440 } 441 defer client.Stop() 442 443 // Wait for an error 444 var conn bool 445 var connErr error 446 testutil.WaitForResult(func() (bool, error) { 447 conn, connErr = client.ConnectionEstablished() 448 if !conn { 449 return false, fmt.Errorf("Should connect") 450 } 451 452 if connErr == nil { 453 return false, fmt.Errorf("expect an error") 454 } 455 456 return true, nil 457 }, func(err error) { 458 t.Fatalf("bad: %v", err) 459 }) 460 461 errStr := connErr.Error() 462 if !strings.Contains(errStr, "does not exist") { 463 t.Fatalf("Expect does not exist error") 464 } 465 } 466 467 func TestVaultClient_ValidateToken(t *testing.T) { 468 t.Parallel() 469 v := testutil.NewTestVault(t) 470 defer v.Stop() 471 472 // Set the configs token in a new test role 473 vaultPolicies := map[string]string{ 474 "nomad-role-create": nomadRoleCreatePolicy, 475 "token-lookup": tokenLookupPolicy, 476 } 477 data := map[string]interface{}{ 478 "allowed_policies": "token-lookup,nomad-role-create", 479 "period": 10, 480 } 481 v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, []string{"token-lookup", "nomad-role-create"}) 482 483 logger := testlog.HCLogger(t) 484 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 485 client, err := NewVaultClient(v.Config, logger, nil) 486 if err != nil { 487 t.Fatalf("failed to build vault client: %v", err) 488 } 489 defer client.Stop() 490 491 // Wait for an error 492 var conn bool 493 var connErr error 494 testutil.WaitForResult(func() (bool, error) { 495 conn, connErr = client.ConnectionEstablished() 496 if !conn { 497 return false, fmt.Errorf("Should connect") 498 } 499 500 if connErr == nil { 501 return false, fmt.Errorf("expect an error") 502 } 503 504 return true, nil 505 }, func(err error) { 506 t.Fatalf("bad: %v", err) 507 }) 508 509 errStr := connErr.Error() 510 if !strings.Contains(errStr, vaultTokenRevokePath) { 511 t.Fatalf("Expect revoke error") 512 } 513 if !strings.Contains(errStr, fmt.Sprintf(vaultRoleLookupPath, "test")) { 514 t.Fatalf("Expect explicit max ttl error") 515 } 516 if !strings.Contains(errStr, "token must have one of the following") { 517 t.Fatalf("Expect explicit max ttl error") 518 } 519 } 520 521 func TestVaultClient_SetActive(t *testing.T) { 522 t.Parallel() 523 v := testutil.NewTestVault(t) 524 defer v.Stop() 525 526 logger := testlog.HCLogger(t) 527 client, err := NewVaultClient(v.Config, logger, nil) 528 if err != nil { 529 t.Fatalf("failed to build vault client: %v", err) 530 } 531 defer client.Stop() 532 533 waitForConnection(client, t) 534 535 // Do a lookup and expect an error about not being active 536 _, err = client.LookupToken(context.Background(), "123") 537 if err == nil || !strings.Contains(err.Error(), "not active") { 538 t.Fatalf("Expected not-active error: %v", err) 539 } 540 541 client.SetActive(true) 542 543 // Do a lookup of ourselves 544 _, err = client.LookupToken(context.Background(), v.RootToken) 545 if err != nil { 546 t.Fatalf("Unexpected error: %v", err) 547 } 548 } 549 550 // Test that we can update the config and things keep working 551 func TestVaultClient_SetConfig(t *testing.T) { 552 t.Parallel() 553 v := testutil.NewTestVault(t) 554 defer v.Stop() 555 556 v2 := testutil.NewTestVault(t) 557 defer v2.Stop() 558 559 // Set the configs token in a new test role 560 v2.Config.Token = defaultTestVaultWhitelistRoleAndToken(v2, t, 20) 561 562 logger := testlog.HCLogger(t) 563 client, err := NewVaultClient(v.Config, logger, nil) 564 if err != nil { 565 t.Fatalf("failed to build vault client: %v", err) 566 } 567 defer client.Stop() 568 569 waitForConnection(client, t) 570 571 if client.tokenData == nil || len(client.tokenData.Policies) != 1 { 572 t.Fatalf("unexpected token: %v", client.tokenData) 573 } 574 575 // Update the config 576 if err := client.SetConfig(v2.Config); err != nil { 577 t.Fatalf("SetConfig failed: %v", err) 578 } 579 580 waitForConnection(client, t) 581 582 if client.tokenData == nil || len(client.tokenData.Policies) != 3 { 583 t.Fatalf("unexpected token: %v", client.tokenData) 584 } 585 586 // Test that when SetConfig is called with the same configuration, it is a 587 // no-op 588 failCh := make(chan struct{}, 1) 589 go func() { 590 tomb := client.tomb 591 select { 592 case <-tomb.Dying(): 593 close(failCh) 594 case <-time.After(1 * time.Second): 595 return 596 } 597 }() 598 599 // Update the config 600 if err := client.SetConfig(v2.Config); err != nil { 601 t.Fatalf("SetConfig failed: %v", err) 602 } 603 604 select { 605 case <-failCh: 606 t.Fatalf("Tomb shouldn't have exited") 607 case <-time.After(1 * time.Second): 608 return 609 } 610 } 611 612 // TestVaultClient_SetConfig_Deadlock asserts that calling SetConfig 613 // concurrently with establishConnection does not deadlock. 614 func TestVaultClient_SetConfig_Deadlock(t *testing.T) { 615 t.Parallel() 616 v := testutil.NewTestVault(t) 617 defer v.Stop() 618 619 v2 := testutil.NewTestVault(t) 620 defer v2.Stop() 621 622 // Set the configs token in a new test role 623 v2.Config.Token = defaultTestVaultWhitelistRoleAndToken(v2, t, 20) 624 625 logger := testlog.HCLogger(t) 626 client, err := NewVaultClient(v.Config, logger, nil) 627 if err != nil { 628 t.Fatalf("failed to build vault client: %v", err) 629 } 630 defer client.Stop() 631 632 for i := 0; i < 100; i++ { 633 // Alternate configs to cause updates 634 conf := v.Config 635 if i%2 == 0 { 636 conf = v2.Config 637 } 638 if err := client.SetConfig(conf); err != nil { 639 t.Fatalf("SetConfig failed: %v", err) 640 } 641 } 642 } 643 644 // Test that we can disable vault 645 func TestVaultClient_SetConfig_Disable(t *testing.T) { 646 t.Parallel() 647 v := testutil.NewTestVault(t) 648 defer v.Stop() 649 650 logger := testlog.HCLogger(t) 651 client, err := NewVaultClient(v.Config, logger, nil) 652 if err != nil { 653 t.Fatalf("failed to build vault client: %v", err) 654 } 655 defer client.Stop() 656 657 waitForConnection(client, t) 658 659 if client.tokenData == nil || len(client.tokenData.Policies) != 1 { 660 t.Fatalf("unexpected token: %v", client.tokenData) 661 } 662 663 // Disable vault 664 f := false 665 config := config.VaultConfig{ 666 Enabled: &f, 667 } 668 669 // Update the config 670 if err := client.SetConfig(&config); err != nil { 671 t.Fatalf("SetConfig failed: %v", err) 672 } 673 674 if client.Enabled() || client.Running() { 675 t.Fatalf("SetConfig should have stopped client") 676 } 677 } 678 679 func TestVaultClient_RenewalLoop(t *testing.T) { 680 t.Parallel() 681 v := testutil.NewTestVault(t) 682 defer v.Stop() 683 684 // Set the configs token in a new test role 685 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 686 687 // Start the client 688 logger := testlog.HCLogger(t) 689 client, err := NewVaultClient(v.Config, logger, nil) 690 if err != nil { 691 t.Fatalf("failed to build vault client: %v", err) 692 } 693 defer client.Stop() 694 695 // Sleep 8 seconds and ensure we have a non-zero TTL 696 time.Sleep(8 * time.Second) 697 698 // Get the current TTL 699 a := v.Client.Auth().Token() 700 s2, err := a.Lookup(v.Config.Token) 701 if err != nil { 702 t.Fatalf("failed to lookup token: %v", err) 703 } 704 705 ttl := parseTTLFromLookup(s2, t) 706 if ttl == 0 { 707 t.Fatalf("token renewal failed; ttl %v", ttl) 708 } 709 710 if client.currentExpiration.Before(time.Now()) { 711 t.Fatalf("found current expiration to be in past %s", time.Until(client.currentExpiration)) 712 } 713 } 714 715 func TestVaultClientRenewUpdatesExpiration(t *testing.T) { 716 t.Parallel() 717 v := testutil.NewTestVault(t) 718 defer v.Stop() 719 720 // Set the configs token in a new test role 721 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 722 723 // Start the client 724 logger := testlog.HCLogger(t) 725 client, err := NewVaultClient(v.Config, logger, nil) 726 if err != nil { 727 t.Fatalf("failed to build vault client: %v", err) 728 } 729 defer client.Stop() 730 731 // Get the current TTL 732 a := v.Client.Auth().Token() 733 s2, err := a.Lookup(v.Config.Token) 734 if err != nil { 735 t.Fatalf("failed to lookup token: %v", err) 736 } 737 exp0 := time.Now().Add(time.Duration(parseTTLFromLookup(s2, t)) * time.Second) 738 739 time.Sleep(1 * time.Second) 740 741 _, err = client.renew() 742 require.NoError(t, err) 743 exp1 := client.currentExpiration 744 require.True(t, exp0.Before(exp1)) 745 746 time.Sleep(1 * time.Second) 747 748 _, err = client.renew() 749 require.NoError(t, err) 750 exp2 := client.currentExpiration 751 require.True(t, exp1.Before(exp2)) 752 } 753 754 func TestVaultClient_StopsAfterPermissionError(t *testing.T) { 755 t.Parallel() 756 v := testutil.NewTestVault(t) 757 defer v.Stop() 758 759 // Set the configs token in a new test role 760 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 2) 761 762 // Start the client 763 logger := testlog.HCLogger(t) 764 client, err := NewVaultClient(v.Config, logger, nil) 765 if err != nil { 766 t.Fatalf("failed to build vault client: %v", err) 767 } 768 defer client.Stop() 769 770 time.Sleep(500 * time.Millisecond) 771 772 assert.True(t, client.isRenewLoopActive()) 773 774 // Get the current TTL 775 a := v.Client.Auth().Token() 776 assert.NoError(t, a.RevokeSelf("")) 777 778 testutil.WaitForResult(func() (bool, error) { 779 if !client.isRenewLoopActive() { 780 return true, nil 781 } else { 782 return false, errors.New("renew loop should terminate after token is revoked") 783 } 784 }, func(err error) { 785 t.Fatalf("err: %v", err) 786 }) 787 } 788 func TestVaultClient_LoopsUntilCannotRenew(t *testing.T) { 789 t.Parallel() 790 v := testutil.NewTestVault(t) 791 defer v.Stop() 792 793 // Set the configs token in a new test role 794 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 795 796 // Start the client 797 logger := testlog.HCLogger(t) 798 client, err := NewVaultClient(v.Config, logger, nil) 799 if err != nil { 800 t.Fatalf("failed to build vault client: %v", err) 801 } 802 defer client.Stop() 803 804 // Sleep 8 seconds and ensure we have a non-zero TTL 805 time.Sleep(8 * time.Second) 806 807 // Get the current TTL 808 a := v.Client.Auth().Token() 809 s2, err := a.Lookup(v.Config.Token) 810 if err != nil { 811 t.Fatalf("failed to lookup token: %v", err) 812 } 813 814 ttl := parseTTLFromLookup(s2, t) 815 if ttl == 0 { 816 t.Fatalf("token renewal failed; ttl %v", ttl) 817 } 818 819 if client.currentExpiration.Before(time.Now()) { 820 t.Fatalf("found current expiration to be in past %s", time.Until(client.currentExpiration)) 821 } 822 } 823 824 func parseTTLFromLookup(s *vapi.Secret, t *testing.T) int64 { 825 if s == nil { 826 t.Fatalf("nil secret") 827 } else if s.Data == nil { 828 t.Fatalf("nil data block in secret") 829 } 830 831 ttlRaw, ok := s.Data["ttl"] 832 if !ok { 833 t.Fatalf("no ttl") 834 } 835 836 ttlNumber, ok := ttlRaw.(json.Number) 837 if !ok { 838 t.Fatalf("failed to convert ttl %q to json Number", ttlRaw) 839 } 840 841 ttl, err := ttlNumber.Int64() 842 if err != nil { 843 t.Fatalf("Failed to get ttl from json.Number: %v", err) 844 } 845 846 return ttl 847 } 848 849 func TestVaultClient_LookupToken_Invalid(t *testing.T) { 850 t.Parallel() 851 tr := true 852 conf := &config.VaultConfig{ 853 Enabled: &tr, 854 Addr: "http://foobar:12345", 855 Token: uuid.Generate(), 856 } 857 858 // Enable vault but use a bad address so it never establishes a conn 859 logger := testlog.HCLogger(t) 860 client, err := NewVaultClient(conf, logger, nil) 861 if err != nil { 862 t.Fatalf("failed to build vault client: %v", err) 863 } 864 client.SetActive(true) 865 defer client.Stop() 866 867 _, err = client.LookupToken(context.Background(), "foo") 868 if err == nil || !strings.Contains(err.Error(), "established") { 869 t.Fatalf("Expected error because connection to Vault hasn't been made: %v", err) 870 } 871 } 872 873 func TestVaultClient_LookupToken_Root(t *testing.T) { 874 t.Parallel() 875 v := testutil.NewTestVault(t) 876 defer v.Stop() 877 878 logger := testlog.HCLogger(t) 879 client, err := NewVaultClient(v.Config, logger, nil) 880 if err != nil { 881 t.Fatalf("failed to build vault client: %v", err) 882 } 883 client.SetActive(true) 884 defer client.Stop() 885 886 waitForConnection(client, t) 887 888 // Lookup ourselves 889 s, err := client.LookupToken(context.Background(), v.Config.Token) 890 if err != nil { 891 t.Fatalf("self lookup failed: %v", err) 892 } 893 894 policies, err := PoliciesFrom(s) 895 if err != nil { 896 t.Fatalf("failed to parse policies: %v", err) 897 } 898 899 expected := []string{"root"} 900 if !reflect.DeepEqual(policies, expected) { 901 t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) 902 } 903 904 // Create a token with a different set of policies 905 expected = []string{"default"} 906 req := vapi.TokenCreateRequest{ 907 Policies: expected, 908 } 909 s, err = v.Client.Auth().Token().Create(&req) 910 if err != nil { 911 t.Fatalf("failed to create child token: %v", err) 912 } 913 914 // Get the client token 915 if s == nil || s.Auth == nil { 916 t.Fatalf("bad secret response: %+v", s) 917 } 918 919 // Lookup new child 920 s, err = client.LookupToken(context.Background(), s.Auth.ClientToken) 921 if err != nil { 922 t.Fatalf("self lookup failed: %v", err) 923 } 924 925 policies, err = PoliciesFrom(s) 926 if err != nil { 927 t.Fatalf("failed to parse policies: %v", err) 928 } 929 930 if !reflect.DeepEqual(policies, expected) { 931 t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) 932 } 933 } 934 935 func TestVaultClient_LookupToken_Role(t *testing.T) { 936 t.Parallel() 937 v := testutil.NewTestVault(t) 938 defer v.Stop() 939 940 // Set the configs token in a new test role 941 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 942 943 logger := testlog.HCLogger(t) 944 client, err := NewVaultClient(v.Config, logger, nil) 945 if err != nil { 946 t.Fatalf("failed to build vault client: %v", err) 947 } 948 client.SetActive(true) 949 defer client.Stop() 950 951 waitForConnection(client, t) 952 953 // Lookup ourselves 954 s, err := client.LookupToken(context.Background(), v.Config.Token) 955 if err != nil { 956 t.Fatalf("self lookup failed: %v", err) 957 } 958 959 policies, err := PoliciesFrom(s) 960 if err != nil { 961 t.Fatalf("failed to parse policies: %v", err) 962 } 963 964 expected := []string{"default", "nomad-role-create", "nomad-role-management"} 965 if !reflect.DeepEqual(policies, expected) { 966 t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) 967 } 968 969 // Create a token with a different set of policies 970 expected = []string{"default"} 971 req := vapi.TokenCreateRequest{ 972 Policies: expected, 973 } 974 s, err = v.Client.Auth().Token().Create(&req) 975 if err != nil { 976 t.Fatalf("failed to create child token: %v", err) 977 } 978 979 // Get the client token 980 if s == nil || s.Auth == nil { 981 t.Fatalf("bad secret response: %+v", s) 982 } 983 984 // Lookup new child 985 s, err = client.LookupToken(context.Background(), s.Auth.ClientToken) 986 if err != nil { 987 t.Fatalf("self lookup failed: %v", err) 988 } 989 990 policies, err = PoliciesFrom(s) 991 if err != nil { 992 t.Fatalf("failed to parse policies: %v", err) 993 } 994 995 if !reflect.DeepEqual(policies, expected) { 996 t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) 997 } 998 } 999 1000 func TestVaultClient_LookupToken_RateLimit(t *testing.T) { 1001 t.Parallel() 1002 v := testutil.NewTestVault(t) 1003 defer v.Stop() 1004 1005 logger := testlog.HCLogger(t) 1006 client, err := NewVaultClient(v.Config, logger, nil) 1007 if err != nil { 1008 t.Fatalf("failed to build vault client: %v", err) 1009 } 1010 client.SetActive(true) 1011 defer client.Stop() 1012 1013 waitForConnection(client, t) 1014 1015 client.setLimit(rate.Limit(1.0)) 1016 1017 // Spin up many requests. These should block 1018 ctx, cancel := context.WithCancel(context.Background()) 1019 1020 cancels := 0 1021 numRequests := 20 1022 unblock := make(chan struct{}) 1023 for i := 0; i < numRequests; i++ { 1024 go func() { 1025 // Lookup ourselves 1026 _, err := client.LookupToken(ctx, v.Config.Token) 1027 if err != nil { 1028 if err == context.Canceled { 1029 cancels += 1 1030 return 1031 } 1032 t.Fatalf("self lookup failed: %v", err) 1033 return 1034 } 1035 1036 // Cancel the context 1037 close(unblock) 1038 }() 1039 } 1040 1041 select { 1042 case <-time.After(5 * time.Second): 1043 t.Fatalf("timeout") 1044 case <-unblock: 1045 cancel() 1046 } 1047 1048 desired := numRequests - 1 1049 testutil.WaitForResult(func() (bool, error) { 1050 if desired-cancels > 2 { 1051 return false, fmt.Errorf("Incorrect number of cancels; got %d; want %d", cancels, desired) 1052 } 1053 1054 return true, nil 1055 }, func(err error) { 1056 t.Fatal(err) 1057 }) 1058 } 1059 1060 func TestVaultClient_CreateToken_Root(t *testing.T) { 1061 t.Parallel() 1062 v := testutil.NewTestVault(t) 1063 defer v.Stop() 1064 1065 logger := testlog.HCLogger(t) 1066 client, err := NewVaultClient(v.Config, logger, nil) 1067 if err != nil { 1068 t.Fatalf("failed to build vault client: %v", err) 1069 } 1070 client.SetActive(true) 1071 defer client.Stop() 1072 1073 waitForConnection(client, t) 1074 1075 // Create an allocation that requires a Vault policy 1076 a := mock.Alloc() 1077 task := a.Job.TaskGroups[0].Tasks[0] 1078 task.Vault = &structs.Vault{Policies: []string{"default"}} 1079 1080 s, err := client.CreateToken(context.Background(), a, task.Name) 1081 if err != nil { 1082 t.Fatalf("CreateToken failed: %v", err) 1083 } 1084 1085 // Ensure that created secret is a wrapped token 1086 if s == nil || s.WrapInfo == nil { 1087 t.Fatalf("Bad secret: %#v", s) 1088 } 1089 1090 d, err := time.ParseDuration(vaultTokenCreateTTL) 1091 if err != nil { 1092 t.Fatalf("bad: %v", err) 1093 } 1094 1095 if s.WrapInfo.WrappedAccessor == "" { 1096 t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) 1097 } else if s.WrapInfo.Token == "" { 1098 t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) 1099 } else if s.WrapInfo.TTL != int(d.Seconds()) { 1100 t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) 1101 } 1102 } 1103 1104 func TestVaultClient_CreateToken_Whitelist_Role(t *testing.T) { 1105 t.Parallel() 1106 v := testutil.NewTestVault(t) 1107 defer v.Stop() 1108 1109 // Set the configs token in a new test role 1110 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 1111 1112 // Start the client 1113 logger := testlog.HCLogger(t) 1114 client, err := NewVaultClient(v.Config, logger, nil) 1115 if err != nil { 1116 t.Fatalf("failed to build vault client: %v", err) 1117 } 1118 client.SetActive(true) 1119 defer client.Stop() 1120 1121 waitForConnection(client, t) 1122 1123 // Create an allocation that requires a Vault policy 1124 a := mock.Alloc() 1125 task := a.Job.TaskGroups[0].Tasks[0] 1126 task.Vault = &structs.Vault{Policies: []string{"default"}} 1127 1128 s, err := client.CreateToken(context.Background(), a, task.Name) 1129 if err != nil { 1130 t.Fatalf("CreateToken failed: %v", err) 1131 } 1132 1133 // Ensure that created secret is a wrapped token 1134 if s == nil || s.WrapInfo == nil { 1135 t.Fatalf("Bad secret: %#v", s) 1136 } 1137 1138 d, err := time.ParseDuration(vaultTokenCreateTTL) 1139 if err != nil { 1140 t.Fatalf("bad: %v", err) 1141 } 1142 1143 if s.WrapInfo.WrappedAccessor == "" { 1144 t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) 1145 } else if s.WrapInfo.Token == "" { 1146 t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) 1147 } else if s.WrapInfo.TTL != int(d.Seconds()) { 1148 t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) 1149 } 1150 } 1151 1152 func TestVaultClient_CreateToken_Root_Target_Role(t *testing.T) { 1153 t.Parallel() 1154 v := testutil.NewTestVault(t) 1155 defer v.Stop() 1156 1157 // Create the test role 1158 defaultTestVaultWhitelistRoleAndToken(v, t, 5) 1159 1160 // Target the test role 1161 v.Config.Role = "test" 1162 1163 // Start the client 1164 logger := testlog.HCLogger(t) 1165 client, err := NewVaultClient(v.Config, logger, nil) 1166 if err != nil { 1167 t.Fatalf("failed to build vault client: %v", err) 1168 } 1169 client.SetActive(true) 1170 defer client.Stop() 1171 1172 waitForConnection(client, t) 1173 1174 // Create an allocation that requires a Vault policy 1175 a := mock.Alloc() 1176 task := a.Job.TaskGroups[0].Tasks[0] 1177 task.Vault = &structs.Vault{Policies: []string{"default"}} 1178 1179 s, err := client.CreateToken(context.Background(), a, task.Name) 1180 if err != nil { 1181 t.Fatalf("CreateToken failed: %v", err) 1182 } 1183 1184 // Ensure that created secret is a wrapped token 1185 if s == nil || s.WrapInfo == nil { 1186 t.Fatalf("Bad secret: %#v", s) 1187 } 1188 1189 d, err := time.ParseDuration(vaultTokenCreateTTL) 1190 if err != nil { 1191 t.Fatalf("bad: %v", err) 1192 } 1193 1194 if s.WrapInfo.WrappedAccessor == "" { 1195 t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) 1196 } else if s.WrapInfo.Token == "" { 1197 t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) 1198 } else if s.WrapInfo.TTL != int(d.Seconds()) { 1199 t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) 1200 } 1201 } 1202 1203 func TestVaultClient_CreateToken_Blacklist_Role(t *testing.T) { 1204 t.Parallel() 1205 // Need to skip if test is 0.6.4 1206 version, err := testutil.VaultVersion() 1207 if err != nil { 1208 t.Fatalf("failed to determine version: %v", err) 1209 } 1210 1211 if strings.Contains(version, "v0.6.4") { 1212 t.Skipf("Vault has a regression in v0.6.4 that this test hits") 1213 } 1214 1215 v := testutil.NewTestVault(t) 1216 defer v.Stop() 1217 1218 // Set the configs token in a new test role 1219 v.Config.Token = defaultTestVaultBlacklistRoleAndToken(v, t, 5) 1220 v.Config.Role = "test" 1221 1222 // Start the client 1223 logger := testlog.HCLogger(t) 1224 client, err := NewVaultClient(v.Config, logger, nil) 1225 if err != nil { 1226 t.Fatalf("failed to build vault client: %v", err) 1227 } 1228 client.SetActive(true) 1229 defer client.Stop() 1230 1231 waitForConnection(client, t) 1232 1233 // Create an allocation that requires a Vault policy 1234 a := mock.Alloc() 1235 task := a.Job.TaskGroups[0].Tasks[0] 1236 task.Vault = &structs.Vault{Policies: []string{"secrets"}} 1237 1238 s, err := client.CreateToken(context.Background(), a, task.Name) 1239 if err != nil { 1240 t.Fatalf("CreateToken failed: %v", err) 1241 } 1242 1243 // Ensure that created secret is a wrapped token 1244 if s == nil || s.WrapInfo == nil { 1245 t.Fatalf("Bad secret: %#v", s) 1246 } 1247 1248 d, err := time.ParseDuration(vaultTokenCreateTTL) 1249 if err != nil { 1250 t.Fatalf("bad: %v", err) 1251 } 1252 1253 if s.WrapInfo.WrappedAccessor == "" { 1254 t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) 1255 } else if s.WrapInfo.Token == "" { 1256 t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) 1257 } else if s.WrapInfo.TTL != int(d.Seconds()) { 1258 t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) 1259 } 1260 } 1261 1262 func TestVaultClient_CreateToken_Role_InvalidToken(t *testing.T) { 1263 t.Parallel() 1264 v := testutil.NewTestVault(t) 1265 defer v.Stop() 1266 1267 // Set the configs token in a new test role 1268 defaultTestVaultWhitelistRoleAndToken(v, t, 5) 1269 v.Config.Token = "foo-bar" 1270 1271 // Start the client 1272 logger := testlog.HCLogger(t) 1273 client, err := NewVaultClient(v.Config, logger, nil) 1274 if err != nil { 1275 t.Fatalf("failed to build vault client: %v", err) 1276 } 1277 client.SetActive(true) 1278 defer client.Stop() 1279 1280 testutil.WaitForResult(func() (bool, error) { 1281 established, err := client.ConnectionEstablished() 1282 if !established { 1283 return false, fmt.Errorf("Should establish") 1284 } 1285 return err != nil, nil 1286 }, func(err error) { 1287 t.Fatalf("Connection not established") 1288 }) 1289 1290 // Create an allocation that requires a Vault policy 1291 a := mock.Alloc() 1292 task := a.Job.TaskGroups[0].Tasks[0] 1293 task.Vault = &structs.Vault{Policies: []string{"default"}} 1294 1295 _, err = client.CreateToken(context.Background(), a, task.Name) 1296 if err == nil || !strings.Contains(err.Error(), "failed to establish connection to Vault") { 1297 t.Fatalf("CreateToken should have failed: %v", err) 1298 } 1299 } 1300 1301 func TestVaultClient_CreateToken_Role_Unrecoverable(t *testing.T) { 1302 t.Parallel() 1303 v := testutil.NewTestVault(t) 1304 defer v.Stop() 1305 1306 // Set the configs token in a new test role 1307 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 1308 1309 // Start the client 1310 logger := testlog.HCLogger(t) 1311 client, err := NewVaultClient(v.Config, logger, nil) 1312 if err != nil { 1313 t.Fatalf("failed to build vault client: %v", err) 1314 } 1315 client.SetActive(true) 1316 defer client.Stop() 1317 1318 waitForConnection(client, t) 1319 1320 // Create an allocation that requires a Vault policy 1321 a := mock.Alloc() 1322 task := a.Job.TaskGroups[0].Tasks[0] 1323 task.Vault = &structs.Vault{Policies: []string{"unknown_policy"}} 1324 1325 _, err = client.CreateToken(context.Background(), a, task.Name) 1326 if err == nil { 1327 t.Fatalf("CreateToken should have failed: %v", err) 1328 } 1329 1330 _, ok := err.(structs.Recoverable) 1331 if ok { 1332 t.Fatalf("CreateToken should not be a recoverable error type: %v (%T)", err, err) 1333 } 1334 } 1335 1336 func TestVaultClient_CreateToken_Prestart(t *testing.T) { 1337 t.Parallel() 1338 vconfig := &config.VaultConfig{ 1339 Enabled: helper.BoolToPtr(true), 1340 Token: uuid.Generate(), 1341 Addr: "http://127.0.0.1:0", 1342 } 1343 1344 logger := testlog.HCLogger(t) 1345 client, err := NewVaultClient(vconfig, logger, nil) 1346 if err != nil { 1347 t.Fatalf("failed to build vault client: %v", err) 1348 } 1349 client.SetActive(true) 1350 defer client.Stop() 1351 1352 // Create an allocation that requires a Vault policy 1353 a := mock.Alloc() 1354 task := a.Job.TaskGroups[0].Tasks[0] 1355 task.Vault = &structs.Vault{Policies: []string{"default"}} 1356 1357 _, err = client.CreateToken(context.Background(), a, task.Name) 1358 if err == nil { 1359 t.Fatalf("CreateToken should have failed: %v", err) 1360 } 1361 1362 if rerr, ok := err.(*structs.RecoverableError); !ok { 1363 t.Fatalf("Err should have been type recoverable error") 1364 } else if ok && !rerr.IsRecoverable() { 1365 t.Fatalf("Err should have been recoverable") 1366 } 1367 } 1368 1369 func TestVaultClient_MarkForRevocation(t *testing.T) { 1370 vconfig := &config.VaultConfig{ 1371 Enabled: helper.BoolToPtr(true), 1372 Token: uuid.Generate(), 1373 Addr: "http://127.0.0.1:0", 1374 } 1375 logger := testlog.HCLogger(t) 1376 client, err := NewVaultClient(vconfig, logger, nil) 1377 require.NoError(t, err) 1378 1379 client.SetActive(true) 1380 defer client.Stop() 1381 1382 // Create some VaultAccessors 1383 vas := []*structs.VaultAccessor{ 1384 mock.VaultAccessor(), 1385 mock.VaultAccessor(), 1386 } 1387 1388 err = client.MarkForRevocation(vas) 1389 require.NoError(t, err) 1390 1391 // Wasn't committed 1392 require.Len(t, client.revoking, 2) 1393 require.Equal(t, 2, client.stats().TrackedForRevoke) 1394 1395 } 1396 func TestVaultClient_RevokeTokens_PreEstablishs(t *testing.T) { 1397 t.Parallel() 1398 vconfig := &config.VaultConfig{ 1399 Enabled: helper.BoolToPtr(true), 1400 Token: uuid.Generate(), 1401 Addr: "http://127.0.0.1:0", 1402 } 1403 logger := testlog.HCLogger(t) 1404 client, err := NewVaultClient(vconfig, logger, nil) 1405 if err != nil { 1406 t.Fatalf("failed to build vault client: %v", err) 1407 } 1408 client.SetActive(true) 1409 defer client.Stop() 1410 1411 // Create some VaultAccessors 1412 vas := []*structs.VaultAccessor{ 1413 mock.VaultAccessor(), 1414 mock.VaultAccessor(), 1415 } 1416 1417 if err := client.RevokeTokens(context.Background(), vas, false); err != nil { 1418 t.Fatalf("RevokeTokens failed: %v", err) 1419 } 1420 1421 // Wasn't committed 1422 if len(client.revoking) != 0 { 1423 t.Fatalf("didn't add to revoke loop") 1424 } 1425 1426 if err := client.RevokeTokens(context.Background(), vas, true); err != nil { 1427 t.Fatalf("RevokeTokens failed: %v", err) 1428 } 1429 1430 // Was committed 1431 if len(client.revoking) != 2 { 1432 t.Fatalf("didn't add to revoke loop") 1433 } 1434 1435 if client.stats().TrackedForRevoke != 2 { 1436 t.Fatalf("didn't add to revoke loop") 1437 } 1438 } 1439 1440 // TestVaultClient_RevokeTokens_Failures_TTL asserts that 1441 // the registered TTL doesn't get extended on retries 1442 func TestVaultClient_RevokeTokens_Failures_TTL(t *testing.T) { 1443 t.Parallel() 1444 vconfig := &config.VaultConfig{ 1445 Enabled: helper.BoolToPtr(true), 1446 Token: uuid.Generate(), 1447 Addr: "http://127.0.0.1:0", 1448 } 1449 logger := testlog.HCLogger(t) 1450 client, err := NewVaultClient(vconfig, logger, nil) 1451 if err != nil { 1452 t.Fatalf("failed to build vault client: %v", err) 1453 } 1454 client.SetActive(true) 1455 defer client.Stop() 1456 1457 // Create some VaultAccessors 1458 vas := []*structs.VaultAccessor{ 1459 mock.VaultAccessor(), 1460 mock.VaultAccessor(), 1461 } 1462 1463 err = client.RevokeTokens(context.Background(), vas, true) 1464 require.NoError(t, err) 1465 1466 // Was committed 1467 require.Len(t, client.revoking, 2) 1468 1469 // set TTL 1470 ttl := time.Now().Add(50 * time.Second) 1471 client.revoking[vas[0]] = ttl 1472 client.revoking[vas[1]] = ttl 1473 1474 // revoke again and ensure that TTL isn't extended 1475 err = client.RevokeTokens(context.Background(), vas, true) 1476 require.NoError(t, err) 1477 1478 require.Len(t, client.revoking, 2) 1479 expected := map[*structs.VaultAccessor]time.Time{ 1480 vas[0]: ttl, 1481 vas[1]: ttl, 1482 } 1483 require.Equal(t, expected, client.revoking) 1484 } 1485 1486 func TestVaultClient_RevokeTokens_Root(t *testing.T) { 1487 t.Parallel() 1488 v := testutil.NewTestVault(t) 1489 defer v.Stop() 1490 1491 purged := 0 1492 purge := func(accessors []*structs.VaultAccessor) error { 1493 purged += len(accessors) 1494 return nil 1495 } 1496 1497 logger := testlog.HCLogger(t) 1498 client, err := NewVaultClient(v.Config, logger, purge) 1499 if err != nil { 1500 t.Fatalf("failed to build vault client: %v", err) 1501 } 1502 client.SetActive(true) 1503 defer client.Stop() 1504 1505 waitForConnection(client, t) 1506 1507 // Create some vault tokens 1508 auth := v.Client.Auth().Token() 1509 req := vapi.TokenCreateRequest{ 1510 Policies: []string{"default"}, 1511 } 1512 t1, err := auth.Create(&req) 1513 if err != nil { 1514 t.Fatalf("Failed to create vault token: %v", err) 1515 } 1516 if t1 == nil || t1.Auth == nil { 1517 t.Fatalf("bad secret response: %+v", t1) 1518 } 1519 t2, err := auth.Create(&req) 1520 if err != nil { 1521 t.Fatalf("Failed to create vault token: %v", err) 1522 } 1523 if t2 == nil || t2.Auth == nil { 1524 t.Fatalf("bad secret response: %+v", t2) 1525 } 1526 1527 // Create two VaultAccessors 1528 vas := []*structs.VaultAccessor{ 1529 {Accessor: t1.Auth.Accessor}, 1530 {Accessor: t2.Auth.Accessor}, 1531 } 1532 1533 // Issue a token revocation 1534 if err := client.RevokeTokens(context.Background(), vas, true); err != nil { 1535 t.Fatalf("RevokeTokens failed: %v", err) 1536 } 1537 1538 // Lookup the token and make sure we get an error 1539 if s, err := auth.Lookup(t1.Auth.ClientToken); err == nil { 1540 t.Fatalf("Revoked token lookup didn't fail: %+v", s) 1541 } 1542 if s, err := auth.Lookup(t2.Auth.ClientToken); err == nil { 1543 t.Fatalf("Revoked token lookup didn't fail: %+v", s) 1544 } 1545 1546 if purged != 2 { 1547 t.Fatalf("Expected purged 2; got %d", purged) 1548 } 1549 } 1550 1551 func TestVaultClient_RevokeTokens_Role(t *testing.T) { 1552 t.Parallel() 1553 v := testutil.NewTestVault(t) 1554 defer v.Stop() 1555 1556 // Set the configs token in a new test role 1557 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 1558 1559 purged := 0 1560 purge := func(accessors []*structs.VaultAccessor) error { 1561 purged += len(accessors) 1562 return nil 1563 } 1564 1565 logger := testlog.HCLogger(t) 1566 client, err := NewVaultClient(v.Config, logger, purge) 1567 if err != nil { 1568 t.Fatalf("failed to build vault client: %v", err) 1569 } 1570 client.SetActive(true) 1571 defer client.Stop() 1572 1573 waitForConnection(client, t) 1574 1575 // Create some vault tokens 1576 auth := v.Client.Auth().Token() 1577 req := vapi.TokenCreateRequest{ 1578 Policies: []string{"default"}, 1579 } 1580 t1, err := auth.Create(&req) 1581 if err != nil { 1582 t.Fatalf("Failed to create vault token: %v", err) 1583 } 1584 if t1 == nil || t1.Auth == nil { 1585 t.Fatalf("bad secret response: %+v", t1) 1586 } 1587 t2, err := auth.Create(&req) 1588 if err != nil { 1589 t.Fatalf("Failed to create vault token: %v", err) 1590 } 1591 if t2 == nil || t2.Auth == nil { 1592 t.Fatalf("bad secret response: %+v", t2) 1593 } 1594 1595 // Create two VaultAccessors 1596 vas := []*structs.VaultAccessor{ 1597 {Accessor: t1.Auth.Accessor}, 1598 {Accessor: t2.Auth.Accessor}, 1599 } 1600 1601 // Issue a token revocation 1602 if err := client.RevokeTokens(context.Background(), vas, true); err != nil { 1603 t.Fatalf("RevokeTokens failed: %v", err) 1604 } 1605 1606 // Lookup the token and make sure we get an error 1607 if purged != 2 { 1608 t.Fatalf("Expected purged 2; got %d", purged) 1609 } 1610 if s, err := auth.Lookup(t1.Auth.ClientToken); err == nil { 1611 t.Fatalf("Revoked token lookup didn't fail: %+v", s) 1612 } 1613 if s, err := auth.Lookup(t2.Auth.ClientToken); err == nil { 1614 t.Fatalf("Revoked token lookup didn't fail: %+v", s) 1615 } 1616 } 1617 1618 // TestVaultClient_RevokeTokens_Idempotent asserts that token revocation 1619 // is idempotent, and can cope with cases if token was deleted out of band. 1620 func TestVaultClient_RevokeTokens_Idempotent(t *testing.T) { 1621 t.Parallel() 1622 v := testutil.NewTestVault(t) 1623 defer v.Stop() 1624 1625 // Set the configs token in a new test role 1626 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 1627 1628 purged := map[string]struct{}{} 1629 purge := func(accessors []*structs.VaultAccessor) error { 1630 for _, accessor := range accessors { 1631 purged[accessor.Accessor] = struct{}{} 1632 } 1633 return nil 1634 } 1635 1636 logger := testlog.HCLogger(t) 1637 client, err := NewVaultClient(v.Config, logger, purge) 1638 if err != nil { 1639 t.Fatalf("failed to build vault client: %v", err) 1640 } 1641 client.SetActive(true) 1642 defer client.Stop() 1643 1644 waitForConnection(client, t) 1645 1646 // Create some vault tokens 1647 auth := v.Client.Auth().Token() 1648 req := vapi.TokenCreateRequest{ 1649 Policies: []string{"default"}, 1650 } 1651 t1, err := auth.Create(&req) 1652 require.NoError(t, err) 1653 require.NotNil(t, t1) 1654 require.NotNil(t, t1.Auth) 1655 1656 t2, err := auth.Create(&req) 1657 require.NoError(t, err) 1658 require.NotNil(t, t2) 1659 require.NotNil(t, t2.Auth) 1660 1661 t3, err := auth.Create(&req) 1662 require.NoError(t, err) 1663 require.NotNil(t, t3) 1664 require.NotNil(t, t3.Auth) 1665 1666 // revoke t3 out of band 1667 err = auth.RevokeAccessor(t3.Auth.Accessor) 1668 require.NoError(t, err) 1669 1670 // Create two VaultAccessors 1671 vas := []*structs.VaultAccessor{ 1672 {Accessor: t1.Auth.Accessor}, 1673 {Accessor: t2.Auth.Accessor}, 1674 {Accessor: t3.Auth.Accessor}, 1675 } 1676 1677 // Issue a token revocation 1678 err = client.RevokeTokens(context.Background(), vas, true) 1679 require.NoError(t, err) 1680 require.Empty(t, client.revoking) 1681 1682 // revoke token again 1683 err = client.RevokeTokens(context.Background(), vas, true) 1684 require.NoError(t, err) 1685 require.Empty(t, client.revoking) 1686 1687 // Lookup the token and make sure we get an error 1688 require.Len(t, purged, 3) 1689 require.Contains(t, purged, t1.Auth.Accessor) 1690 require.Contains(t, purged, t2.Auth.Accessor) 1691 require.Contains(t, purged, t3.Auth.Accessor) 1692 s, err := auth.Lookup(t1.Auth.ClientToken) 1693 require.Errorf(t, err, "failed to purge token: %v", s) 1694 s, err = auth.Lookup(t2.Auth.ClientToken) 1695 require.Errorf(t, err, "failed to purge token: %v", s) 1696 } 1697 1698 // TestVaultClient_RevokeDaemon_Bounded asserts that token revocation 1699 // batches are bounded in size. 1700 func TestVaultClient_RevokeDaemon_Bounded(t *testing.T) { 1701 t.Parallel() 1702 v := testutil.NewTestVault(t) 1703 defer v.Stop() 1704 1705 // Set the configs token in a new test role 1706 v.Config.Token = defaultTestVaultWhitelistRoleAndToken(v, t, 5) 1707 1708 // Disable client until we can change settings for testing 1709 conf := v.Config.Copy() 1710 conf.Enabled = helper.BoolToPtr(false) 1711 1712 const ( 1713 batchSize = 100 1714 batches = 3 1715 ) 1716 resultCh := make(chan error, batches) 1717 var totalPurges int64 1718 1719 // Purge function asserts batches are always < batchSize 1720 purge := func(vas []*structs.VaultAccessor) error { 1721 if len(vas) > batchSize { 1722 resultCh <- fmt.Errorf("too many Vault accessors in batch: %d > %d", len(vas), batchSize) 1723 } else { 1724 resultCh <- nil 1725 } 1726 atomic.AddInt64(&totalPurges, int64(len(vas))) 1727 1728 return nil 1729 } 1730 1731 logger := testlog.HCLogger(t) 1732 client, err := NewVaultClient(conf, logger, purge) 1733 require.NoError(t, err) 1734 1735 // Override settings for testing and then enable client 1736 client.maxRevokeBatchSize = batchSize 1737 client.revocationIntv = 3 * time.Millisecond 1738 conf = v.Config.Copy() 1739 conf.Enabled = helper.BoolToPtr(true) 1740 require.NoError(t, client.SetConfig(conf)) 1741 1742 client.SetActive(true) 1743 defer client.Stop() 1744 1745 waitForConnection(client, t) 1746 1747 // Create more tokens in Nomad than can fit in a batch; they don't need 1748 // to exist in Vault. 1749 accessors := make([]*structs.VaultAccessor, batchSize*batches) 1750 for i := 0; i < len(accessors); i++ { 1751 accessors[i] = &structs.VaultAccessor{Accessor: "abcd"} 1752 } 1753 1754 // Mark for revocation 1755 require.NoError(t, client.MarkForRevocation(accessors)) 1756 1757 // Wait for tokens to be revoked 1758 for i := 0; i < batches; i++ { 1759 select { 1760 case err := <-resultCh: 1761 require.NoError(t, err) 1762 case <-time.After(10 * time.Second): 1763 // 10 seconds should be plenty long to process 3 1764 // batches at a 3ms tick interval! 1765 t.Errorf("timed out processing %d batches. %d/%d complete in 10s", 1766 batches, i, batches) 1767 } 1768 } 1769 1770 require.Equal(t, int64(len(accessors)), atomic.LoadInt64(&totalPurges)) 1771 } 1772 1773 func waitForConnection(v *vaultClient, t *testing.T) { 1774 testutil.WaitForResult(func() (bool, error) { 1775 return v.ConnectionEstablished() 1776 }, func(err error) { 1777 t.Fatalf("Connection not established") 1778 }) 1779 } 1780 1781 func TestVaultClient_nextBackoff(t *testing.T) { 1782 simpleCases := []struct { 1783 name string 1784 initBackoff float64 1785 1786 // define range of acceptable backoff values accounting for random factor 1787 rangeMin float64 1788 rangeMax float64 1789 }{ 1790 {"simple case", 7.0, 8.7, 17.60}, 1791 {"too low", 2.0, 5.0, 10.0}, 1792 {"too large", 100, 30.0, 60.0}, 1793 } 1794 1795 for _, c := range simpleCases { 1796 t.Run(c.name, func(t *testing.T) { 1797 b := nextBackoff(c.initBackoff, time.Now().Add(10*time.Hour)) 1798 if !(c.rangeMin <= b && b <= c.rangeMax) { 1799 t.Fatalf("Expected backoff within [%v, %v] but found %v", c.rangeMin, c.rangeMax, b) 1800 } 1801 }) 1802 } 1803 1804 // some edge cases 1805 t.Run("close to expiry", func(t *testing.T) { 1806 b := nextBackoff(20, time.Now().Add(1100*time.Millisecond)) 1807 if b != 5.0 { 1808 t.Fatalf("Expected backoff is 5 but found %v", b) 1809 } 1810 }) 1811 1812 t.Run("past expiry", func(t *testing.T) { 1813 b := nextBackoff(20, time.Now().Add(-1100*time.Millisecond)) 1814 if !(60 <= b && b <= 120) { 1815 t.Fatalf("Expected backoff within [%v, %v] but found %v", 60, 120, b) 1816 } 1817 }) 1818 }