github.com/hernad/nomad@v1.6.112/nomad/vault_test.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package nomad 5 6 import ( 7 "context" 8 "encoding/json" 9 "errors" 10 "fmt" 11 "math/rand" 12 "reflect" 13 "strings" 14 "sync/atomic" 15 "testing" 16 "time" 17 18 "github.com/hernad/nomad/ci" 19 "github.com/stretchr/testify/assert" 20 "github.com/stretchr/testify/require" 21 22 "golang.org/x/time/rate" 23 24 "github.com/hernad/nomad/helper/pointer" 25 "github.com/hernad/nomad/helper/testlog" 26 "github.com/hernad/nomad/helper/uuid" 27 "github.com/hernad/nomad/nomad/mock" 28 "github.com/hernad/nomad/nomad/structs" 29 "github.com/hernad/nomad/nomad/structs/config" 30 "github.com/hernad/nomad/testutil" 31 vapi "github.com/hashicorp/vault/api" 32 vaultconsts "github.com/hashicorp/vault/sdk/helper/consts" 33 ) 34 35 const ( 36 // nomadRoleManagementPolicy is a policy that allows nomad to manage tokens 37 nomadRoleManagementPolicy = ` 38 path "auth/token/renew-self" { 39 capabilities = ["update"] 40 } 41 42 path "auth/token/lookup" { 43 capabilities = ["update"] 44 } 45 46 path "auth/token/roles/test" { 47 capabilities = ["read"] 48 } 49 50 path "auth/token/revoke-accessor" { 51 capabilities = ["update"] 52 } 53 ` 54 55 // tokenLookupPolicy allows a token to be looked up 56 tokenLookupPolicy = ` 57 path "auth/token/lookup" { 58 capabilities = ["update"] 59 } 60 ` 61 62 // nomadRoleCreatePolicy gives the ability to create the role and derive tokens 63 // from the test role 64 nomadRoleCreatePolicy = ` 65 path "auth/token/create/test" { 66 capabilities = ["create", "update"] 67 } 68 ` 69 70 // secretPolicy gives access to the secret mount 71 secretPolicy = ` 72 path "secret/*" { 73 capabilities = ["create", "read", "update", "delete", "list"] 74 } 75 ` 76 ) 77 78 // defaultTestVaultAllowlistRoleAndToken creates a test Vault role and returns a token 79 // created in that role 80 func defaultTestVaultAllowlistRoleAndToken(v *testutil.TestVault, t *testing.T, rolePeriod int) string { 81 vaultPolicies := map[string]string{ 82 "nomad-role-create": nomadRoleCreatePolicy, 83 "nomad-role-management": nomadRoleManagementPolicy, 84 } 85 d := make(map[string]interface{}, 2) 86 d["allowed_policies"] = "nomad-role-create,nomad-role-management" 87 d["period"] = rolePeriod 88 return testVaultRoleAndToken(v, t, vaultPolicies, d, 89 []string{"nomad-role-create", "nomad-role-management"}) 90 } 91 92 // defaultTestVaultDenylistRoleAndToken creates a test Vault role using 93 // disallowed_policies and returns a token created in that role 94 func defaultTestVaultDenylistRoleAndToken(v *testutil.TestVault, t *testing.T, rolePeriod int) string { 95 vaultPolicies := map[string]string{ 96 "nomad-role-create": nomadRoleCreatePolicy, 97 "nomad-role-management": nomadRoleManagementPolicy, 98 "secrets": secretPolicy, 99 } 100 101 // Create the role 102 d := make(map[string]interface{}, 2) 103 d["disallowed_policies"] = "nomad-role-create" 104 d["period"] = rolePeriod 105 testVaultRoleAndToken(v, t, vaultPolicies, d, []string{"default"}) 106 107 // Create a token that can use the role 108 a := v.Client.Auth().Token() 109 req := &vapi.TokenCreateRequest{ 110 Policies: []string{"nomad-role-create", "nomad-role-management"}, 111 } 112 s, err := a.Create(req) 113 if err != nil { 114 t.Fatalf("failed to create child token: %v", err) 115 } 116 117 if s == nil || s.Auth == nil { 118 t.Fatalf("bad secret response: %+v", s) 119 } 120 121 return s.Auth.ClientToken 122 } 123 124 // testVaultRoleAndToken writes the vaultPolicies to vault and then creates a 125 // test role with the passed data. After that it derives a token from the role 126 // with the tokenPolicies 127 func testVaultRoleAndToken(v *testutil.TestVault, t *testing.T, vaultPolicies map[string]string, 128 data map[string]interface{}, tokenPolicies []string) string { 129 // Write the policies 130 sys := v.Client.Sys() 131 for p, data := range vaultPolicies { 132 if err := sys.PutPolicy(p, data); err != nil { 133 t.Fatalf("failed to create %q policy: %v", p, err) 134 } 135 } 136 137 // Build a role 138 l := v.Client.Logical() 139 l.Write("auth/token/roles/test", data) 140 141 // Create a new token with the role 142 a := v.Client.Auth().Token() 143 req := vapi.TokenCreateRequest{ 144 Policies: tokenPolicies, 145 } 146 s, err := a.CreateWithRole(&req, "test") 147 if err != nil { 148 t.Fatalf("failed to create child token: %v", err) 149 } 150 151 // Get the client token 152 if s == nil || s.Auth == nil { 153 t.Fatalf("bad secret response: %+v", s) 154 } 155 156 return s.Auth.ClientToken 157 } 158 159 func TestVaultClient_BadConfig(t *testing.T) { 160 ci.Parallel(t) 161 conf := &config.VaultConfig{} 162 logger := testlog.HCLogger(t) 163 164 // Should be no error since Vault is not enabled 165 _, err := NewVaultClient(nil, logger, nil, nil) 166 if err == nil || !strings.Contains(err.Error(), "valid") { 167 t.Fatalf("expected config error: %v", err) 168 } 169 170 tr := true 171 conf.Enabled = &tr 172 _, err = NewVaultClient(conf, logger, nil, nil) 173 if err == nil || !strings.Contains(err.Error(), "token must be set") { 174 t.Fatalf("Expected token unset error: %v", err) 175 } 176 177 conf.Token = "123" 178 _, err = NewVaultClient(conf, logger, nil, nil) 179 if err == nil || !strings.Contains(err.Error(), "address must be set") { 180 t.Fatalf("Expected address unset error: %v", err) 181 } 182 } 183 184 // TestVaultClient_WithNamespaceSupport tests that the Vault namespace config, if present, will result in the 185 // namespace header being set on the created Vault client. 186 func TestVaultClient_WithNamespaceSupport(t *testing.T) { 187 ci.Parallel(t) 188 require := require.New(t) 189 tr := true 190 testNs := "test-namespace" 191 conf := &config.VaultConfig{ 192 Addr: "https://vault.service.consul:8200", 193 Enabled: &tr, 194 Token: "testvaulttoken", 195 Namespace: testNs, 196 } 197 logger := testlog.HCLogger(t) 198 199 // Should be no error since Vault is not enabled 200 c, err := NewVaultClient(conf, logger, nil, nil) 201 if err != nil { 202 t.Fatalf("failed to build vault client: %v", err) 203 } 204 205 require.Equal(testNs, c.client.Headers().Get(vaultconsts.NamespaceHeaderName)) 206 require.Equal("", c.clientSys.Headers().Get(vaultconsts.NamespaceHeaderName)) 207 require.NotEqual(c.clientSys, c.client) 208 } 209 210 // TestVaultClient_WithoutNamespaceSupport tests that the Vault namespace config, if present, will result in the 211 // namespace header being set on the created Vault client. 212 func TestVaultClient_WithoutNamespaceSupport(t *testing.T) { 213 ci.Parallel(t) 214 require := require.New(t) 215 tr := true 216 conf := &config.VaultConfig{ 217 Addr: "https://vault.service.consul:8200", 218 Enabled: &tr, 219 Token: "testvaulttoken", 220 Namespace: "", 221 } 222 logger := testlog.HCLogger(t) 223 224 // Should be no error since Vault is not enabled 225 c, err := NewVaultClient(conf, logger, nil, nil) 226 if err != nil { 227 t.Fatalf("failed to build vault client: %v", err) 228 } 229 230 require.Equal("", c.client.Headers().Get(vaultconsts.NamespaceHeaderName)) 231 require.Equal("", c.clientSys.Headers().Get(vaultconsts.NamespaceHeaderName)) 232 require.Equal(c.clientSys, c.client) 233 } 234 235 // started separately. 236 // Test that the Vault Client can establish a connection even if it is started 237 // before Vault is available. 238 func TestVaultClient_EstablishConnection(t *testing.T) { 239 ci.Parallel(t) 240 for i := 10; i >= 0; i-- { 241 v := testutil.NewTestVaultDelayed(t) 242 logger := testlog.HCLogger(t) 243 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 244 client, err := NewVaultClient(v.Config, logger, nil, nil) 245 if err != nil { 246 t.Fatalf("failed to build vault client: %v", err) 247 } 248 249 // Sleep a little while and check that no connection has been established. 250 time.Sleep(100 * time.Duration(testutil.TestMultiplier()) * time.Millisecond) 251 if established, _ := client.ConnectionEstablished(); established { 252 t.Fatalf("ConnectionEstablished() returned true before Vault server started") 253 } 254 255 // Start Vault 256 if err := v.Start(); err != nil { 257 v.Stop() 258 client.Stop() 259 260 if i == 0 { 261 t.Fatalf("Failed to start vault: %v", err) 262 } 263 264 wait := time.Duration(rand.Int31n(2000)) * time.Millisecond 265 time.Sleep(wait) 266 continue 267 } 268 269 var waitErr error 270 testutil.WaitForResult(func() (bool, error) { 271 return client.ConnectionEstablished() 272 }, func(err error) { 273 waitErr = err 274 }) 275 276 v.Stop() 277 client.Stop() 278 if waitErr != nil { 279 if i == 0 { 280 t.Fatalf("Failed to start vault: %v", err) 281 } 282 283 wait := time.Duration(rand.Int31n(2000)) * time.Millisecond 284 time.Sleep(wait) 285 continue 286 } 287 288 break 289 } 290 } 291 292 func TestVaultClient_ValidateRole(t *testing.T) { 293 ci.Parallel(t) 294 v := testutil.NewTestVault(t) 295 defer v.Stop() 296 297 // Set the configs token in a new test role 298 vaultPolicies := map[string]string{ 299 "nomad-role-create": nomadRoleCreatePolicy, 300 "nomad-role-management": nomadRoleManagementPolicy, 301 } 302 data := map[string]interface{}{ 303 "allowed_policies": "default,root", 304 "orphan": true, 305 "renewable": true, 306 "token_explicit_max_ttl": 10, 307 } 308 v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, nil) 309 310 logger := testlog.HCLogger(t) 311 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 312 client, err := NewVaultClient(v.Config, logger, nil, nil) 313 require.NoError(t, err) 314 315 defer client.Stop() 316 317 // Wait for an error 318 var conn bool 319 var connErr error 320 testutil.WaitForResult(func() (bool, error) { 321 conn, connErr = client.ConnectionEstablished() 322 if !conn { 323 return false, fmt.Errorf("Should connect") 324 } 325 326 if connErr == nil { 327 return false, fmt.Errorf("expect an error") 328 } 329 330 return true, nil 331 }, func(err error) { 332 require.NoError(t, err) 333 }) 334 335 require.Contains(t, connErr.Error(), "explicit max ttl") 336 require.Contains(t, connErr.Error(), "non-zero period") 337 } 338 339 // TestVaultClient_ValidateRole_Success asserts that a valid token role 340 // gets marked as valid 341 func TestVaultClient_ValidateRole_Success(t *testing.T) { 342 ci.Parallel(t) 343 v := testutil.NewTestVault(t) 344 defer v.Stop() 345 346 // Set the configs token in a new test role 347 vaultPolicies := map[string]string{ 348 "nomad-role-create": nomadRoleCreatePolicy, 349 "nomad-role-management": nomadRoleManagementPolicy, 350 } 351 data := map[string]interface{}{ 352 "allowed_policies": "default,root", 353 "orphan": true, 354 "renewable": true, 355 "token_period": 1000, 356 } 357 v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, nil) 358 359 logger := testlog.HCLogger(t) 360 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 361 client, err := NewVaultClient(v.Config, logger, nil, nil) 362 require.NoError(t, err) 363 364 defer client.Stop() 365 366 // Wait for an error 367 var conn bool 368 var connErr error 369 testutil.WaitForResult(func() (bool, error) { 370 conn, connErr = client.ConnectionEstablished() 371 if !conn { 372 return false, fmt.Errorf("Should connect") 373 } 374 375 if connErr != nil { 376 return false, connErr 377 } 378 379 return true, nil 380 }, func(err error) { 381 require.NoError(t, err) 382 }) 383 } 384 385 // TestVaultClient_ValidateRole_Deprecated_Success asserts that a valid token 386 // role gets marked as valid, even if it uses deprecated field, period 387 func TestVaultClient_ValidateRole_Deprecated_Success(t *testing.T) { 388 ci.Parallel(t) 389 v := testutil.NewTestVault(t) 390 defer v.Stop() 391 392 // Set the configs token in a new test role 393 vaultPolicies := map[string]string{ 394 "nomad-role-create": nomadRoleCreatePolicy, 395 "nomad-role-management": nomadRoleManagementPolicy, 396 } 397 data := map[string]interface{}{ 398 "allowed_policies": "default,root", 399 "orphan": true, 400 "renewable": true, 401 "period": 1000, 402 } 403 v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, nil) 404 405 logger := testlog.HCLogger(t) 406 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 407 client, err := NewVaultClient(v.Config, logger, nil, nil) 408 require.NoError(t, err) 409 410 defer client.Stop() 411 412 // Wait for an error 413 var conn bool 414 var connErr error 415 testutil.WaitForResult(func() (bool, error) { 416 conn, connErr = client.ConnectionEstablished() 417 if !conn { 418 return false, fmt.Errorf("Should connect") 419 } 420 421 if connErr != nil { 422 return false, connErr 423 } 424 425 return true, nil 426 }, func(err error) { 427 require.NoError(t, err) 428 }) 429 } 430 431 func TestVaultClient_ValidateRole_NonExistent(t *testing.T) { 432 ci.Parallel(t) 433 v := testutil.NewTestVault(t) 434 defer v.Stop() 435 436 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 437 v.Config.Token = v.RootToken 438 logger := testlog.HCLogger(t) 439 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 440 v.Config.Role = "test-nonexistent" 441 client, err := NewVaultClient(v.Config, logger, nil, nil) 442 if err != nil { 443 t.Fatalf("failed to build vault client: %v", err) 444 } 445 defer client.Stop() 446 447 // Wait for an error 448 var conn bool 449 var connErr error 450 testutil.WaitForResult(func() (bool, error) { 451 conn, connErr = client.ConnectionEstablished() 452 if !conn { 453 return false, fmt.Errorf("Should connect") 454 } 455 456 if connErr == nil { 457 return false, fmt.Errorf("expect an error") 458 } 459 460 return true, nil 461 }, func(err error) { 462 t.Fatalf("bad: %v", err) 463 }) 464 465 errStr := connErr.Error() 466 if !strings.Contains(errStr, "does not exist") { 467 t.Fatalf("Expect does not exist error") 468 } 469 } 470 471 func TestVaultClient_ValidateToken(t *testing.T) { 472 ci.Parallel(t) 473 v := testutil.NewTestVault(t) 474 defer v.Stop() 475 476 // Set the configs token in a new test role 477 vaultPolicies := map[string]string{ 478 "nomad-role-create": nomadRoleCreatePolicy, 479 "token-lookup": tokenLookupPolicy, 480 } 481 data := map[string]interface{}{ 482 "allowed_policies": "token-lookup,nomad-role-create", 483 "period": 10, 484 } 485 v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, []string{"token-lookup", "nomad-role-create"}) 486 487 logger := testlog.HCLogger(t) 488 v.Config.ConnectionRetryIntv = 100 * time.Millisecond 489 client, err := NewVaultClient(v.Config, logger, nil, nil) 490 if err != nil { 491 t.Fatalf("failed to build vault client: %v", err) 492 } 493 defer client.Stop() 494 495 // Wait for an error 496 var conn bool 497 var connErr error 498 testutil.WaitForResult(func() (bool, error) { 499 conn, connErr = client.ConnectionEstablished() 500 if !conn { 501 return false, fmt.Errorf("Should connect") 502 } 503 504 if connErr == nil { 505 return false, fmt.Errorf("expect an error") 506 } 507 508 return true, nil 509 }, func(err error) { 510 t.Fatalf("bad: %v", err) 511 }) 512 513 errStr := connErr.Error() 514 if !strings.Contains(errStr, vaultTokenRevokePath) { 515 t.Fatalf("Expect revoke error") 516 } 517 if !strings.Contains(errStr, fmt.Sprintf(vaultRoleLookupPath, "test")) { 518 t.Fatalf("Expect explicit max ttl error") 519 } 520 if !strings.Contains(errStr, "token must have one of the following") { 521 t.Fatalf("Expect explicit max ttl error") 522 } 523 } 524 525 func TestVaultClient_SetActive(t *testing.T) { 526 ci.Parallel(t) 527 v := testutil.NewTestVault(t) 528 defer v.Stop() 529 530 logger := testlog.HCLogger(t) 531 client, err := NewVaultClient(v.Config, logger, nil, nil) 532 if err != nil { 533 t.Fatalf("failed to build vault client: %v", err) 534 } 535 defer client.Stop() 536 537 waitForConnection(client, t) 538 539 // Do a lookup and expect an error about not being active 540 _, err = client.LookupToken(context.Background(), "123") 541 if err == nil || !strings.Contains(err.Error(), "not active") { 542 t.Fatalf("Expected not-active error: %v", err) 543 } 544 545 client.SetActive(true) 546 547 // Do a lookup of ourselves 548 _, err = client.LookupToken(context.Background(), v.RootToken) 549 if err != nil { 550 t.Fatalf("Unexpected error: %v", err) 551 } 552 } 553 554 // Test that we can update the config and things keep working 555 func TestVaultClient_SetConfig(t *testing.T) { 556 ci.Parallel(t) 557 v := testutil.NewTestVault(t) 558 defer v.Stop() 559 560 v2 := testutil.NewTestVault(t) 561 defer v2.Stop() 562 563 // Set the configs token in a new test role 564 v2.Config.Token = defaultTestVaultAllowlistRoleAndToken(v2, t, 20) 565 566 logger := testlog.HCLogger(t) 567 client, err := NewVaultClient(v.Config, logger, nil, nil) 568 if err != nil { 569 t.Fatalf("failed to build vault client: %v", err) 570 } 571 defer client.Stop() 572 573 waitForConnection(client, t) 574 575 if client.tokenData == nil || len(client.tokenData.Policies) != 1 { 576 t.Fatalf("unexpected token: %v", client.tokenData) 577 } 578 579 // Update the config 580 if err := client.SetConfig(v2.Config); err != nil { 581 t.Fatalf("SetConfig failed: %v", err) 582 } 583 584 waitForConnection(client, t) 585 586 if client.tokenData == nil || len(client.tokenData.Policies) != 3 { 587 t.Fatalf("unexpected token: %v", client.tokenData) 588 } 589 590 // Test that when SetConfig is called with the same configuration, it is a 591 // no-op 592 failCh := make(chan struct{}, 1) 593 go func() { 594 tomb := client.tomb 595 select { 596 case <-tomb.Dying(): 597 close(failCh) 598 case <-time.After(1 * time.Second): 599 return 600 } 601 }() 602 603 // Update the config 604 if err := client.SetConfig(v2.Config); err != nil { 605 t.Fatalf("SetConfig failed: %v", err) 606 } 607 608 select { 609 case <-failCh: 610 t.Fatalf("Tomb shouldn't have exited") 611 case <-time.After(1 * time.Second): 612 return 613 } 614 } 615 616 // TestVaultClient_SetConfig_Deadlock asserts that calling SetConfig 617 // concurrently with establishConnection does not deadlock. 618 func TestVaultClient_SetConfig_Deadlock(t *testing.T) { 619 ci.Parallel(t) 620 v := testutil.NewTestVault(t) 621 defer v.Stop() 622 623 v2 := testutil.NewTestVault(t) 624 defer v2.Stop() 625 626 // Set the configs token in a new test role 627 v2.Config.Token = defaultTestVaultAllowlistRoleAndToken(v2, t, 20) 628 629 logger := testlog.HCLogger(t) 630 client, err := NewVaultClient(v.Config, logger, nil, nil) 631 if err != nil { 632 t.Fatalf("failed to build vault client: %v", err) 633 } 634 defer client.Stop() 635 636 for i := 0; i < 100; i++ { 637 // Alternate configs to cause updates 638 conf := v.Config 639 if i%2 == 0 { 640 conf = v2.Config 641 } 642 if err := client.SetConfig(conf); err != nil { 643 t.Fatalf("SetConfig failed: %v", err) 644 } 645 } 646 } 647 648 // Test that we can disable vault 649 func TestVaultClient_SetConfig_Disable(t *testing.T) { 650 ci.Parallel(t) 651 v := testutil.NewTestVault(t) 652 defer v.Stop() 653 654 logger := testlog.HCLogger(t) 655 client, err := NewVaultClient(v.Config, logger, nil, nil) 656 if err != nil { 657 t.Fatalf("failed to build vault client: %v", err) 658 } 659 defer client.Stop() 660 661 waitForConnection(client, t) 662 663 if client.tokenData == nil || len(client.tokenData.Policies) != 1 { 664 t.Fatalf("unexpected token: %v", client.tokenData) 665 } 666 667 // Disable vault 668 f := false 669 config := config.VaultConfig{ 670 Enabled: &f, 671 } 672 673 // Update the config 674 if err := client.SetConfig(&config); err != nil { 675 t.Fatalf("SetConfig failed: %v", err) 676 } 677 678 if client.Enabled() || client.Running() { 679 t.Fatalf("SetConfig should have stopped client") 680 } 681 } 682 683 func TestVaultClient_RenewalLoop(t *testing.T) { 684 ci.Parallel(t) 685 v := testutil.NewTestVault(t) 686 defer v.Stop() 687 688 // Set the configs token in a new test role 689 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 690 691 // Start the client 692 logger := testlog.HCLogger(t) 693 client, err := NewVaultClient(v.Config, logger, nil, nil) 694 if err != nil { 695 t.Fatalf("failed to build vault client: %v", err) 696 } 697 defer client.Stop() 698 699 // Sleep 8 seconds and ensure we have a non-zero TTL 700 time.Sleep(8 * time.Second) 701 702 // Get the current TTL 703 a := v.Client.Auth().Token() 704 s2, err := a.Lookup(v.Config.Token) 705 if err != nil { 706 t.Fatalf("failed to lookup token: %v", err) 707 } 708 709 ttl := parseTTLFromLookup(s2, t) 710 if ttl == 0 { 711 t.Fatalf("token renewal failed; ttl %v", ttl) 712 } 713 714 if client.currentExpiration.Before(time.Now()) { 715 t.Fatalf("found current expiration to be in past %s", time.Until(client.currentExpiration)) 716 } 717 } 718 719 func TestVaultClientRenewUpdatesExpiration(t *testing.T) { 720 ci.Parallel(t) 721 v := testutil.NewTestVault(t) 722 defer v.Stop() 723 724 // Set the configs token in a new test role 725 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 726 727 // Start the client 728 logger := testlog.HCLogger(t) 729 client, err := NewVaultClient(v.Config, logger, nil, nil) 730 if err != nil { 731 t.Fatalf("failed to build vault client: %v", err) 732 } 733 defer client.Stop() 734 735 // Get the current TTL 736 a := v.Client.Auth().Token() 737 s2, err := a.Lookup(v.Config.Token) 738 if err != nil { 739 t.Fatalf("failed to lookup token: %v", err) 740 } 741 exp0 := time.Now().Add(time.Duration(parseTTLFromLookup(s2, t)) * time.Second) 742 743 time.Sleep(1 * time.Second) 744 745 _, err = client.renew() 746 require.NoError(t, err) 747 exp1 := client.currentExpiration 748 require.True(t, exp0.Before(exp1)) 749 750 time.Sleep(1 * time.Second) 751 752 _, err = client.renew() 753 require.NoError(t, err) 754 exp2 := client.currentExpiration 755 require.True(t, exp1.Before(exp2)) 756 } 757 758 func TestVaultClient_StopsAfterPermissionError(t *testing.T) { 759 ci.Parallel(t) 760 v := testutil.NewTestVault(t) 761 defer v.Stop() 762 763 // Set the configs token in a new test role 764 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 2) 765 766 // Start the client 767 logger := testlog.HCLogger(t) 768 client, err := NewVaultClient(v.Config, logger, nil, nil) 769 if err != nil { 770 t.Fatalf("failed to build vault client: %v", err) 771 } 772 defer client.Stop() 773 774 time.Sleep(500 * time.Millisecond) 775 776 assert.True(t, client.isRenewLoopActive()) 777 778 // Get the current TTL 779 a := v.Client.Auth().Token() 780 assert.NoError(t, a.RevokeSelf("")) 781 782 testutil.WaitForResult(func() (bool, error) { 783 if !client.isRenewLoopActive() { 784 return true, nil 785 } else { 786 return false, errors.New("renew loop should terminate after token is revoked") 787 } 788 }, func(err error) { 789 t.Fatalf("err: %v", err) 790 }) 791 } 792 func TestVaultClient_LoopsUntilCannotRenew(t *testing.T) { 793 ci.Parallel(t) 794 v := testutil.NewTestVault(t) 795 defer v.Stop() 796 797 // Set the configs token in a new test role 798 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 799 800 // Start the client 801 logger := testlog.HCLogger(t) 802 client, err := NewVaultClient(v.Config, logger, nil, nil) 803 if err != nil { 804 t.Fatalf("failed to build vault client: %v", err) 805 } 806 defer client.Stop() 807 808 // Sleep 8 seconds and ensure we have a non-zero TTL 809 time.Sleep(8 * time.Second) 810 811 // Get the current TTL 812 a := v.Client.Auth().Token() 813 s2, err := a.Lookup(v.Config.Token) 814 if err != nil { 815 t.Fatalf("failed to lookup token: %v", err) 816 } 817 818 ttl := parseTTLFromLookup(s2, t) 819 if ttl == 0 { 820 t.Fatalf("token renewal failed; ttl %v", ttl) 821 } 822 823 if client.currentExpiration.Before(time.Now()) { 824 t.Fatalf("found current expiration to be in past %s", time.Until(client.currentExpiration)) 825 } 826 } 827 828 func parseTTLFromLookup(s *vapi.Secret, t *testing.T) int64 { 829 if s == nil { 830 t.Fatalf("nil secret") 831 } else if s.Data == nil { 832 t.Fatalf("nil data block in secret") 833 } 834 835 ttlRaw, ok := s.Data["ttl"] 836 if !ok { 837 t.Fatalf("no ttl") 838 } 839 840 ttlNumber, ok := ttlRaw.(json.Number) 841 if !ok { 842 t.Fatalf("failed to convert ttl %q to json Number", ttlRaw) 843 } 844 845 ttl, err := ttlNumber.Int64() 846 if err != nil { 847 t.Fatalf("Failed to get ttl from json.Number: %v", err) 848 } 849 850 return ttl 851 } 852 853 func TestVaultClient_LookupToken_Invalid(t *testing.T) { 854 ci.Parallel(t) 855 tr := true 856 conf := &config.VaultConfig{ 857 Enabled: &tr, 858 Addr: "http://foobar:12345", 859 Token: uuid.Generate(), 860 } 861 862 // Enable vault but use a bad address so it never establishes a conn 863 logger := testlog.HCLogger(t) 864 client, err := NewVaultClient(conf, logger, nil, nil) 865 if err != nil { 866 t.Fatalf("failed to build vault client: %v", err) 867 } 868 client.SetActive(true) 869 defer client.Stop() 870 871 _, err = client.LookupToken(context.Background(), "foo") 872 if err == nil || !strings.Contains(err.Error(), "established") { 873 t.Fatalf("Expected error because connection to Vault hasn't been made: %v", err) 874 } 875 } 876 877 func TestVaultClient_LookupToken_Root(t *testing.T) { 878 ci.Parallel(t) 879 v := testutil.NewTestVault(t) 880 defer v.Stop() 881 882 logger := testlog.HCLogger(t) 883 client, err := NewVaultClient(v.Config, logger, nil, nil) 884 if err != nil { 885 t.Fatalf("failed to build vault client: %v", err) 886 } 887 client.SetActive(true) 888 defer client.Stop() 889 890 waitForConnection(client, t) 891 892 // Lookup ourselves 893 s, err := client.LookupToken(context.Background(), v.Config.Token) 894 if err != nil { 895 t.Fatalf("self lookup failed: %v", err) 896 } 897 898 policies, err := s.TokenPolicies() 899 if err != nil { 900 t.Fatalf("failed to parse policies: %v", err) 901 } 902 903 expected := []string{"root"} 904 if !reflect.DeepEqual(policies, expected) { 905 t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) 906 } 907 908 // Create a token with a different set of policies 909 expected = []string{"default"} 910 req := vapi.TokenCreateRequest{ 911 Policies: expected, 912 } 913 s, err = v.Client.Auth().Token().Create(&req) 914 if err != nil { 915 t.Fatalf("failed to create child token: %v", err) 916 } 917 918 // Get the client token 919 if s == nil || s.Auth == nil { 920 t.Fatalf("bad secret response: %+v", s) 921 } 922 923 // Lookup new child 924 s, err = client.LookupToken(context.Background(), s.Auth.ClientToken) 925 if err != nil { 926 t.Fatalf("self lookup failed: %v", err) 927 } 928 929 policies, err = s.TokenPolicies() 930 if err != nil { 931 t.Fatalf("failed to parse policies: %v", err) 932 } 933 934 if !reflect.DeepEqual(policies, expected) { 935 t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) 936 } 937 } 938 939 func TestVaultClient_LookupToken_Role(t *testing.T) { 940 ci.Parallel(t) 941 v := testutil.NewTestVault(t) 942 defer v.Stop() 943 944 // Set the configs token in a new test role 945 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 946 947 logger := testlog.HCLogger(t) 948 client, err := NewVaultClient(v.Config, logger, nil, nil) 949 if err != nil { 950 t.Fatalf("failed to build vault client: %v", err) 951 } 952 client.SetActive(true) 953 defer client.Stop() 954 955 waitForConnection(client, t) 956 957 // Lookup ourselves 958 s, err := client.LookupToken(context.Background(), v.Config.Token) 959 if err != nil { 960 t.Fatalf("self lookup failed: %v", err) 961 } 962 963 policies, err := s.TokenPolicies() 964 if err != nil { 965 t.Fatalf("failed to parse policies: %v", err) 966 } 967 968 expected := []string{"default", "nomad-role-create", "nomad-role-management"} 969 if !reflect.DeepEqual(policies, expected) { 970 t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) 971 } 972 973 // Create a token with a different set of policies 974 expected = []string{"default"} 975 req := vapi.TokenCreateRequest{ 976 Policies: expected, 977 } 978 s, err = v.Client.Auth().Token().Create(&req) 979 if err != nil { 980 t.Fatalf("failed to create child token: %v", err) 981 } 982 983 // Get the client token 984 if s == nil || s.Auth == nil { 985 t.Fatalf("bad secret response: %+v", s) 986 } 987 988 // Lookup new child 989 s, err = client.LookupToken(context.Background(), s.Auth.ClientToken) 990 if err != nil { 991 t.Fatalf("self lookup failed: %v", err) 992 } 993 994 policies, err = s.TokenPolicies() 995 if err != nil { 996 t.Fatalf("failed to parse policies: %v", err) 997 } 998 999 if !reflect.DeepEqual(policies, expected) { 1000 t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) 1001 } 1002 } 1003 1004 func TestVaultClient_LookupToken_RateLimit(t *testing.T) { 1005 ci.Parallel(t) 1006 v := testutil.NewTestVault(t) 1007 defer v.Stop() 1008 1009 logger := testlog.HCLogger(t) 1010 client, err := NewVaultClient(v.Config, logger, nil, nil) 1011 if err != nil { 1012 t.Fatalf("failed to build vault client: %v", err) 1013 } 1014 client.SetActive(true) 1015 defer client.Stop() 1016 1017 waitForConnection(client, t) 1018 1019 client.setLimit(rate.Limit(1.0)) 1020 testRateLimit(t, 20, client, func(ctx context.Context) error { 1021 // Lookup ourselves 1022 _, err := client.LookupToken(ctx, v.Config.Token) 1023 return err 1024 }) 1025 } 1026 1027 func TestVaultClient_CreateToken_Root(t *testing.T) { 1028 ci.Parallel(t) 1029 v := testutil.NewTestVault(t) 1030 defer v.Stop() 1031 1032 logger := testlog.HCLogger(t) 1033 client, err := NewVaultClient(v.Config, logger, nil, nil) 1034 if err != nil { 1035 t.Fatalf("failed to build vault client: %v", err) 1036 } 1037 client.SetActive(true) 1038 defer client.Stop() 1039 1040 waitForConnection(client, t) 1041 1042 // Create an allocation that requires a Vault policy 1043 a := mock.Alloc() 1044 task := a.Job.TaskGroups[0].Tasks[0] 1045 task.Vault = &structs.Vault{Policies: []string{"default"}} 1046 1047 s, err := client.CreateToken(context.Background(), a, task.Name) 1048 if err != nil { 1049 t.Fatalf("CreateToken failed: %v", err) 1050 } 1051 1052 // Ensure that created secret is a wrapped token 1053 if s == nil || s.WrapInfo == nil { 1054 t.Fatalf("Bad secret: %#v", s) 1055 } 1056 1057 d, err := time.ParseDuration(vaultTokenCreateTTL) 1058 if err != nil { 1059 t.Fatalf("bad: %v", err) 1060 } 1061 1062 if s.WrapInfo.WrappedAccessor == "" { 1063 t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) 1064 } else if s.WrapInfo.Token == "" { 1065 t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) 1066 } else if s.WrapInfo.TTL != int(d.Seconds()) { 1067 t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) 1068 } 1069 } 1070 1071 func TestVaultClient_CreateToken_Allowlist_Role(t *testing.T) { 1072 ci.Parallel(t) 1073 1074 v := testutil.NewTestVault(t) 1075 defer v.Stop() 1076 1077 // Set the configs token in a new test role 1078 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 1079 1080 // Start the client 1081 logger := testlog.HCLogger(t) 1082 client, err := NewVaultClient(v.Config, logger, nil, nil) 1083 if err != nil { 1084 t.Fatalf("failed to build vault client: %v", err) 1085 } 1086 client.SetActive(true) 1087 defer client.Stop() 1088 1089 waitForConnection(client, t) 1090 1091 // Create an allocation that requires a Vault policy 1092 a := mock.Alloc() 1093 task := a.Job.TaskGroups[0].Tasks[0] 1094 task.Vault = &structs.Vault{Policies: []string{"default"}} 1095 1096 s, err := client.CreateToken(context.Background(), a, task.Name) 1097 if err != nil { 1098 t.Fatalf("CreateToken failed: %v", err) 1099 } 1100 1101 // Ensure that created secret is a wrapped token 1102 if s == nil || s.WrapInfo == nil { 1103 t.Fatalf("Bad secret: %#v", s) 1104 } 1105 1106 d, err := time.ParseDuration(vaultTokenCreateTTL) 1107 if err != nil { 1108 t.Fatalf("bad: %v", err) 1109 } 1110 1111 if s.WrapInfo.WrappedAccessor == "" { 1112 t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) 1113 } else if s.WrapInfo.Token == "" { 1114 t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) 1115 } else if s.WrapInfo.TTL != int(d.Seconds()) { 1116 t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) 1117 } 1118 } 1119 1120 func TestVaultClient_CreateToken_Root_Target_Role(t *testing.T) { 1121 ci.Parallel(t) 1122 v := testutil.NewTestVault(t) 1123 defer v.Stop() 1124 1125 // Create the test role 1126 defaultTestVaultAllowlistRoleAndToken(v, t, 5) 1127 1128 // Target the test role 1129 v.Config.Role = "test" 1130 1131 // Start the client 1132 logger := testlog.HCLogger(t) 1133 client, err := NewVaultClient(v.Config, logger, nil, nil) 1134 if err != nil { 1135 t.Fatalf("failed to build vault client: %v", err) 1136 } 1137 client.SetActive(true) 1138 defer client.Stop() 1139 1140 waitForConnection(client, t) 1141 1142 // Create an allocation that requires a Vault policy 1143 a := mock.Alloc() 1144 task := a.Job.TaskGroups[0].Tasks[0] 1145 task.Vault = &structs.Vault{Policies: []string{"default"}} 1146 1147 s, err := client.CreateToken(context.Background(), a, task.Name) 1148 if err != nil { 1149 t.Fatalf("CreateToken failed: %v", err) 1150 } 1151 1152 // Ensure that created secret is a wrapped token 1153 if s == nil || s.WrapInfo == nil { 1154 t.Fatalf("Bad secret: %#v", s) 1155 } 1156 1157 d, err := time.ParseDuration(vaultTokenCreateTTL) 1158 if err != nil { 1159 t.Fatalf("bad: %v", err) 1160 } 1161 1162 if s.WrapInfo.WrappedAccessor == "" { 1163 t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) 1164 } else if s.WrapInfo.Token == "" { 1165 t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) 1166 } else if s.WrapInfo.TTL != int(d.Seconds()) { 1167 t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) 1168 } 1169 } 1170 1171 func TestVaultClient_CreateToken_Denylist_Role(t *testing.T) { 1172 ci.Parallel(t) 1173 1174 // Need to skip if test is 0.6.4 1175 version, err := testutil.VaultVersion() 1176 if err != nil { 1177 t.Fatalf("failed to determine version: %v", err) 1178 } 1179 1180 if strings.Contains(version, "v0.6.4") { 1181 t.Skipf("Vault has a regression in v0.6.4 that this test hits") 1182 } 1183 1184 v := testutil.NewTestVault(t) 1185 defer v.Stop() 1186 1187 // Set the configs token in a new test role 1188 v.Config.Token = defaultTestVaultDenylistRoleAndToken(v, t, 5) 1189 v.Config.Role = "test" 1190 1191 // Start the client 1192 logger := testlog.HCLogger(t) 1193 client, err := NewVaultClient(v.Config, logger, nil, nil) 1194 if err != nil { 1195 t.Fatalf("failed to build vault client: %v", err) 1196 } 1197 client.SetActive(true) 1198 defer client.Stop() 1199 1200 waitForConnection(client, t) 1201 1202 // Create an allocation that requires a Vault policy 1203 a := mock.Alloc() 1204 task := a.Job.TaskGroups[0].Tasks[0] 1205 task.Vault = &structs.Vault{Policies: []string{"secrets"}} 1206 1207 s, err := client.CreateToken(context.Background(), a, task.Name) 1208 if err != nil { 1209 t.Fatalf("CreateToken failed: %v", err) 1210 } 1211 1212 // Ensure that created secret is a wrapped token 1213 if s == nil || s.WrapInfo == nil { 1214 t.Fatalf("Bad secret: %#v", s) 1215 } 1216 1217 d, err := time.ParseDuration(vaultTokenCreateTTL) 1218 if err != nil { 1219 t.Fatalf("bad: %v", err) 1220 } 1221 1222 if s.WrapInfo.WrappedAccessor == "" { 1223 t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) 1224 } else if s.WrapInfo.Token == "" { 1225 t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) 1226 } else if s.WrapInfo.TTL != int(d.Seconds()) { 1227 t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) 1228 } 1229 } 1230 1231 func TestVaultClient_CreateToken_Role_InvalidToken(t *testing.T) { 1232 ci.Parallel(t) 1233 v := testutil.NewTestVault(t) 1234 defer v.Stop() 1235 1236 // Set the configs token in a new test role 1237 defaultTestVaultAllowlistRoleAndToken(v, t, 5) 1238 v.Config.Token = "foo-bar" 1239 1240 // Start the client 1241 logger := testlog.HCLogger(t) 1242 client, err := NewVaultClient(v.Config, logger, nil, nil) 1243 if err != nil { 1244 t.Fatalf("failed to build vault client: %v", err) 1245 } 1246 client.SetActive(true) 1247 defer client.Stop() 1248 1249 testutil.WaitForResult(func() (bool, error) { 1250 established, err := client.ConnectionEstablished() 1251 if !established { 1252 return false, fmt.Errorf("Should establish") 1253 } 1254 return err != nil, nil 1255 }, func(err error) { 1256 t.Fatalf("Connection not established") 1257 }) 1258 1259 // Create an allocation that requires a Vault policy 1260 a := mock.Alloc() 1261 task := a.Job.TaskGroups[0].Tasks[0] 1262 task.Vault = &structs.Vault{Policies: []string{"default"}} 1263 1264 _, err = client.CreateToken(context.Background(), a, task.Name) 1265 if err == nil || !strings.Contains(err.Error(), "failed to establish connection to Vault") { 1266 t.Fatalf("CreateToken should have failed: %v", err) 1267 } 1268 } 1269 1270 func TestVaultClient_CreateToken_Role_Unrecoverable(t *testing.T) { 1271 ci.Parallel(t) 1272 v := testutil.NewTestVault(t) 1273 defer v.Stop() 1274 1275 // Set the configs token in a new test role 1276 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 1277 1278 // Start the client 1279 logger := testlog.HCLogger(t) 1280 client, err := NewVaultClient(v.Config, logger, nil, nil) 1281 if err != nil { 1282 t.Fatalf("failed to build vault client: %v", err) 1283 } 1284 client.SetActive(true) 1285 defer client.Stop() 1286 1287 waitForConnection(client, t) 1288 1289 // Create an allocation that requires a Vault policy 1290 a := mock.Alloc() 1291 task := a.Job.TaskGroups[0].Tasks[0] 1292 task.Vault = &structs.Vault{Policies: []string{"unknown_policy"}} 1293 1294 _, err = client.CreateToken(context.Background(), a, task.Name) 1295 if err == nil { 1296 t.Fatalf("CreateToken should have failed: %v", err) 1297 } 1298 1299 _, ok := err.(structs.Recoverable) 1300 if ok { 1301 t.Fatalf("CreateToken should not be a recoverable error type: %v (%T)", err, err) 1302 } 1303 } 1304 1305 func TestVaultClient_CreateToken_Prestart(t *testing.T) { 1306 ci.Parallel(t) 1307 vconfig := &config.VaultConfig{ 1308 Enabled: pointer.Of(true), 1309 Token: uuid.Generate(), 1310 Addr: "http://127.0.0.1:0", 1311 } 1312 1313 logger := testlog.HCLogger(t) 1314 client, err := NewVaultClient(vconfig, logger, nil, nil) 1315 if err != nil { 1316 t.Fatalf("failed to build vault client: %v", err) 1317 } 1318 client.SetActive(true) 1319 defer client.Stop() 1320 1321 // Create an allocation that requires a Vault policy 1322 a := mock.Alloc() 1323 task := a.Job.TaskGroups[0].Tasks[0] 1324 task.Vault = &structs.Vault{Policies: []string{"default"}} 1325 1326 _, err = client.CreateToken(context.Background(), a, task.Name) 1327 if err == nil { 1328 t.Fatalf("CreateToken should have failed: %v", err) 1329 } 1330 1331 if rerr, ok := err.(*structs.RecoverableError); !ok { 1332 t.Fatalf("Err should have been type recoverable error") 1333 } else if ok && !rerr.IsRecoverable() { 1334 t.Fatalf("Err should have been recoverable") 1335 } 1336 } 1337 1338 func TestVaultClient_MarkForRevocation(t *testing.T) { 1339 vconfig := &config.VaultConfig{ 1340 Enabled: pointer.Of(true), 1341 Token: uuid.Generate(), 1342 Addr: "http://127.0.0.1:0", 1343 } 1344 logger := testlog.HCLogger(t) 1345 client, err := NewVaultClient(vconfig, logger, nil, nil) 1346 require.NoError(t, err) 1347 1348 client.SetActive(true) 1349 defer client.Stop() 1350 1351 // Create some VaultAccessors 1352 vas := []*structs.VaultAccessor{ 1353 mock.VaultAccessor(), 1354 mock.VaultAccessor(), 1355 } 1356 1357 err = client.MarkForRevocation(vas) 1358 require.NoError(t, err) 1359 1360 // Wasn't committed 1361 require.Len(t, client.revoking, 2) 1362 require.Equal(t, 2, client.stats().TrackedForRevoke) 1363 1364 } 1365 func TestVaultClient_RevokeTokens_PreEstablishs(t *testing.T) { 1366 ci.Parallel(t) 1367 vconfig := &config.VaultConfig{ 1368 Enabled: pointer.Of(true), 1369 Token: uuid.Generate(), 1370 Addr: "http://127.0.0.1:0", 1371 } 1372 logger := testlog.HCLogger(t) 1373 client, err := NewVaultClient(vconfig, logger, nil, nil) 1374 if err != nil { 1375 t.Fatalf("failed to build vault client: %v", err) 1376 } 1377 client.SetActive(true) 1378 defer client.Stop() 1379 1380 // Create some VaultAccessors 1381 vas := []*structs.VaultAccessor{ 1382 mock.VaultAccessor(), 1383 mock.VaultAccessor(), 1384 } 1385 1386 if err := client.RevokeTokens(context.Background(), vas, false); err != nil { 1387 t.Fatalf("RevokeTokens failed: %v", err) 1388 } 1389 1390 // Wasn't committed 1391 if len(client.revoking) != 0 { 1392 t.Fatalf("didn't add to revoke loop") 1393 } 1394 1395 if err := client.RevokeTokens(context.Background(), vas, true); err != nil { 1396 t.Fatalf("RevokeTokens failed: %v", err) 1397 } 1398 1399 // Was committed 1400 if len(client.revoking) != 2 { 1401 t.Fatalf("didn't add to revoke loop") 1402 } 1403 1404 if client.stats().TrackedForRevoke != 2 { 1405 t.Fatalf("didn't add to revoke loop") 1406 } 1407 } 1408 1409 // TestVaultClient_RevokeTokens_Failures_TTL asserts that 1410 // the registered TTL doesn't get extended on retries 1411 func TestVaultClient_RevokeTokens_Failures_TTL(t *testing.T) { 1412 ci.Parallel(t) 1413 vconfig := &config.VaultConfig{ 1414 Enabled: pointer.Of(true), 1415 Token: uuid.Generate(), 1416 Addr: "http://127.0.0.1:0", 1417 } 1418 logger := testlog.HCLogger(t) 1419 client, err := NewVaultClient(vconfig, logger, nil, nil) 1420 if err != nil { 1421 t.Fatalf("failed to build vault client: %v", err) 1422 } 1423 client.SetActive(true) 1424 defer client.Stop() 1425 1426 // Create some VaultAccessors 1427 vas := []*structs.VaultAccessor{ 1428 mock.VaultAccessor(), 1429 mock.VaultAccessor(), 1430 } 1431 1432 err = client.RevokeTokens(context.Background(), vas, true) 1433 require.NoError(t, err) 1434 1435 // Was committed 1436 require.Len(t, client.revoking, 2) 1437 1438 // set TTL 1439 ttl := time.Now().Add(50 * time.Second) 1440 client.revoking[vas[0]] = ttl 1441 client.revoking[vas[1]] = ttl 1442 1443 // revoke again and ensure that TTL isn't extended 1444 err = client.RevokeTokens(context.Background(), vas, true) 1445 require.NoError(t, err) 1446 1447 require.Len(t, client.revoking, 2) 1448 expected := map[*structs.VaultAccessor]time.Time{ 1449 vas[0]: ttl, 1450 vas[1]: ttl, 1451 } 1452 require.Equal(t, expected, client.revoking) 1453 } 1454 1455 func TestVaultClient_RevokeTokens_Root(t *testing.T) { 1456 ci.Parallel(t) 1457 v := testutil.NewTestVault(t) 1458 defer v.Stop() 1459 1460 purged := 0 1461 purge := func(accessors []*structs.VaultAccessor) error { 1462 purged += len(accessors) 1463 return nil 1464 } 1465 1466 logger := testlog.HCLogger(t) 1467 client, err := NewVaultClient(v.Config, logger, purge, nil) 1468 if err != nil { 1469 t.Fatalf("failed to build vault client: %v", err) 1470 } 1471 client.SetActive(true) 1472 defer client.Stop() 1473 1474 waitForConnection(client, t) 1475 1476 // Create some vault tokens 1477 auth := v.Client.Auth().Token() 1478 req := vapi.TokenCreateRequest{ 1479 Policies: []string{"default"}, 1480 } 1481 t1, err := auth.Create(&req) 1482 if err != nil { 1483 t.Fatalf("Failed to create vault token: %v", err) 1484 } 1485 if t1 == nil || t1.Auth == nil { 1486 t.Fatalf("bad secret response: %+v", t1) 1487 } 1488 t2, err := auth.Create(&req) 1489 if err != nil { 1490 t.Fatalf("Failed to create vault token: %v", err) 1491 } 1492 if t2 == nil || t2.Auth == nil { 1493 t.Fatalf("bad secret response: %+v", t2) 1494 } 1495 1496 // Create two VaultAccessors 1497 vas := []*structs.VaultAccessor{ 1498 {Accessor: t1.Auth.Accessor}, 1499 {Accessor: t2.Auth.Accessor}, 1500 } 1501 1502 // Issue a token revocation 1503 if err := client.RevokeTokens(context.Background(), vas, true); err != nil { 1504 t.Fatalf("RevokeTokens failed: %v", err) 1505 } 1506 1507 // Lookup the token and make sure we get an error 1508 if s, err := auth.Lookup(t1.Auth.ClientToken); err == nil { 1509 t.Fatalf("Revoked token lookup didn't fail: %+v", s) 1510 } 1511 if s, err := auth.Lookup(t2.Auth.ClientToken); err == nil { 1512 t.Fatalf("Revoked token lookup didn't fail: %+v", s) 1513 } 1514 1515 if purged != 2 { 1516 t.Fatalf("Expected purged 2; got %d", purged) 1517 } 1518 } 1519 1520 func TestVaultClient_RevokeTokens_Role(t *testing.T) { 1521 ci.Parallel(t) 1522 v := testutil.NewTestVault(t) 1523 defer v.Stop() 1524 1525 // Set the configs token in a new test role 1526 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 1527 1528 purged := 0 1529 purge := func(accessors []*structs.VaultAccessor) error { 1530 purged += len(accessors) 1531 return nil 1532 } 1533 1534 logger := testlog.HCLogger(t) 1535 client, err := NewVaultClient(v.Config, logger, purge, nil) 1536 if err != nil { 1537 t.Fatalf("failed to build vault client: %v", err) 1538 } 1539 client.SetActive(true) 1540 defer client.Stop() 1541 1542 waitForConnection(client, t) 1543 1544 // Create some vault tokens 1545 auth := v.Client.Auth().Token() 1546 req := vapi.TokenCreateRequest{ 1547 Policies: []string{"default"}, 1548 } 1549 t1, err := auth.Create(&req) 1550 if err != nil { 1551 t.Fatalf("Failed to create vault token: %v", err) 1552 } 1553 if t1 == nil || t1.Auth == nil { 1554 t.Fatalf("bad secret response: %+v", t1) 1555 } 1556 t2, err := auth.Create(&req) 1557 if err != nil { 1558 t.Fatalf("Failed to create vault token: %v", err) 1559 } 1560 if t2 == nil || t2.Auth == nil { 1561 t.Fatalf("bad secret response: %+v", t2) 1562 } 1563 1564 // Create two VaultAccessors 1565 vas := []*structs.VaultAccessor{ 1566 {Accessor: t1.Auth.Accessor}, 1567 {Accessor: t2.Auth.Accessor}, 1568 } 1569 1570 // Issue a token revocation 1571 if err := client.RevokeTokens(context.Background(), vas, true); err != nil { 1572 t.Fatalf("RevokeTokens failed: %v", err) 1573 } 1574 1575 // Lookup the token and make sure we get an error 1576 if purged != 2 { 1577 t.Fatalf("Expected purged 2; got %d", purged) 1578 } 1579 if s, err := auth.Lookup(t1.Auth.ClientToken); err == nil { 1580 t.Fatalf("Revoked token lookup didn't fail: %+v", s) 1581 } 1582 if s, err := auth.Lookup(t2.Auth.ClientToken); err == nil { 1583 t.Fatalf("Revoked token lookup didn't fail: %+v", s) 1584 } 1585 } 1586 1587 // TestVaultClient_RevokeTokens_Idempotent asserts that token revocation 1588 // is idempotent, and can cope with cases if token was deleted out of band. 1589 func TestVaultClient_RevokeTokens_Idempotent(t *testing.T) { 1590 ci.Parallel(t) 1591 v := testutil.NewTestVault(t) 1592 defer v.Stop() 1593 1594 // Set the configs token in a new test role 1595 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 1596 1597 purged := map[string]struct{}{} 1598 purge := func(accessors []*structs.VaultAccessor) error { 1599 for _, accessor := range accessors { 1600 purged[accessor.Accessor] = struct{}{} 1601 } 1602 return nil 1603 } 1604 1605 logger := testlog.HCLogger(t) 1606 client, err := NewVaultClient(v.Config, logger, purge, nil) 1607 if err != nil { 1608 t.Fatalf("failed to build vault client: %v", err) 1609 } 1610 client.SetActive(true) 1611 defer client.Stop() 1612 1613 waitForConnection(client, t) 1614 1615 // Create some vault tokens 1616 auth := v.Client.Auth().Token() 1617 req := vapi.TokenCreateRequest{ 1618 Policies: []string{"default"}, 1619 } 1620 t1, err := auth.Create(&req) 1621 require.NoError(t, err) 1622 require.NotNil(t, t1) 1623 require.NotNil(t, t1.Auth) 1624 1625 t2, err := auth.Create(&req) 1626 require.NoError(t, err) 1627 require.NotNil(t, t2) 1628 require.NotNil(t, t2.Auth) 1629 1630 t3, err := auth.Create(&req) 1631 require.NoError(t, err) 1632 require.NotNil(t, t3) 1633 require.NotNil(t, t3.Auth) 1634 1635 // revoke t3 out of band 1636 err = auth.RevokeAccessor(t3.Auth.Accessor) 1637 require.NoError(t, err) 1638 1639 // Create two VaultAccessors 1640 vas := []*structs.VaultAccessor{ 1641 {Accessor: t1.Auth.Accessor}, 1642 {Accessor: t2.Auth.Accessor}, 1643 {Accessor: t3.Auth.Accessor}, 1644 } 1645 1646 // Issue a token revocation 1647 err = client.RevokeTokens(context.Background(), vas, true) 1648 require.NoError(t, err) 1649 require.Empty(t, client.revoking) 1650 1651 // revoke token again 1652 err = client.RevokeTokens(context.Background(), vas, true) 1653 require.NoError(t, err) 1654 require.Empty(t, client.revoking) 1655 1656 // Lookup the token and make sure we get an error 1657 require.Len(t, purged, 3) 1658 require.Contains(t, purged, t1.Auth.Accessor) 1659 require.Contains(t, purged, t2.Auth.Accessor) 1660 require.Contains(t, purged, t3.Auth.Accessor) 1661 s, err := auth.Lookup(t1.Auth.ClientToken) 1662 require.Errorf(t, err, "failed to purge token: %v", s) 1663 s, err = auth.Lookup(t2.Auth.ClientToken) 1664 require.Errorf(t, err, "failed to purge token: %v", s) 1665 } 1666 1667 // TestVaultClient_RevokeDaemon_Bounded asserts that token revocation 1668 // batches are bounded in size. 1669 func TestVaultClient_RevokeDaemon_Bounded(t *testing.T) { 1670 ci.Parallel(t) 1671 v := testutil.NewTestVault(t) 1672 defer v.Stop() 1673 1674 // Set the configs token in a new test role 1675 v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) 1676 1677 // Disable client until we can change settings for testing 1678 conf := v.Config.Copy() 1679 conf.Enabled = pointer.Of(false) 1680 1681 const ( 1682 batchSize = 100 1683 batches = 3 1684 ) 1685 resultCh := make(chan error, batches) 1686 var totalPurges int64 1687 1688 // Purge function asserts batches are always < batchSize 1689 purge := func(vas []*structs.VaultAccessor) error { 1690 if len(vas) > batchSize { 1691 resultCh <- fmt.Errorf("too many Vault accessors in batch: %d > %d", len(vas), batchSize) 1692 } else { 1693 resultCh <- nil 1694 } 1695 atomic.AddInt64(&totalPurges, int64(len(vas))) 1696 1697 return nil 1698 } 1699 1700 logger := testlog.HCLogger(t) 1701 client, err := NewVaultClient(conf, logger, purge, nil) 1702 require.NoError(t, err) 1703 1704 // Override settings for testing and then enable client 1705 client.maxRevokeBatchSize = batchSize 1706 client.revocationIntv = 3 * time.Millisecond 1707 conf = v.Config.Copy() 1708 conf.Enabled = pointer.Of(true) 1709 require.NoError(t, client.SetConfig(conf)) 1710 1711 client.SetActive(true) 1712 defer client.Stop() 1713 1714 waitForConnection(client, t) 1715 1716 // Create more tokens in Nomad than can fit in a batch; they don't need 1717 // to exist in Vault. 1718 accessors := make([]*structs.VaultAccessor, batchSize*batches) 1719 for i := 0; i < len(accessors); i++ { 1720 accessors[i] = &structs.VaultAccessor{Accessor: "abcd"} 1721 } 1722 1723 // Mark for revocation 1724 require.NoError(t, client.MarkForRevocation(accessors)) 1725 1726 // Wait for tokens to be revoked 1727 for i := 0; i < batches; i++ { 1728 select { 1729 case err := <-resultCh: 1730 require.NoError(t, err) 1731 case <-time.After(10 * time.Second): 1732 // 10 seconds should be plenty long to process 3 1733 // batches at a 3ms tick interval! 1734 t.Errorf("timed out processing %d batches. %d/%d complete in 10s", 1735 batches, i, batches) 1736 } 1737 } 1738 1739 require.Equal(t, int64(len(accessors)), atomic.LoadInt64(&totalPurges)) 1740 } 1741 1742 func waitForConnection(v *vaultClient, t *testing.T) { 1743 testutil.WaitForResult(func() (bool, error) { 1744 return v.ConnectionEstablished() 1745 }, func(err error) { 1746 t.Fatalf("Connection not established") 1747 }) 1748 } 1749 1750 func TestVaultClient_nextBackoff(t *testing.T) { 1751 ci.Parallel(t) 1752 1753 simpleCases := []struct { 1754 name string 1755 initBackoff float64 1756 1757 // define range of acceptable backoff values accounting for random factor 1758 rangeMin float64 1759 rangeMax float64 1760 }{ 1761 {"simple case", 7.0, 8.7, 17.60}, 1762 {"too low", 2.0, 5.0, 10.0}, 1763 {"too large", 100, 30.0, 60.0}, 1764 } 1765 1766 for _, c := range simpleCases { 1767 t.Run(c.name, func(t *testing.T) { 1768 b := nextBackoff(c.initBackoff, time.Now().Add(10*time.Hour)) 1769 if !(c.rangeMin <= b && b <= c.rangeMax) { 1770 t.Fatalf("Expected backoff within [%v, %v] but found %v", c.rangeMin, c.rangeMax, b) 1771 } 1772 }) 1773 } 1774 1775 // some edge cases 1776 t.Run("close to expiry", func(t *testing.T) { 1777 b := nextBackoff(20, time.Now().Add(1100*time.Millisecond)) 1778 if b != 5.0 { 1779 t.Fatalf("Expected backoff is 5 but found %v", b) 1780 } 1781 }) 1782 1783 t.Run("past expiry", func(t *testing.T) { 1784 b := nextBackoff(20, time.Now().Add(-1100*time.Millisecond)) 1785 if !(60 <= b && b <= 120) { 1786 t.Fatalf("Expected backoff within [%v, %v] but found %v", 60, 120, b) 1787 } 1788 }) 1789 } 1790 1791 func testRateLimit(t *testing.T, count int, client *vaultClient, fn func(context.Context) error) { 1792 // Spin up many requests. These should block 1793 ctx, cancel := context.WithCancel(context.Background()) 1794 defer cancel() 1795 1796 cancels := 0 1797 unblock := make(chan struct{}) 1798 for i := 0; i < count; i++ { 1799 go func() { 1800 err := fn(ctx) 1801 if err != nil { 1802 if err == context.Canceled { 1803 cancels += 1 1804 return 1805 } 1806 t.Errorf("request failed: %v", err) 1807 return 1808 } 1809 1810 // Cancel the context 1811 close(unblock) 1812 }() 1813 } 1814 1815 select { 1816 case <-time.After(5 * time.Second): 1817 t.Fatalf("timeout") 1818 case <-unblock: 1819 cancel() 1820 } 1821 1822 desired := count - 1 1823 testutil.WaitForResult(func() (bool, error) { 1824 if desired-cancels > 2 { 1825 return false, fmt.Errorf("Incorrect number of cancels; got %d; want %d", cancels, desired) 1826 } 1827 1828 return true, nil 1829 }, func(err error) { 1830 t.Fatal(err) 1831 }) 1832 }