github.com/clly/consul@v1.4.5/agent/consul/connect_ca_endpoint_test.go (about) 1 package consul 2 3 import ( 4 "crypto/x509" 5 "encoding/pem" 6 "fmt" 7 "os" 8 "sync" 9 "testing" 10 "time" 11 12 "github.com/stretchr/testify/require" 13 14 "github.com/hashicorp/consul/agent/connect" 15 ca "github.com/hashicorp/consul/agent/connect/ca" 16 "github.com/hashicorp/consul/agent/structs" 17 "github.com/hashicorp/consul/testrpc" 18 "github.com/hashicorp/consul/testutil/retry" 19 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 20 "github.com/stretchr/testify/assert" 21 ) 22 23 func testParseCert(t *testing.T, pemValue string) *x509.Certificate { 24 cert, err := connect.ParseCert(pemValue) 25 if err != nil { 26 t.Fatal(err) 27 } 28 return cert 29 } 30 31 // Test listing root CAs. 32 func TestConnectCARoots(t *testing.T) { 33 t.Parallel() 34 35 assert := assert.New(t) 36 require := require.New(t) 37 dir1, s1 := testServer(t) 38 defer os.RemoveAll(dir1) 39 defer s1.Shutdown() 40 codec := rpcClient(t, s1) 41 defer codec.Close() 42 43 testrpc.WaitForTestAgent(t, s1.RPC, "dc1") 44 45 // Insert some CAs 46 state := s1.fsm.State() 47 ca1 := connect.TestCA(t, nil) 48 ca2 := connect.TestCA(t, nil) 49 ca2.Active = false 50 idx, _, err := state.CARoots(nil) 51 require.NoError(err) 52 ok, err := state.CARootSetCAS(idx, idx, []*structs.CARoot{ca1, ca2}) 53 assert.True(ok) 54 require.NoError(err) 55 _, caCfg, err := state.CAConfig() 56 require.NoError(err) 57 58 // Request 59 args := &structs.DCSpecificRequest{ 60 Datacenter: "dc1", 61 } 62 var reply structs.IndexedCARoots 63 require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", args, &reply)) 64 65 // Verify 66 assert.Equal(ca1.ID, reply.ActiveRootID) 67 assert.Len(reply.Roots, 2) 68 for _, r := range reply.Roots { 69 // These must never be set, for security 70 assert.Equal("", r.SigningCert) 71 assert.Equal("", r.SigningKey) 72 } 73 assert.Equal(fmt.Sprintf("%s.consul", caCfg.ClusterID), reply.TrustDomain) 74 } 75 76 func TestConnectCAConfig_GetSet(t *testing.T) { 77 t.Parallel() 78 79 assert := assert.New(t) 80 dir1, s1 := testServer(t) 81 defer os.RemoveAll(dir1) 82 defer s1.Shutdown() 83 codec := rpcClient(t, s1) 84 defer codec.Close() 85 86 testrpc.WaitForTestAgent(t, s1.RPC, "dc1") 87 88 // Get the starting config 89 { 90 args := &structs.DCSpecificRequest{ 91 Datacenter: "dc1", 92 } 93 var reply structs.CAConfiguration 94 assert.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationGet", args, &reply)) 95 96 actual, err := ca.ParseConsulCAConfig(reply.Config) 97 assert.NoError(err) 98 expected, err := ca.ParseConsulCAConfig(s1.config.CAConfig.Config) 99 assert.NoError(err) 100 assert.Equal(reply.Provider, s1.config.CAConfig.Provider) 101 assert.Equal(actual, expected) 102 } 103 104 // Update a config value 105 newConfig := &structs.CAConfiguration{ 106 Provider: "consul", 107 Config: map[string]interface{}{ 108 "PrivateKey": "", 109 "RootCert": "", 110 "RotationPeriod": 180 * 24 * time.Hour, 111 }, 112 } 113 { 114 args := &structs.CARequest{ 115 Datacenter: "dc1", 116 Config: newConfig, 117 } 118 var reply interface{} 119 retry.Run(t, func(r *retry.R) { 120 r.Check(msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply)) 121 }) 122 } 123 124 // Verify the new config was set 125 { 126 args := &structs.DCSpecificRequest{ 127 Datacenter: "dc1", 128 } 129 var reply structs.CAConfiguration 130 assert.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationGet", args, &reply)) 131 132 actual, err := ca.ParseConsulCAConfig(reply.Config) 133 assert.NoError(err) 134 expected, err := ca.ParseConsulCAConfig(newConfig.Config) 135 assert.NoError(err) 136 assert.Equal(reply.Provider, newConfig.Provider) 137 assert.Equal(actual, expected) 138 } 139 } 140 141 func TestConnectCAConfig_TriggerRotation(t *testing.T) { 142 t.Parallel() 143 144 assert := assert.New(t) 145 require := require.New(t) 146 dir1, s1 := testServer(t) 147 defer os.RemoveAll(dir1) 148 defer s1.Shutdown() 149 codec := rpcClient(t, s1) 150 defer codec.Close() 151 152 testrpc.WaitForTestAgent(t, s1.RPC, "dc1") 153 154 // Store the current root 155 rootReq := &structs.DCSpecificRequest{ 156 Datacenter: "dc1", 157 } 158 var rootList structs.IndexedCARoots 159 require.Nil(msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", rootReq, &rootList)) 160 assert.Len(rootList.Roots, 1) 161 oldRoot := rootList.Roots[0] 162 163 // Update the provider config to use a new private key, which should 164 // cause a rotation. 165 _, newKey, err := connect.GeneratePrivateKey() 166 assert.NoError(err) 167 newConfig := &structs.CAConfiguration{ 168 Provider: "consul", 169 Config: map[string]interface{}{ 170 "PrivateKey": newKey, 171 "RootCert": "", 172 "RotationPeriod": 90 * 24 * time.Hour, 173 }, 174 } 175 { 176 args := &structs.CARequest{ 177 Datacenter: "dc1", 178 Config: newConfig, 179 } 180 var reply interface{} 181 182 require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationSet", args, &reply)) 183 } 184 185 // Make sure the new root has been added along with an intermediate 186 // cross-signed by the old root. 187 var newRootPEM string 188 { 189 args := &structs.DCSpecificRequest{ 190 Datacenter: "dc1", 191 } 192 var reply structs.IndexedCARoots 193 require.Nil(msgpackrpc.CallWithCodec(codec, "ConnectCA.Roots", args, &reply)) 194 assert.Len(reply.Roots, 2) 195 196 for _, r := range reply.Roots { 197 if r.ID == oldRoot.ID { 198 // The old root should no longer be marked as the active root, 199 // and none of its other fields should have changed. 200 assert.False(r.Active) 201 assert.Equal(r.Name, oldRoot.Name) 202 assert.Equal(r.RootCert, oldRoot.RootCert) 203 assert.Equal(r.SigningCert, oldRoot.SigningCert) 204 assert.Equal(r.IntermediateCerts, oldRoot.IntermediateCerts) 205 } else { 206 newRootPEM = r.RootCert 207 // The new root should have a valid cross-signed cert from the old 208 // root as an intermediate. 209 assert.True(r.Active) 210 assert.Len(r.IntermediateCerts, 1) 211 212 xc := testParseCert(t, r.IntermediateCerts[0]) 213 oldRootCert := testParseCert(t, oldRoot.RootCert) 214 newRootCert := testParseCert(t, r.RootCert) 215 216 // Should have the authority key ID and signature algo of the 217 // (old) signing CA. 218 assert.Equal(xc.AuthorityKeyId, oldRootCert.AuthorityKeyId) 219 assert.NotEqual(xc.SubjectKeyId, oldRootCert.SubjectKeyId) 220 assert.Equal(xc.SignatureAlgorithm, oldRootCert.SignatureAlgorithm) 221 222 // The common name and SAN should not have changed. 223 assert.Equal(xc.Subject.CommonName, newRootCert.Subject.CommonName) 224 assert.Equal(xc.URIs, newRootCert.URIs) 225 } 226 } 227 } 228 229 // Verify the new config was set. 230 { 231 args := &structs.DCSpecificRequest{ 232 Datacenter: "dc1", 233 } 234 var reply structs.CAConfiguration 235 require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.ConfigurationGet", args, &reply)) 236 237 actual, err := ca.ParseConsulCAConfig(reply.Config) 238 require.NoError(err) 239 expected, err := ca.ParseConsulCAConfig(newConfig.Config) 240 require.NoError(err) 241 assert.Equal(reply.Provider, newConfig.Provider) 242 assert.Equal(actual, expected) 243 } 244 245 // Verify that new leaf certs get the cross-signed intermediate bundled 246 { 247 // Generate a CSR and request signing 248 spiffeId := connect.TestSpiffeIDService(t, "web") 249 csr, _ := connect.TestCSR(t, spiffeId) 250 args := &structs.CASignRequest{ 251 Datacenter: "dc1", 252 CSR: csr, 253 } 254 var reply structs.IssuedCert 255 require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", args, &reply)) 256 257 // Verify that the cert is signed by the new CA 258 { 259 roots := x509.NewCertPool() 260 require.True(roots.AppendCertsFromPEM([]byte(newRootPEM))) 261 leaf, err := connect.ParseCert(reply.CertPEM) 262 require.NoError(err) 263 _, err = leaf.Verify(x509.VerifyOptions{ 264 Roots: roots, 265 }) 266 require.NoError(err) 267 } 268 269 // And that it validates via the intermediate 270 { 271 roots := x509.NewCertPool() 272 assert.True(roots.AppendCertsFromPEM([]byte(oldRoot.RootCert))) 273 leaf, err := connect.ParseCert(reply.CertPEM) 274 require.NoError(err) 275 276 // Make sure the intermediate was returned as well as leaf 277 _, rest := pem.Decode([]byte(reply.CertPEM)) 278 require.NotEmpty(rest) 279 280 intermediates := x509.NewCertPool() 281 require.True(intermediates.AppendCertsFromPEM(rest)) 282 283 _, err = leaf.Verify(x509.VerifyOptions{ 284 Roots: roots, 285 Intermediates: intermediates, 286 }) 287 require.NoError(err) 288 } 289 290 // Verify other fields 291 assert.Equal("web", reply.Service) 292 assert.Equal(spiffeId.URI().String(), reply.ServiceURI) 293 } 294 } 295 296 // Test CA signing 297 func TestConnectCASign(t *testing.T) { 298 t.Parallel() 299 300 assert := assert.New(t) 301 require := require.New(t) 302 dir1, s1 := testServer(t) 303 defer os.RemoveAll(dir1) 304 defer s1.Shutdown() 305 codec := rpcClient(t, s1) 306 defer codec.Close() 307 308 testrpc.WaitForLeader(t, s1.RPC, "dc1") 309 310 // Generate a CSR and request signing 311 spiffeId := connect.TestSpiffeIDService(t, "web") 312 csr, _ := connect.TestCSR(t, spiffeId) 313 args := &structs.CASignRequest{ 314 Datacenter: "dc1", 315 CSR: csr, 316 } 317 var reply structs.IssuedCert 318 require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", args, &reply)) 319 320 // Generate a second CSR and request signing 321 spiffeId2 := connect.TestSpiffeIDService(t, "web2") 322 csr, _ = connect.TestCSR(t, spiffeId2) 323 args = &structs.CASignRequest{ 324 Datacenter: "dc1", 325 CSR: csr, 326 } 327 328 var reply2 structs.IssuedCert 329 require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", args, &reply2)) 330 require.True(reply2.ModifyIndex > reply.ModifyIndex) 331 332 // Get the current CA 333 state := s1.fsm.State() 334 _, ca, err := state.CARootActive(nil) 335 require.NoError(err) 336 337 // Verify that the cert is signed by the CA 338 roots := x509.NewCertPool() 339 assert.True(roots.AppendCertsFromPEM([]byte(ca.RootCert))) 340 leaf, err := connect.ParseCert(reply.CertPEM) 341 require.NoError(err) 342 _, err = leaf.Verify(x509.VerifyOptions{ 343 Roots: roots, 344 }) 345 require.NoError(err) 346 347 // Verify other fields 348 assert.Equal("web", reply.Service) 349 assert.Equal(spiffeId.URI().String(), reply.ServiceURI) 350 } 351 352 // Bench how long Signing RPC takes. This was used to ballpark reasonable 353 // default rate limit to protect servers from thundering herds of signing 354 // requests on root rotation. 355 func BenchmarkConnectCASign(b *testing.B) { 356 t := &testing.T{} 357 358 require := require.New(b) 359 dir1, s1 := testServer(t) 360 defer os.RemoveAll(dir1) 361 defer s1.Shutdown() 362 codec := rpcClient(t, s1) 363 defer codec.Close() 364 365 testrpc.WaitForLeader(t, s1.RPC, "dc1") 366 367 // Generate a CSR and request signing 368 spiffeID := connect.TestSpiffeIDService(b, "web") 369 csr, _ := connect.TestCSR(b, spiffeID) 370 args := &structs.CASignRequest{ 371 Datacenter: "dc1", 372 CSR: csr, 373 } 374 var reply structs.IssuedCert 375 376 b.ResetTimer() 377 for n := 0; n < b.N; n++ { 378 require.NoError(msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", args, &reply)) 379 } 380 } 381 382 func TestConnectCASign_rateLimit(t *testing.T) { 383 t.Parallel() 384 385 require := require.New(t) 386 dir1, s1 := testServerWithConfig(t, func(c *Config) { 387 c.Datacenter = "dc1" 388 c.Bootstrap = true 389 c.CAConfig.Config = map[string]interface{}{ 390 // It actually doesn't work as expected with some higher values because 391 // the token bucket is initialized with max(10%, 1) burst which for small 392 // values is 1 and then the test completes so fast it doesn't actually 393 // replenish any tokens so you only get the burst allowed through. This is 394 // OK, running the test slower is likely to be more brittle anyway since 395 // it will become more timing dependent whether the actual rate the 396 // requests are made matches the expectation from the sleeps etc. 397 "CSRMaxPerSecond": 1, 398 } 399 }) 400 defer os.RemoveAll(dir1) 401 defer s1.Shutdown() 402 codec := rpcClient(t, s1) 403 defer codec.Close() 404 405 testrpc.WaitForLeader(t, s1.RPC, "dc1") 406 407 // Generate a CSR and request signing a few times in a loop. 408 spiffeID := connect.TestSpiffeIDService(t, "web") 409 csr, _ := connect.TestCSR(t, spiffeID) 410 args := &structs.CASignRequest{ 411 Datacenter: "dc1", 412 CSR: csr, 413 } 414 var reply structs.IssuedCert 415 416 errs := make([]error, 10) 417 for i := 0; i < len(errs); i++ { 418 errs[i] = msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", args, &reply) 419 } 420 421 limitedCount := 0 422 successCount := 0 423 for _, err := range errs { 424 if err == nil { 425 successCount++ 426 } else if err.Error() == ErrRateLimited.Error() { 427 limitedCount++ 428 } else { 429 require.NoError(err) 430 } 431 } 432 // I've only ever seen this as 1/9 however if the test runs slowly on an 433 // over-subscribed CPU (e.g. in CI) it's possible that later requests could 434 // have had their token replenished and succeed so we allow a little slack - 435 // the test here isn't really the exact token bucket response more a sanity 436 // check that some limiting is being applied. Note that we can't just measure 437 // the time it took to send them all and infer how many should have succeeded 438 // without some complex modeling of the token bucket algorithm. 439 require.Truef(successCount >= 1, "at least 1 CSRs should have succeeded, got %d", successCount) 440 require.Truef(limitedCount >= 7, "at least 7 CSRs should have been rate limited, got %d", limitedCount) 441 } 442 443 func TestConnectCASign_concurrencyLimit(t *testing.T) { 444 t.Parallel() 445 446 require := require.New(t) 447 dir1, s1 := testServerWithConfig(t, func(c *Config) { 448 c.Datacenter = "dc1" 449 c.Bootstrap = true 450 c.CAConfig.Config = map[string]interface{}{ 451 // Must disable the rate limit since it takes precedence 452 "CSRMaxPerSecond": 0, 453 "CSRMaxConcurrent": 1, 454 } 455 }) 456 defer os.RemoveAll(dir1) 457 defer s1.Shutdown() 458 459 testrpc.WaitForLeader(t, s1.RPC, "dc1") 460 461 // Generate a CSR and request signing a few times in a loop. 462 spiffeID := connect.TestSpiffeIDService(t, "web") 463 csr, _ := connect.TestCSR(t, spiffeID) 464 args := &structs.CASignRequest{ 465 Datacenter: "dc1", 466 CSR: csr, 467 } 468 469 var wg sync.WaitGroup 470 471 errs := make(chan error, 10) 472 times := make(chan time.Duration, cap(errs)) 473 start := time.Now() 474 for i := 0; i < cap(errs); i++ { 475 wg.Add(1) 476 go func() { 477 defer wg.Done() 478 codec := rpcClient(t, s1) 479 defer codec.Close() 480 var reply structs.IssuedCert 481 errs <- msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", args, &reply) 482 times <- time.Since(start) 483 }() 484 } 485 486 wg.Wait() 487 close(errs) 488 489 limitedCount := 0 490 successCount := 0 491 var minTime, maxTime time.Duration 492 for err := range errs { 493 elapsed := <-times 494 if elapsed < minTime || minTime == 0 { 495 minTime = elapsed 496 } 497 if elapsed > maxTime { 498 maxTime = elapsed 499 } 500 if err == nil { 501 successCount++ 502 } else if err.Error() == ErrRateLimited.Error() { 503 limitedCount++ 504 } else { 505 require.NoError(err) 506 } 507 } 508 509 // These are very hand wavy - on my mac times look like this: 510 // 2.776009ms 511 // 3.705813ms 512 // 4.527212ms 513 // 5.267755ms 514 // 6.119809ms 515 // 6.958083ms 516 // 7.869179ms 517 // 8.675058ms 518 // 9.512281ms 519 // 10.238183ms 520 // 521 // But it's indistinguishable from noise - even if you disable the concurrency 522 // limiter you get pretty much the same pattern/spread. 523 // 524 // On the other hand it's only timing that stops us from not hitting the 500ms 525 // timeout. On highly CPU constrained CI box this could be brittle if we 526 // assert that we never get rate limited. 527 // 528 // So this test is not super strong - but it's a sanity check at least that 529 // things don't break when configured this way, and through manual 530 // inspection/debug logging etc. we can verify it's actually doing the 531 // concurrency limit thing. If you add a 100ms sleep into the sign endpoint 532 // after the rate limit code for example it makes it much more obvious: 533 // 534 // With 100ms sleep an no concurrency limit: 535 // min=109ms, max=118ms 536 // With concurrency limit of 1: 537 // min=106ms, max=538ms (with ~half hitting the 500ms timeout) 538 // 539 // Without instrumenting the endpoint to make the RPC take an artificially 540 // long time it's hard to know what else we can do to actively detect that the 541 // requests were serialized. 542 t.Logf("min=%s, max=%s", minTime, maxTime) 543 //t.Fail() // Uncomment to see the time spread logged 544 require.Truef(successCount >= 1, "at least 1 CSRs should have succeeded, got %d", successCount) 545 } 546 547 func TestConnectCASignValidation(t *testing.T) { 548 t.Parallel() 549 550 dir1, s1 := testServerWithConfig(t, func(c *Config) { 551 c.ACLDatacenter = "dc1" 552 c.ACLsEnabled = true 553 c.ACLMasterToken = "root" 554 c.ACLDefaultPolicy = "deny" 555 }) 556 defer os.RemoveAll(dir1) 557 defer s1.Shutdown() 558 codec := rpcClient(t, s1) 559 defer codec.Close() 560 561 testrpc.WaitForLeader(t, s1.RPC, "dc1") 562 563 // Create an ACL token with service:write for web* 564 var webToken string 565 { 566 arg := structs.ACLRequest{ 567 Datacenter: "dc1", 568 Op: structs.ACLSet, 569 ACL: structs.ACL{ 570 Name: "User token", 571 Type: structs.ACLTokenTypeClient, 572 Rules: ` 573 service "web" { 574 policy = "write" 575 }`, 576 }, 577 WriteRequest: structs.WriteRequest{Token: "root"}, 578 } 579 require.NoError(t, msgpackrpc.CallWithCodec(codec, "ACL.Apply", &arg, &webToken)) 580 } 581 582 testWebID := connect.TestSpiffeIDService(t, "web") 583 584 tests := []struct { 585 name string 586 id connect.CertURI 587 wantErr string 588 }{ 589 { 590 name: "different cluster", 591 id: &connect.SpiffeIDService{ 592 Host: "55555555-4444-3333-2222-111111111111.consul", 593 Namespace: testWebID.Namespace, 594 Datacenter: testWebID.Datacenter, 595 Service: testWebID.Service, 596 }, 597 wantErr: "different trust domain", 598 }, 599 { 600 name: "same cluster should validate", 601 id: testWebID, 602 wantErr: "", 603 }, 604 { 605 name: "same cluster, CSR for a different DC should NOT validate", 606 id: &connect.SpiffeIDService{ 607 Host: testWebID.Host, 608 Namespace: testWebID.Namespace, 609 Datacenter: "dc2", 610 Service: testWebID.Service, 611 }, 612 wantErr: "different datacenter", 613 }, 614 { 615 name: "same cluster and DC, different service should not have perms", 616 id: &connect.SpiffeIDService{ 617 Host: testWebID.Host, 618 Namespace: testWebID.Namespace, 619 Datacenter: testWebID.Datacenter, 620 Service: "db", 621 }, 622 wantErr: "Permission denied", 623 }, 624 } 625 626 for _, tt := range tests { 627 t.Run(tt.name, func(t *testing.T) { 628 csr, _ := connect.TestCSR(t, tt.id) 629 args := &structs.CASignRequest{ 630 Datacenter: "dc1", 631 CSR: csr, 632 WriteRequest: structs.WriteRequest{Token: webToken}, 633 } 634 var reply structs.IssuedCert 635 err := msgpackrpc.CallWithCodec(codec, "ConnectCA.Sign", args, &reply) 636 if tt.wantErr == "" { 637 require.NoError(t, err) 638 // No other validation that is handled in different tests 639 } else { 640 require.Error(t, err) 641 require.Contains(t, err.Error(), tt.wantErr) 642 } 643 }) 644 } 645 }