google.golang.org/grpc@v1.74.2/internal/xds/bootstrap/tlscreds/bundle_ext_test.go

/*
 *
 * Copyright 2023 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package tlscreds_test

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"testing"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/internal/envconfig"
	"google.golang.org/grpc/internal/grpctest"
	"google.golang.org/grpc/internal/stubserver"
	"google.golang.org/grpc/internal/testutils"
	"google.golang.org/grpc/internal/xds/bootstrap/tlscreds"
	testgrpc "google.golang.org/grpc/interop/grpc_testing"
	testpb "google.golang.org/grpc/interop/grpc_testing"
	"google.golang.org/grpc/status"
	"google.golang.org/grpc/testdata"
)

const defaultTestTimeout = 5 * time.Second

type s struct {
	grpctest.Tester
}

func Test(t *testing.T) {
	grpctest.RunSubTests(t, s{})
}

type Closable interface {
	Close()
}

func (s) TestValidTlsBuilder(t *testing.T) {
	caCert := testdata.Path("x509/server_ca_cert.pem")
	clientCert := testdata.Path("x509/client1_cert.pem")
	clientKey := testdata.Path("x509/client1_key.pem")
	clientSpiffeBundle := testdata.Path("spiffe_end2end/client_spiffebundle.json")
	tests := []struct {
		name string
		jd   string
	}{
		{
			name: "Absent configuration",
			jd:   `null`,
		},
		{
			name: "Empty configuration",
			jd:   `{}`,
		},
		{
			name: "Only CA certificate chain",
			jd:   fmt.Sprintf(`{"ca_certificate_file": "%s"}`, caCert),
		},
		{
			name: "Only private key and certificate chain",
			jd:   fmt.Sprintf(`{"certificate_file":"%s","private_key_file":"%s"}`, clientCert, clientKey),
		},
		{
			name: "CA chain, private key and certificate chain",
			jd:   fmt.Sprintf(`{"ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s"}`, caCert, clientCert, clientKey),
		},
		{
			name: "Only refresh interval", jd: `{"refresh_interval": "1s"}`,
		},
		{
			name: "Refresh interval and CA certificate chain",
			jd:   fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file": "%s"}`, caCert),
		},
		{
			name: "Refresh interval, private key and certificate chain",
			jd:   fmt.Sprintf(`{"refresh_interval": "1s","certificate_file":"%s","private_key_file":"%s"}`, clientCert, clientKey),
		},
		{
			name: "Refresh interval, CA chain, private key and certificate chain",
			jd:   fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s"}`, caCert, clientCert, clientKey),
		},
		{
			name: "Refresh interval, CA chain, private key, certificate chain, spiffe bundle",
			jd:   fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s","spiffe_trust_bundle_map_file":"%s"}`, caCert, clientCert, clientKey, clientSpiffeBundle),
		},
		{
			name: "Unknown field",
			jd:   `{"unknown_field": "foo"}`,
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			msg := json.RawMessage(test.jd)
			_, stop, err := tlscreds.NewBundle(msg)
			if err != nil {
				t.Fatalf("NewBundle(%s) returned error %s when expected to succeed", test.jd, err)
			}
			stop()
		})
	}
}

func (s) TestInvalidTlsBuilder(t *testing.T) {
	tests := []struct {
		name, jd, wantErrPrefix string
	}{
		{
			name:          "Wrong type in json",
			jd:            `{"ca_certificate_file": 1}`,
			wantErrPrefix: "failed to unmarshal config:"},
		{
			name:          "Missing private key",
			jd:            fmt.Sprintf(`{"certificate_file":"%s"}`, testdata.Path("x509/server_cert.pem")),
			wantErrPrefix: "pemfile: private key file and identity cert file should be both specified or not specified",
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			msg := json.RawMessage(test.jd)
			_, stop, err := tlscreds.NewBundle(msg)
			if err == nil || !strings.HasPrefix(err.Error(), test.wantErrPrefix) {
				if stop != nil {
					stop()
				}
				t.Fatalf("NewBundle(%s): got error %s, want an error with prefix %s", msg, err, test.wantErrPrefix)
			}
		})
	}
}

func (s) TestCaReloading(t *testing.T) {
	serverCa, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
	if err != nil {
		t.Fatalf("Failed to read test CA cert: %s", err)
	}

	// Write CA certs to a temporary file so that we can modify it later.
	caPath := t.TempDir() + "/ca.pem"
	if err = os.WriteFile(caPath, serverCa, 0644); err != nil {
		t.Fatalf("Failed to write test CA cert: %v", err)
	}
	cfg := fmt.Sprintf(`{
		"ca_certificate_file": "%s",
		"refresh_interval": ".01s"
	}`, caPath)
	tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
	if err != nil {
		t.Fatalf("Failed to create TLS bundle: %v", err)
	}
	defer stop()

	serverCredentials := grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert))
	server := stubserver.StartTestService(t, nil, serverCredentials)

	conn, err := grpc.NewClient(
		server.Address,
		grpc.WithCredentialsBundle(tlsBundle),
		grpc.WithAuthority("x.test.example.com"),
	)
	if err != nil {
		t.Fatalf("Error dialing: %v", err)
	}
	defer conn.Close()

	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()

	client := testgrpc.NewTestServiceClient(conn)
	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
		t.Errorf("Error calling EmptyCall: %v", err)
	}
	// close the server and create a new one to force client to do a new
	// handshake.
	server.Stop()

	invalidCa, err := os.ReadFile(testdata.Path("ca.pem"))
	if err != nil {
		t.Fatalf("Failed to read test CA cert: %v", err)
	}
	// unload root cert
	err = os.WriteFile(caPath, invalidCa, 0644)
	if err != nil {
		t.Fatalf("Failed to write test CA cert: %v", err)
	}
	for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
		ss := stubserver.StubServer{
			Address:    server.Address,
			EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
		}
		server = stubserver.StartTestService(t, &ss, serverCredentials)

		// Client handshake should eventually fail because the client CA was
		// reloaded, and thus the server cert is signed by an unknown CA.
		t.Log(server)
		_, err = client.EmptyCall(ctx, &testpb.Empty{})
		const wantErr = "certificate signed by unknown authority"
		if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) {
			// Certs have reloaded.
			server.Stop()
			break
		}
		t.Logf("EmptyCall() got err: %s, want code: %s, want err: %s", err, codes.Unavailable, wantErr)
		server.Stop()
	}
	if ctx.Err() != nil {
		t.Errorf("Timed out waiting for CA certs reloading")
	}
}

// Test_SPIFFE_Reloading sets up a client and server. The client is configured
// to use a SPIFFE bundle map, and the server is configured to use TLS creds
// compatible with this bundle. A handshake is performed and connection is
// expected to be successful. Then we change the client's SPIFFE Bundle Map file
// on disk to one that should fail with the server's credentials. This change
// should be picked up by the client via our file reloading. Another handshake
// is performed and checked for failure, ensuring that gRPC is correctly using
// the changed-on-disk bundle map.
func (s) Test_SPIFFE_Reloading(t *testing.T) {
	testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
	clientSPIFFEBundle, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json"))
	if err != nil {
		t.Fatalf("Failed to read test SPIFFE bundle: %v", err)
	}

	// Write CA certs to a temporary file so that we can modify it later.
	spiffePath := t.TempDir() + "/client_spiffe.json"
	if err = os.WriteFile(spiffePath, clientSPIFFEBundle, 0644); err != nil {
		t.Fatalf("Failed to write test SPIFFE Bundle %v: %v", clientSPIFFEBundle, err)
	}
	cfg := fmt.Sprintf(`{
		"spiffe_trust_bundle_map_file": "%s",
		"refresh_interval": ".01s"
	}`, spiffePath)
	tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
	if err != nil {
		t.Fatalf("Failed to create TLS bundle: %v", err)
	}
	defer stop()

	l, err := testutils.LocalTCPListener()
	if err != nil {
		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
	}
	lis := testutils.NewRestartableListener(l)
	defer lis.Close()
	ss := stubserver.StubServer{
		Listener:   lis,
		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
	}

	serverCredentials := grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.NoClientCert))
	server := stubserver.StartTestService(t, &ss, serverCredentials)

	defer server.Stop()

	conn, err := grpc.NewClient(
		server.Address,
		grpc.WithCredentialsBundle(tlsBundle),
		grpc.WithAuthority("x.test.example.com"),
	)
	if err != nil {
		t.Fatalf("grpc.NewClient(%q) failed: %v", server.Address, err)
	}
	defer conn.Close()

	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()

	client := testgrpc.NewTestServiceClient(conn)
	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
		t.Errorf("Error calling EmptyCall: %v", err)
	}

	// Setup the wrong bundle to be reloaded
	wrongBundle, err := os.ReadFile(testdata.Path("spiffe_end2end/server_spiffebundle.json"))
	if err != nil {
		t.Fatalf("Failed to read test spiffe bundle %v: %v", "spiffe_end2end/server_spiffebundle.json", err)
	}
	// Write the bundle that will fail to the tmp file path to be reloaded
	err = os.WriteFile(spiffePath, wrongBundle, 0644)
	if err != nil {
		t.Fatalf("Failed to write test spiffe bundle %v: %v", "spiffe_end2end/server_spiffebundle.json", err)
	}

	for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
		// Stop and restart the listener to force new handshakes
		lis.Stop()
		lis.Restart()
		// Client handshake should eventually fail because the client CA was
		// reloaded, and thus the server cert is signed by an unknown CA.
		t.Log(server)
		_, err = client.EmptyCall(ctx, &testpb.Empty{})
		const wantErr = "no bundle found for peer certificates trust domain"
		if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) {
			// Certs have reloaded.
			server.Stop()
			break
		}
		t.Logf("EmptyCall() got err: %s, want code: %s, want err: %s", err, codes.Unavailable, wantErr)
	}
	if ctx.Err() != nil {
		t.Errorf("Timed out waiting for CA certs reloading")
	}
}

func (s) TestMTLS(t *testing.T) {
	s := stubserver.StartTestService(t, nil, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)))
	defer s.Stop()

	cfg := fmt.Sprintf(`{
		"ca_certificate_file": "%s",
		"certificate_file": "%s",
		"private_key_file": "%s"
	}`,
		testdata.Path("x509/server_ca_cert.pem"),
		testdata.Path("x509/client1_cert.pem"),
		testdata.Path("x509/client1_key.pem"))
	tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
	if err != nil {
		t.Fatalf("Failed to create TLS bundle: %v", err)
	}
	defer stop()
	conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
	if err != nil {
		t.Fatalf("Error dialing: %v", err)
	}
	defer conn.Close()
	client := testgrpc.NewTestServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
		t.Errorf("EmptyCall(): got error %v when expected to succeed", err)
	}
}

// Test_MTLS_SPIFFE configures a client and server. The server has a certificate
// chain that is compatible with the client's configured SPIFFE bundle map. An
// MTLS connection is attempted between the two and checked for success.
func (s) Test_MTLS_SPIFFE(t *testing.T) {
	testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
	tests := []struct {
		name         string
		serverOption grpc.ServerOption
	}{
		{
			name:         "MTLS SPIFFE",
			serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)),
		},
		{
			name:         "MTLS SPIFFE Chain",
			serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFEChain(t, tls.RequireAndVerifyClientCert)),
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			s := stubserver.StartTestService(t, nil, tc.serverOption)
			defer s.Stop()

			cfg := fmt.Sprintf(`{
	"certificate_file": "%s",
	"private_key_file": "%s",
	"spiffe_trust_bundle_map_file": "%s"
}`,
				testdata.Path("spiffe_end2end/client_spiffe.pem"),
				testdata.Path("spiffe_end2end/client.key"),
				testdata.Path("spiffe_end2end/client_spiffebundle.json"))
			tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
			if err != nil {
				t.Fatalf("Failed to create TLS bundle: %v", err)
			}
			defer stop()
			conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
			if err != nil {
				t.Fatalf("Error dialing: %v", err)
			}
			defer conn.Close()
			client := testgrpc.NewTestServiceClient(conn)
			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
			defer cancel()
			if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
				t.Errorf("EmptyCall(): got error %v when expected to succeed", err)
			}
		})
	}
}

// Test_MTLS_SPIFFE_FlagDisabled configures a client and server. The server has
// a certificate chain that is compatible with the client's configured SPIFFE
// bundle map. However, the XDS flag that enabled SPIFFE usage is disabled. An
// MTLS connection is attempted between the two and checked for failure.
func (s) Test_MTLS_SPIFFE_FlagDisabled(t *testing.T) {
	testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, false)
	serverOption := grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert))
	s := stubserver.StartTestService(t, nil, serverOption)
	defer s.Stop()

	cfg := fmt.Sprintf(`{
"certificate_file": "%s",
"private_key_file": "%s",
"spiffe_trust_bundle_map_file": "%s"
}`,
		testdata.Path("spiffe_end2end/client_spiffe.pem"),
		testdata.Path("spiffe_end2end/client.key"),
		testdata.Path("spiffe_end2end/client_spiffebundle.json"))
	tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
	if err != nil {
		t.Fatalf("Failed to create TLS bundle: %v", err)
	}
	defer stop()
	conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
	if err != nil {
		t.Fatalf("Error dialing: %v", err)
	}
	defer conn.Close()
	client := testgrpc.NewTestServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil {
		t.Errorf("EmptyCall(): got success want failure")
	}
}

func (s) Test_MTLS_SPIFFE_Failure(t *testing.T) {
	testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
	tests := []struct {
		name             string
		certFile         string
		keyFile          string
		spiffeBundleFile string
		serverOption     grpc.ServerOption
		wantErrContains  string
		wantErrCode      codes.Code
	}{
		{
			name:             "No matching trust domain in bundle",
			certFile:         "spiffe_end2end/client_spiffe.pem",
			keyFile:          "spiffe_end2end/client.key",
			spiffeBundleFile: "spiffe_end2end/server_spiffebundle.json",
			serverOption:     grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)),
			wantErrContains:  "spiffe: no bundle found for peer certificates",
			wantErrCode:      codes.Unavailable,
		},
		{
			name:             "Server cert has no valid SPIFFE URIs",
			certFile:         "spiffe_end2end/client_spiffe.pem",
			keyFile:          "spiffe_end2end/client.key",
			spiffeBundleFile: "spiffe_end2end/client_spiffebundle.json",
			serverOption:     grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)),
			wantErrContains:  "spiffe: could not get spiffe ID from peer leaf cert",
			wantErrCode:      codes.Unavailable,
		},
		{
			name:             "Server cert has valid spiffe ID but doesn't chain to the root CA",
			certFile:         "spiffe_end2end/client_spiffe.pem",
			keyFile:          "spiffe_end2end/client.key",
			spiffeBundleFile: "spiffe_end2end/client_spiffebundle.json",
			serverOption:     grpc.Creds(testutils.CreateServerTLSCredentialsValidSPIFFEButWrongCA(t, tls.RequireAndVerifyClientCert)),
			wantErrContains:  "spiffe: x509 certificate Verify failed: x509: certificate signed by unknown authority",
			wantErrCode:      codes.Unavailable,
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			s := stubserver.StartTestService(t, nil, tc.serverOption)
			defer s.Stop()
			cfg := fmt.Sprintf(`{
"certificate_file": "%s",
"private_key_file": "%s",
"spiffe_trust_bundle_map_file": "%s"
}`,
				testdata.Path(tc.certFile),
				testdata.Path(tc.keyFile),
				testdata.Path(tc.spiffeBundleFile))
			tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
			if err != nil {
				t.Fatalf("Failed to create TLS bundle: %v", err)
			}
			defer stop()
			conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
			if err != nil {
				t.Fatalf("grpc.NewClient(%q) failed: %v", s.Address, err)
			}
			defer conn.Close()
			client := testgrpc.NewTestServiceClient(conn)
			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
			defer cancel()
			if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil {
				t.Errorf("EmptyCall(): got success. want failure")
			}
			if status.Code(err) != tc.wantErrCode {
				t.Errorf("EmptyCall(): failed with wrong error. got code %v. want code: %v", status.Code(err), tc.wantErrCode)
			}
			if !strings.Contains(err.Error(), tc.wantErrContains) {
				t.Errorf("EmptyCall(): failed with wrong error. got %v. want contains: %v", err, tc.wantErrContains)
			}
		})
	}
}