istio.io/istio@v0.0.0-20240520182934-d79c90f27776/pilot/pkg/bootstrap/server_test.go (about)

     1  // Copyright Istio Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  package bootstrap
    15  
    16  import (
    17  	"bytes"
    18  	"context"
    19  	"crypto/tls"
    20  	"fmt"
    21  	"net"
    22  	"net/http"
    23  	"os"
    24  	"path/filepath"
    25  	"testing"
    26  	"time"
    27  
    28  	. "github.com/onsi/gomega"
    29  	"golang.org/x/net/http2"
    30  	cert "k8s.io/api/certificates/v1"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  
    33  	"istio.io/istio/pilot/pkg/features"
    34  	"istio.io/istio/pilot/pkg/keycertbundle"
    35  	"istio.io/istio/pilot/pkg/server"
    36  	kubecontroller "istio.io/istio/pilot/pkg/serviceregistry/kube/controller"
    37  	"istio.io/istio/pkg/config/constants"
    38  	"istio.io/istio/pkg/filewatcher"
    39  	"istio.io/istio/pkg/kube"
    40  	"istio.io/istio/pkg/test"
    41  	"istio.io/istio/pkg/test/util/assert"
    42  	"istio.io/istio/pkg/test/util/retry"
    43  	"istio.io/istio/pkg/testcerts"
    44  	"istio.io/istio/security/pkg/pki/util"
    45  )
    46  
    47  func loadCertFilesAtPaths(t TLSFSLoadPaths) error {
    48  	// create cert directories if not existing
    49  	if err := os.MkdirAll(filepath.Dir(t.testTLSCertFilePath), os.ModePerm); err != nil {
    50  		return fmt.Errorf("Mkdirall(%v) failed: %v", t.testTLSCertFilePath, err)
    51  	}
    52  
    53  	if err := os.MkdirAll(filepath.Dir(t.testTLSKeyFilePath), os.ModePerm); err != nil {
    54  		return fmt.Errorf("Mkdirall(%v) failed: %v", t.testTLSKeyFilePath, err)
    55  	}
    56  
    57  	if err := os.MkdirAll(filepath.Dir(t.testCaCertFilePath), os.ModePerm); err != nil {
    58  		return fmt.Errorf("Mkdirall(%v) failed: %v", t.testCaCertFilePath, err)
    59  	}
    60  
    61  	// load key and cert files.
    62  	if err := os.WriteFile(t.testTLSCertFilePath, testcerts.ServerCert, 0o644); err != nil { // nolint: vetshadow
    63  		return fmt.Errorf("WriteFile(%v) failed: %v", t.testTLSCertFilePath, err)
    64  	}
    65  	if err := os.WriteFile(t.testTLSKeyFilePath, testcerts.ServerKey, 0o644); err != nil { // nolint: vetshadow
    66  		return fmt.Errorf("WriteFile(%v) failed: %v", t.testTLSKeyFilePath, err)
    67  	}
    68  	if err := os.WriteFile(t.testCaCertFilePath, testcerts.CACert, 0o644); err != nil { // nolint: vetshadow
    69  		return fmt.Errorf("WriteFile(%v) failed: %v", t.testCaCertFilePath, err)
    70  	}
    71  
    72  	return nil
    73  }
    74  
    75  func cleanupCertFileSystemFiles(t TLSFSLoadPaths) error {
    76  	if err := os.Remove(t.testTLSCertFilePath); err != nil {
    77  		return fmt.Errorf("Test cleanup failed, could not delete %s", t.testTLSCertFilePath)
    78  	}
    79  
    80  	if err := os.Remove(t.testTLSKeyFilePath); err != nil {
    81  		return fmt.Errorf("Test cleanup failed, could not delete %s", t.testTLSKeyFilePath)
    82  	}
    83  
    84  	if err := os.Remove(t.testCaCertFilePath); err != nil {
    85  		return fmt.Errorf("Test cleanup failed, could not delete %s", t.testCaCertFilePath)
    86  	}
    87  	return nil
    88  }
    89  
    90  // This struct will indicate for each test case
    91  // where tls assets will be loaded on disk
    92  type TLSFSLoadPaths struct {
    93  	testTLSCertFilePath string
    94  	testTLSKeyFilePath  string
    95  	testCaCertFilePath  string
    96  }
    97  
    98  func TestNewServerCertInit(t *testing.T) {
    99  	configDir := t.TempDir()
   100  
   101  	tlsArgCertsDir := t.TempDir()
   102  
   103  	tlsArgcertFile := filepath.Join(tlsArgCertsDir, "cert-file.pem")
   104  	tlsArgkeyFile := filepath.Join(tlsArgCertsDir, "key-file.pem")
   105  	tlsArgcaCertFile := filepath.Join(tlsArgCertsDir, "ca-cert.pem")
   106  
   107  	cases := []struct {
   108  		name                      string
   109  		FSCertsPaths              TLSFSLoadPaths
   110  		tlsOptions                *TLSOptions
   111  		enableCA                  bool
   112  		certProvider              string
   113  		expNewCert                bool
   114  		expCert                   []byte
   115  		expKey                    []byte
   116  		expSecureDiscoveryService bool
   117  	}{
   118  		{
   119  			name:         "Load from existing DNS cert",
   120  			FSCertsPaths: TLSFSLoadPaths{tlsArgcertFile, tlsArgkeyFile, tlsArgcaCertFile},
   121  			tlsOptions: &TLSOptions{
   122  				CertFile:   tlsArgcertFile,
   123  				KeyFile:    tlsArgkeyFile,
   124  				CaCertFile: tlsArgcaCertFile,
   125  			},
   126  			enableCA:                  false,
   127  			certProvider:              constants.CertProviderKubernetes,
   128  			expNewCert:                false,
   129  			expCert:                   testcerts.ServerCert,
   130  			expKey:                    testcerts.ServerKey,
   131  			expSecureDiscoveryService: true,
   132  		},
   133  		{
   134  			name:         "Create new DNS cert using Istiod",
   135  			FSCertsPaths: TLSFSLoadPaths{},
   136  			tlsOptions: &TLSOptions{
   137  				CertFile:   "",
   138  				KeyFile:    "",
   139  				CaCertFile: "",
   140  			},
   141  			enableCA:                  true,
   142  			certProvider:              constants.CertProviderIstiod,
   143  			expNewCert:                true,
   144  			expCert:                   []byte{},
   145  			expKey:                    []byte{},
   146  			expSecureDiscoveryService: true,
   147  		},
   148  		{
   149  			name:         "No DNS cert created because CA is disabled",
   150  			FSCertsPaths: TLSFSLoadPaths{},
   151  			tlsOptions:   &TLSOptions{},
   152  			enableCA:     false,
   153  			certProvider: constants.CertProviderIstiod,
   154  			expNewCert:   false,
   155  			expCert:      []byte{},
   156  			expKey:       []byte{},
   157  		},
   158  		{
   159  			name: "DNS cert loaded because it is in known even if CA is Disabled",
   160  			FSCertsPaths: TLSFSLoadPaths{
   161  				constants.DefaultPilotTLSCert,
   162  				constants.DefaultPilotTLSKey,
   163  				constants.DefaultPilotTLSCaCert,
   164  			},
   165  			tlsOptions:                &TLSOptions{},
   166  			enableCA:                  false,
   167  			certProvider:              constants.CertProviderNone,
   168  			expNewCert:                false,
   169  			expCert:                   testcerts.ServerCert,
   170  			expKey:                    testcerts.ServerKey,
   171  			expSecureDiscoveryService: true,
   172  		},
   173  		{
   174  			name: "DNS cert loaded from known location, even if CA is Disabled, with a fallback CA path",
   175  			FSCertsPaths: TLSFSLoadPaths{
   176  				constants.DefaultPilotTLSCert,
   177  				constants.DefaultPilotTLSKey,
   178  				constants.DefaultPilotTLSCaCertAlternatePath,
   179  			},
   180  			tlsOptions:                &TLSOptions{},
   181  			enableCA:                  false,
   182  			certProvider:              constants.CertProviderNone,
   183  			expNewCert:                false,
   184  			expCert:                   testcerts.ServerCert,
   185  			expKey:                    testcerts.ServerKey,
   186  			expSecureDiscoveryService: true,
   187  		},
   188  		{
   189  			name:         "No cert provider",
   190  			FSCertsPaths: TLSFSLoadPaths{},
   191  			tlsOptions:   &TLSOptions{},
   192  			enableCA:     true,
   193  			certProvider: constants.CertProviderNone,
   194  			expNewCert:   false,
   195  			expCert:      []byte{},
   196  			expKey:       []byte{},
   197  		},
   198  	}
   199  
   200  	for _, c := range cases {
   201  		t.Run(c.name, func(t *testing.T) {
   202  			test.SetForTest(t, &features.PilotCertProvider, c.certProvider)
   203  			test.SetForTest(t, &features.EnableCAServer, c.enableCA)
   204  
   205  			// check if we have some tls assets to write for test
   206  			if c.FSCertsPaths != (TLSFSLoadPaths{}) {
   207  				err := loadCertFilesAtPaths(c.FSCertsPaths)
   208  				if err != nil {
   209  					t.Fatal(err.Error())
   210  				}
   211  
   212  				defer cleanupCertFileSystemFiles(c.FSCertsPaths)
   213  			}
   214  
   215  			args := NewPilotArgs(func(p *PilotArgs) {
   216  				p.Namespace = "istio-system"
   217  				p.ServerOptions = DiscoveryServerOptions{
   218  					// Dynamically assign all ports.
   219  					HTTPAddr:       ":0",
   220  					MonitoringAddr: ":0",
   221  					GRPCAddr:       ":0",
   222  					SecureGRPCAddr: ":0",
   223  					TLSOptions:     *c.tlsOptions,
   224  				}
   225  				p.RegistryOptions = RegistryOptions{
   226  					FileDir: configDir,
   227  				}
   228  
   229  				p.ShutdownDuration = 1 * time.Millisecond
   230  			})
   231  			g := NewWithT(t)
   232  			s, err := NewServer(args, func(s *Server) {
   233  				s.kubeClient = kube.NewFakeClient()
   234  			})
   235  			g.Expect(err).To(Succeed())
   236  			stop := make(chan struct{})
   237  			g.Expect(s.Start(stop)).To(Succeed())
   238  			defer func() {
   239  				close(stop)
   240  				s.WaitUntilCompletion()
   241  			}()
   242  
   243  			if c.expNewCert {
   244  				if istiodCert, err := s.getIstiodCertificate(nil); istiodCert == nil || err != nil {
   245  					t.Errorf("Istiod failed to generate new DNS cert")
   246  				}
   247  			} else {
   248  				if len(c.expCert) != 0 {
   249  					if !checkCert(t, s, c.expCert, c.expKey) {
   250  						t.Errorf("Istiod certificate does not match the expectation")
   251  					}
   252  				} else {
   253  					if _, err := s.getIstiodCertificate(nil); err == nil {
   254  						t.Errorf("Istiod should not generate new DNS cert")
   255  					}
   256  				}
   257  			}
   258  
   259  			if c.expSecureDiscoveryService {
   260  				if s.secureGrpcServer == nil {
   261  					t.Errorf("Istiod secure grpc server was not started.")
   262  				}
   263  			}
   264  		})
   265  	}
   266  }
   267  
   268  func TestReloadIstiodCert(t *testing.T) {
   269  	dir := t.TempDir()
   270  	stop := make(chan struct{})
   271  	s := &Server{
   272  		fileWatcher:             filewatcher.NewWatcher(),
   273  		server:                  server.New(),
   274  		istiodCertBundleWatcher: keycertbundle.NewWatcher(),
   275  	}
   276  
   277  	defer func() {
   278  		close(stop)
   279  		_ = s.fileWatcher.Close()
   280  	}()
   281  
   282  	certFile := filepath.Join(dir, "cert-file.yaml")
   283  	keyFile := filepath.Join(dir, "key-file.yaml")
   284  	caFile := filepath.Join(dir, "ca-file.yaml")
   285  
   286  	// load key and cert files.
   287  	if err := os.WriteFile(certFile, testcerts.ServerCert, 0o644); err != nil { // nolint: vetshadow
   288  		t.Fatalf("WriteFile(%v) failed: %v", certFile, err)
   289  	}
   290  	if err := os.WriteFile(keyFile, testcerts.ServerKey, 0o644); err != nil { // nolint: vetshadow
   291  		t.Fatalf("WriteFile(%v) failed: %v", keyFile, err)
   292  	}
   293  
   294  	if err := os.WriteFile(caFile, testcerts.CACert, 0o644); err != nil { // nolint: vetshadow
   295  		t.Fatalf("WriteFile(%v) failed: %v", caFile, err)
   296  	}
   297  
   298  	tlsOptions := TLSOptions{
   299  		CertFile:   certFile,
   300  		KeyFile:    keyFile,
   301  		CaCertFile: caFile,
   302  	}
   303  
   304  	// setup cert watches.
   305  	if err := s.initCertificateWatches(tlsOptions); err != nil {
   306  		t.Fatalf("initCertificateWatches failed: %v", err)
   307  	}
   308  
   309  	if err := s.initIstiodCertLoader(); err != nil {
   310  		t.Fatalf("istiod unable to load its cert")
   311  	}
   312  
   313  	if err := s.server.Start(stop); err != nil {
   314  		t.Fatalf("Could not invoke startFuncs: %v", err)
   315  	}
   316  
   317  	// Validate that the certs are loaded.
   318  	if !checkCert(t, s, testcerts.ServerCert, testcerts.ServerKey) {
   319  		t.Errorf("Istiod certificate does not match the expectation")
   320  	}
   321  
   322  	// Update cert/key files.
   323  	if err := os.WriteFile(tlsOptions.CertFile, testcerts.RotatedCert, 0o644); err != nil { // nolint: vetshadow
   324  		t.Fatalf("WriteFile(%v) failed: %v", tlsOptions.CertFile, err)
   325  	}
   326  	if err := os.WriteFile(tlsOptions.KeyFile, testcerts.RotatedKey, 0o644); err != nil { // nolint: vetshadow
   327  		t.Fatalf("WriteFile(%v) failed: %v", tlsOptions.KeyFile, err)
   328  	}
   329  
   330  	g := NewWithT(t)
   331  
   332  	// Validate that istiod cert is updated.
   333  	g.Eventually(func() bool {
   334  		return checkCert(t, s, testcerts.RotatedCert, testcerts.RotatedKey)
   335  	}, "10s", "100ms").Should(BeTrue())
   336  }
   337  
   338  func TestNewServer(t *testing.T) {
   339  	// All of the settings to apply and verify. Currently just testing domain suffix,
   340  	// but we should expand this list.
   341  	cases := []struct {
   342  		name             string
   343  		domain           string
   344  		expectedDomain   string
   345  		enableSecureGRPC bool
   346  		jwtRule          string
   347  	}{
   348  		{
   349  			name:           "default domain",
   350  			domain:         "",
   351  			expectedDomain: constants.DefaultClusterLocalDomain,
   352  		},
   353  		{
   354  			name:           "default domain with JwtRule",
   355  			domain:         "",
   356  			expectedDomain: constants.DefaultClusterLocalDomain,
   357  			jwtRule:        `{"issuer": "foo", "jwks_uri": "baz", "audiences": ["aud1", "aud2"]}`,
   358  		},
   359  		{
   360  			name:           "override domain",
   361  			domain:         "mydomain.com",
   362  			expectedDomain: "mydomain.com",
   363  		},
   364  		{
   365  			name:             "override default secured grpc port",
   366  			domain:           "",
   367  			expectedDomain:   constants.DefaultClusterLocalDomain,
   368  			enableSecureGRPC: true,
   369  		},
   370  	}
   371  
   372  	for _, c := range cases {
   373  		t.Run(c.name, func(t *testing.T) {
   374  			configDir := t.TempDir()
   375  
   376  			secureGRPCPort := ""
   377  			if c.enableSecureGRPC {
   378  				secureGRPCPort = ":0"
   379  			}
   380  
   381  			args := NewPilotArgs(func(p *PilotArgs) {
   382  				p.Namespace = "istio-system"
   383  				p.ServerOptions = DiscoveryServerOptions{
   384  					// Dynamically assign all ports.
   385  					HTTPAddr:       ":0",
   386  					MonitoringAddr: ":0",
   387  					GRPCAddr:       ":0",
   388  					SecureGRPCAddr: secureGRPCPort,
   389  				}
   390  				p.RegistryOptions = RegistryOptions{
   391  					KubeOptions: kubecontroller.Options{
   392  						DomainSuffix: c.domain,
   393  					},
   394  					FileDir: configDir,
   395  				}
   396  
   397  				p.ShutdownDuration = 1 * time.Millisecond
   398  
   399  				p.JwtRule = c.jwtRule
   400  			})
   401  
   402  			g := NewWithT(t)
   403  			s, err := NewServer(args, func(s *Server) {
   404  				s.kubeClient = kube.NewFakeClient()
   405  			})
   406  			g.Expect(err).To(Succeed())
   407  			stop := make(chan struct{})
   408  			g.Expect(s.Start(stop)).To(Succeed())
   409  			defer func() {
   410  				close(stop)
   411  				s.WaitUntilCompletion()
   412  			}()
   413  
   414  			g.Expect(s.environment.DomainSuffix).To(Equal(c.expectedDomain))
   415  
   416  			assert.Equal(t, s.secureGrpcServer != nil, c.enableSecureGRPC)
   417  		})
   418  	}
   419  }
   420  
   421  func TestMultiplex(t *testing.T) {
   422  	configDir := t.TempDir()
   423  
   424  	var secureGRPCPort int
   425  	var err error
   426  
   427  	args := NewPilotArgs(func(p *PilotArgs) {
   428  		p.Namespace = "istio-system"
   429  		p.ServerOptions = DiscoveryServerOptions{
   430  			// Dynamically assign all ports.
   431  			HTTPAddr:       ":0",
   432  			MonitoringAddr: ":0",
   433  			GRPCAddr:       "",
   434  			SecureGRPCAddr: fmt.Sprintf(":%d", secureGRPCPort),
   435  		}
   436  		p.RegistryOptions = RegistryOptions{
   437  			FileDir: configDir,
   438  		}
   439  
   440  		p.ShutdownDuration = 1 * time.Millisecond
   441  	})
   442  
   443  	g := NewWithT(t)
   444  	s, err := NewServer(args, func(s *Server) {
   445  		s.kubeClient = kube.NewFakeClient()
   446  	})
   447  	g.Expect(err).To(Succeed())
   448  	stop := make(chan struct{})
   449  	g.Expect(s.Start(stop)).To(Succeed())
   450  	defer func() {
   451  		close(stop)
   452  		s.WaitUntilCompletion()
   453  	}()
   454  	t.Run("h1", func(t *testing.T) {
   455  		c := http.Client{}
   456  		defer c.CloseIdleConnections()
   457  		resp, err := c.Get("http://" + s.httpAddr + "/validate")
   458  		assert.NoError(t, err)
   459  		// Validate returns 400 on no body; if we got this the request works
   460  		assert.Equal(t, resp.StatusCode, 400)
   461  		resp.Body.Close()
   462  	})
   463  	t.Run("h2", func(t *testing.T) {
   464  		c := http.Client{
   465  			Transport: &http2.Transport{
   466  				// Golang doesn't have first class support for h2c, so we provide some workarounds
   467  				// See https://www.mailgun.com/blog/http-2-cleartext-h2c-client-example-go/
   468  				// So http2.Transport doesn't complain the URL scheme isn't 'https'
   469  				AllowHTTP: true,
   470  				// Pretend we are dialing a TLS endpoint. (Note, we ignore the passed tls.Config)
   471  				DialTLSContext: func(_ context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
   472  					return net.Dial(network, addr)
   473  				},
   474  			},
   475  		}
   476  		defer c.CloseIdleConnections()
   477  
   478  		resp, err := c.Get("http://" + s.httpAddr + "/validate")
   479  		assert.NoError(t, err)
   480  		// Validate returns 400 on no body; if we got this the request works
   481  		assert.Equal(t, resp.StatusCode, 400)
   482  		resp.Body.Close()
   483  	})
   484  }
   485  
   486  func TestIstiodCipherSuites(t *testing.T) {
   487  	cases := []struct {
   488  		name               string
   489  		serverCipherSuites []uint16
   490  		clientCipherSuites []uint16
   491  		expectSuccess      bool
   492  	}{
   493  		{
   494  			name:          "default cipher suites",
   495  			expectSuccess: true,
   496  		},
   497  		{
   498  			name:               "client and istiod cipher suites match",
   499  			serverCipherSuites: []uint16{tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
   500  			clientCipherSuites: []uint16{tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
   501  			expectSuccess:      true,
   502  		},
   503  		{
   504  			name:               "client and istiod cipher suites mismatch",
   505  			serverCipherSuites: []uint16{tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
   506  			clientCipherSuites: []uint16{tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384},
   507  			expectSuccess:      false,
   508  		},
   509  	}
   510  
   511  	for _, c := range cases {
   512  		t.Run(c.name, func(t *testing.T) {
   513  			configDir := t.TempDir()
   514  			args := NewPilotArgs(func(p *PilotArgs) {
   515  				p.Namespace = "istio-system"
   516  				p.ServerOptions = DiscoveryServerOptions{
   517  					// Dynamically assign all ports.
   518  					HTTPAddr:       ":0",
   519  					MonitoringAddr: ":0",
   520  					GRPCAddr:       ":0",
   521  					HTTPSAddr:      ":0",
   522  					TLSOptions: TLSOptions{
   523  						CipherSuits: c.serverCipherSuites,
   524  					},
   525  				}
   526  				p.RegistryOptions = RegistryOptions{
   527  					KubeConfig: "config",
   528  					FileDir:    configDir,
   529  				}
   530  
   531  				// Include all of the default plugins
   532  				p.ShutdownDuration = 1 * time.Millisecond
   533  			})
   534  
   535  			g := NewWithT(t)
   536  			s, err := NewServer(args, func(s *Server) {
   537  				s.kubeClient = kube.NewFakeClient()
   538  			})
   539  			g.Expect(err).To(Succeed())
   540  
   541  			stop := make(chan struct{})
   542  			g.Expect(s.Start(stop)).To(Succeed())
   543  			defer func() {
   544  				close(stop)
   545  				s.WaitUntilCompletion()
   546  			}()
   547  		})
   548  	}
   549  }
   550  
   551  func TestInitOIDC(t *testing.T) {
   552  	tests := []struct {
   553  		name      string
   554  		expectErr bool
   555  		jwtRule   string
   556  	}{
   557  		{
   558  			name:      "valid jwt rule",
   559  			expectErr: false,
   560  			jwtRule:   `{"issuer": "foo", "jwks_uri": "baz", "audiences": ["aud1", "aud2"]}`,
   561  		},
   562  		{
   563  			name:      "invalid jwt rule",
   564  			expectErr: true,
   565  			jwtRule:   "invalid",
   566  		},
   567  		{
   568  			name:      "jwt rule with invalid audiences",
   569  			expectErr: true,
   570  			// audiences must be a string array
   571  			jwtRule: `{"issuer": "foo", "jwks_uri": "baz", "audiences": "aud1"}`,
   572  		},
   573  	}
   574  
   575  	for _, tt := range tests {
   576  		t.Run(tt.name, func(t *testing.T) {
   577  			args := &PilotArgs{JwtRule: tt.jwtRule}
   578  
   579  			_, err := initOIDC(args)
   580  			gotErr := err != nil
   581  			if gotErr != tt.expectErr {
   582  				t.Errorf("expect error is %v while actual error is %v", tt.expectErr, gotErr)
   583  			}
   584  		})
   585  	}
   586  }
   587  
   588  func TestWatchDNSCertForK8sCA(t *testing.T) {
   589  	tests := []struct {
   590  		name        string
   591  		certToWatch []byte
   592  		certRotated bool
   593  	}{
   594  		{
   595  			name:        "expired cert rotation",
   596  			certToWatch: testcerts.ExpiredServerCert,
   597  			certRotated: true,
   598  		},
   599  		{
   600  			name:        "valid cert no rotation",
   601  			certToWatch: testcerts.ServerCert,
   602  			certRotated: false,
   603  		},
   604  	}
   605  
   606  	csr := &cert.CertificateSigningRequest{
   607  		ObjectMeta: metav1.ObjectMeta{
   608  			Name: "abc.xyz",
   609  		},
   610  		Status: cert.CertificateSigningRequestStatus{
   611  			Certificate: testcerts.ServerCert,
   612  		},
   613  	}
   614  	s := &Server{
   615  		server:                  server.New(),
   616  		istiodCertBundleWatcher: keycertbundle.NewWatcher(),
   617  		kubeClient:              kube.NewFakeClient(csr),
   618  		dnsNames:                []string{"abc.xyz"},
   619  	}
   620  	s.kubeClient.RunAndWait(test.NewStop(t))
   621  
   622  	for _, tt := range tests {
   623  		t.Run(tt.name, func(t *testing.T) {
   624  			stop := make(chan struct{})
   625  
   626  			s.istiodCertBundleWatcher.SetAndNotify(testcerts.ServerKey, tt.certToWatch, testcerts.CACert)
   627  			go s.RotateDNSCertForK8sCA(stop, "", "test-signer", true, time.Duration(0))
   628  
   629  			var certRotated bool
   630  			var rotatedCertBytes []byte
   631  			err := retry.Until(func() bool {
   632  				rotatedCertBytes = s.istiodCertBundleWatcher.GetKeyCertBundle().CertPem
   633  				st := string(rotatedCertBytes)
   634  				certRotated = st != string(tt.certToWatch)
   635  				return certRotated == tt.certRotated
   636  			}, retry.Timeout(10*time.Second))
   637  
   638  			close(stop)
   639  			if err != nil {
   640  				t.Fatalf("expect certRotated is %v while actual certRotated is %v", tt.certRotated, certRotated)
   641  			}
   642  			cert, certErr := util.ParsePemEncodedCertificate(rotatedCertBytes)
   643  			if certErr != nil {
   644  				t.Fatalf("rotated cert is not valid")
   645  			}
   646  			currTime := time.Now()
   647  			timeToExpire := cert.NotAfter.Sub(currTime)
   648  			if timeToExpire < 0 {
   649  				t.Fatalf("rotated cert is already expired")
   650  			}
   651  		})
   652  	}
   653  }
   654  
   655  func checkCert(t *testing.T, s *Server, cert, key []byte) bool {
   656  	t.Helper()
   657  	actual, err := s.getIstiodCertificate(nil)
   658  	if err != nil {
   659  		t.Fatalf("fail to load fetch certs.")
   660  	}
   661  	expected, err := tls.X509KeyPair(cert, key)
   662  	if err != nil {
   663  		t.Fatalf("fail to load test certs.")
   664  	}
   665  	return bytes.Equal(actual.Certificate[0], expected.Certificate[0])
   666  }
   667  
   668  func TestGetDNSNames(t *testing.T) {
   669  	tests := []struct {
   670  		name             string
   671  		customHost       string
   672  		discoveryAddress string
   673  		revision         string
   674  		sans             []string
   675  	}{
   676  		{
   677  			name:             "no customHost",
   678  			customHost:       "",
   679  			discoveryAddress: "istiod.istio-system.svc.cluster.local",
   680  			revision:         "default",
   681  			sans: []string{
   682  				"istio-pilot.istio-system.svc",
   683  				"istiod-remote.istio-system.svc",
   684  				"istiod.istio-system.svc",
   685  				"istiod.istio-system.svc.cluster.local",
   686  			},
   687  		},
   688  		{
   689  			name:             "default revision",
   690  			customHost:       "a.com,b.com,c.com",
   691  			discoveryAddress: "istiod.istio-system.svc.cluster.local",
   692  			revision:         "default",
   693  			sans: []string{
   694  				"a.com", "b.com", "c.com",
   695  				"istio-pilot.istio-system.svc",
   696  				"istiod-remote.istio-system.svc",
   697  				"istiod.istio-system.svc",
   698  				"istiod.istio-system.svc.cluster.local",
   699  			},
   700  		},
   701  		{
   702  			name:             "empty revision",
   703  			customHost:       "a.com,b.com,c.com",
   704  			discoveryAddress: "istiod.istio-system.svc.cluster.local",
   705  			revision:         "",
   706  			sans: []string{
   707  				"a.com", "b.com", "c.com",
   708  				"istio-pilot.istio-system.svc",
   709  				"istiod-remote.istio-system.svc",
   710  				"istiod.istio-system.svc",
   711  				"istiod.istio-system.svc.cluster.local",
   712  			},
   713  		},
   714  		{
   715  			name:             "canary revision",
   716  			customHost:       "a.com,b.com,c.com",
   717  			discoveryAddress: "istiod.istio-system.svc.cluster.local",
   718  			revision:         "canary",
   719  			sans: []string{
   720  				"a.com", "b.com", "c.com",
   721  				"istio-pilot.istio-system.svc",
   722  				"istiod-canary.istio-system.svc",
   723  				"istiod-remote.istio-system.svc",
   724  				"istiod.istio-system.svc",
   725  				"istiod.istio-system.svc.cluster.local",
   726  			},
   727  		},
   728  		{
   729  			name:             "customHost has duplicate hosts with inner default",
   730  			customHost:       "a.com,b.com,c.com,istiod",
   731  			discoveryAddress: "istiod.istio-system.svc.cluster.local",
   732  			revision:         "canary",
   733  			sans: []string{
   734  				"a.com", "b.com", "c.com",
   735  				"istio-pilot.istio-system.svc",
   736  				"istiod", // from the customHost
   737  				"istiod-canary.istio-system.svc",
   738  				"istiod-remote.istio-system.svc",
   739  				"istiod.istio-system.svc",
   740  				"istiod.istio-system.svc.cluster.local",
   741  			},
   742  		},
   743  		{
   744  			name:             "customHost has duplicate hosts with discovery address",
   745  			customHost:       "a.com,b.com,c.com,test.com",
   746  			discoveryAddress: "test.com",
   747  			revision:         "canary",
   748  			sans: []string{
   749  				"a.com", "b.com", "c.com",
   750  				"istio-pilot.istio-system.svc",
   751  				"istiod-canary.istio-system.svc",
   752  				"istiod-remote.istio-system.svc",
   753  				"istiod.istio-system.svc",
   754  				"test.com",
   755  			},
   756  		},
   757  	}
   758  
   759  	for _, tc := range tests {
   760  		t.Run(tc.name, func(t *testing.T) {
   761  			features.IstiodServiceCustomHost = tc.customHost
   762  			var args PilotArgs
   763  			args.Revision = tc.revision
   764  			args.Namespace = "istio-system"
   765  			sans := getDNSNames(&args, tc.discoveryAddress)
   766  			assert.Equal(t, sans, tc.sans)
   767  		})
   768  	}
   769  }