k8s.io/apiserver@v0.31.1/pkg/util/webhook/webhook_test.go (about)

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package webhook
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"crypto/x509"
    23  	"encoding/json"
    24  	"errors"
    25  	"fmt"
    26  	"net"
    27  	"net/http"
    28  	"net/http/httptest"
    29  	"net/url"
    30  	"os"
    31  	"path/filepath"
    32  	"regexp"
    33  	"strings"
    34  	"testing"
    35  	"time"
    36  
    37  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    38  	"k8s.io/apimachinery/pkg/runtime"
    39  	"k8s.io/apimachinery/pkg/runtime/schema"
    40  	"k8s.io/apimachinery/pkg/util/wait"
    41  	"k8s.io/client-go/kubernetes/scheme"
    42  	"k8s.io/client-go/rest"
    43  	v1 "k8s.io/client-go/tools/clientcmd/api/v1"
    44  	"k8s.io/component-base/metrics"
    45  	"k8s.io/component-base/metrics/legacyregistry"
    46  )
    47  
    48  const (
    49  	errBadCertificate    = "Get .*: remote error: tls: (bad certificate|unknown certificate authority)"
    50  	errNoConfiguration   = "invalid configuration: no configuration has been provided"
    51  	errMissingCertPath   = "invalid configuration: unable to read %s %s for %s due to open %s: .*"
    52  	errSignedByUnknownCA = "Get .*: x509: .*(unknown authority|not standards compliant|not trusted)"
    53  )
    54  
    55  var (
    56  	defaultCluster = v1.NamedCluster{
    57  		Cluster: v1.Cluster{
    58  			Server:                   "https://webhook.example.com",
    59  			CertificateAuthorityData: caCert,
    60  		},
    61  	}
    62  	defaultUser = v1.NamedAuthInfo{
    63  		AuthInfo: v1.AuthInfo{
    64  			ClientCertificateData: clientCert,
    65  			ClientKeyData:         clientKey,
    66  		},
    67  	}
    68  	namedCluster = v1.NamedCluster{
    69  		Cluster: v1.Cluster{
    70  			Server:                   "https://webhook.example.com",
    71  			CertificateAuthorityData: caCert,
    72  		},
    73  		Name: "test-cluster",
    74  	}
    75  	groupVersions = []schema.GroupVersion{}
    76  	retryBackoff  = DefaultRetryBackoffWithInitialDelay(time.Duration(500) * time.Millisecond)
    77  )
    78  
    79  // TestKubeConfigFile ensures that a kube config file, regardless of validity, is handled properly
    80  func TestKubeConfigFile(t *testing.T) {
    81  	badCAPath := "/tmp/missing/ca.pem"
    82  	badClientCertPath := "/tmp/missing/client.pem"
    83  	badClientKeyPath := "/tmp/missing/client-key.pem"
    84  	dir := bootstrapTestDir(t)
    85  
    86  	defer os.RemoveAll(dir)
    87  
    88  	// These tests check for all of the ways in which a Kubernetes config file could be malformed within the context of
    89  	// configuring a webhook.  Configuration issues that arise while using the webhook are tested elsewhere.
    90  	tests := []struct {
    91  		test           string
    92  		cluster        *v1.NamedCluster
    93  		context        *v1.NamedContext
    94  		currentContext string
    95  		user           *v1.NamedAuthInfo
    96  		errRegex       string
    97  	}{
    98  		{
    99  			test:     "missing context (no default, none specified)",
   100  			cluster:  &namedCluster,
   101  			errRegex: errNoConfiguration,
   102  		},
   103  		{
   104  			test:     "missing context (specified context is missing)",
   105  			cluster:  &namedCluster,
   106  			errRegex: errNoConfiguration,
   107  		},
   108  		{
   109  			test: "context without cluster",
   110  			context: &v1.NamedContext{
   111  				Context: v1.Context{},
   112  				Name:    "testing-context",
   113  			},
   114  			currentContext: "testing-context",
   115  			errRegex:       errNoConfiguration,
   116  		},
   117  		{
   118  			test:    "context without user",
   119  			cluster: &namedCluster,
   120  			context: &v1.NamedContext{
   121  				Context: v1.Context{
   122  					Cluster: namedCluster.Name,
   123  				},
   124  				Name: "testing-context",
   125  			},
   126  			currentContext: "testing-context",
   127  			errRegex:       "", // Not an error at parse time, only when using the webhook
   128  		},
   129  		{
   130  			test:    "context with missing cluster",
   131  			cluster: &namedCluster,
   132  			context: &v1.NamedContext{
   133  				Context: v1.Context{
   134  					Cluster: "missing-cluster",
   135  				},
   136  				Name: "fake",
   137  			},
   138  			errRegex: errNoConfiguration,
   139  		},
   140  		{
   141  			test:    "context with missing user",
   142  			cluster: &namedCluster,
   143  			context: &v1.NamedContext{
   144  				Context: v1.Context{
   145  					Cluster:  namedCluster.Name,
   146  					AuthInfo: "missing-user",
   147  				},
   148  				Name: "testing-context",
   149  			},
   150  			currentContext: "testing-context",
   151  			errRegex:       "", // Not an error at parse time, only when using the webhook
   152  		},
   153  		{
   154  			test: "cluster with invalid CA certificate path",
   155  			cluster: &v1.NamedCluster{
   156  				Cluster: v1.Cluster{
   157  					Server:               namedCluster.Cluster.Server,
   158  					CertificateAuthority: badCAPath,
   159  				},
   160  			},
   161  			user:     &defaultUser,
   162  			errRegex: fmt.Sprintf(errMissingCertPath, "certificate-authority", badCAPath, "", badCAPath),
   163  		},
   164  		{
   165  			test: "cluster with invalid CA certificate",
   166  			cluster: &v1.NamedCluster{
   167  				Cluster: v1.Cluster{
   168  					Server:                   namedCluster.Cluster.Server,
   169  					CertificateAuthorityData: caKey, // pretend user put caKey here instead of caCert
   170  				},
   171  			},
   172  			user:     &defaultUser,
   173  			errRegex: "unable to load root certificates: no valid certificate authority data seen",
   174  		},
   175  		{
   176  			test: "cluster with invalid CA certificate - no PEM",
   177  			cluster: &v1.NamedCluster{
   178  				Cluster: v1.Cluster{
   179  					Server:                   namedCluster.Cluster.Server,
   180  					CertificateAuthorityData: []byte(`not a cert`),
   181  				},
   182  			},
   183  			user:     &defaultUser,
   184  			errRegex: "unable to load root certificates: unable to parse bytes as PEM block",
   185  		},
   186  		{
   187  			test: "cluster with invalid CA certificate - parse error",
   188  			cluster: &v1.NamedCluster{
   189  				Cluster: v1.Cluster{
   190  					Server: namedCluster.Cluster.Server,
   191  					CertificateAuthorityData: []byte(`
   192  -----BEGIN CERTIFICATE-----
   193  MIIDGTCCAgGgAwIBAgIUOS2M
   194  -----END CERTIFICATE-----
   195  `),
   196  				},
   197  			},
   198  			user:     &defaultUser,
   199  			errRegex: "unable to load root certificates: failed to parse certificate: (asn1: syntax error: data truncated|x509: malformed certificate)",
   200  		},
   201  		{
   202  			test:    "user with invalid client certificate path",
   203  			cluster: &defaultCluster,
   204  			user: &v1.NamedAuthInfo{
   205  				AuthInfo: v1.AuthInfo{
   206  					ClientCertificate: badClientCertPath,
   207  					ClientKeyData:     defaultUser.AuthInfo.ClientKeyData,
   208  				},
   209  			},
   210  			errRegex: fmt.Sprintf(errMissingCertPath, "client-cert", badClientCertPath, "", badClientCertPath),
   211  		},
   212  		{
   213  			test:    "user with invalid client certificate",
   214  			cluster: &defaultCluster,
   215  			user: &v1.NamedAuthInfo{
   216  				AuthInfo: v1.AuthInfo{
   217  					ClientCertificateData: clientKey,
   218  					ClientKeyData:         defaultUser.AuthInfo.ClientKeyData,
   219  				},
   220  			},
   221  			errRegex: "tls: failed to find certificate PEM data in certificate input, but did find a private key; PEM inputs may have been switched",
   222  		},
   223  		{
   224  			test:    "user with invalid client certificate path",
   225  			cluster: &defaultCluster,
   226  			user: &v1.NamedAuthInfo{
   227  				AuthInfo: v1.AuthInfo{
   228  					ClientCertificateData: defaultUser.AuthInfo.ClientCertificateData,
   229  					ClientKey:             badClientKeyPath,
   230  				},
   231  			},
   232  			errRegex: fmt.Sprintf(errMissingCertPath, "client-key", badClientKeyPath, "", badClientKeyPath),
   233  		},
   234  		{
   235  			test:    "user with invalid client certificate",
   236  			cluster: &defaultCluster,
   237  			user: &v1.NamedAuthInfo{
   238  				AuthInfo: v1.AuthInfo{
   239  					ClientCertificateData: defaultUser.AuthInfo.ClientCertificateData,
   240  					ClientKeyData:         clientCert,
   241  				},
   242  			},
   243  			errRegex: "tls: found a certificate rather than a key in the PEM for the private key",
   244  		},
   245  		{
   246  			test:     "valid configuration (certificate data embedded in config)",
   247  			cluster:  &defaultCluster,
   248  			user:     &defaultUser,
   249  			errRegex: "",
   250  		},
   251  		{
   252  			test: "valid configuration (certificate files referenced in config)",
   253  			cluster: &v1.NamedCluster{
   254  				Cluster: v1.Cluster{
   255  					Server:               "https://webhook.example.com",
   256  					CertificateAuthority: filepath.Join(dir, "ca.pem"),
   257  				},
   258  			},
   259  			user: &v1.NamedAuthInfo{
   260  				AuthInfo: v1.AuthInfo{
   261  					ClientCertificate: filepath.Join(dir, "client.pem"),
   262  					ClientKey:         filepath.Join(dir, "client-key.pem"),
   263  				},
   264  			},
   265  			errRegex: "",
   266  		},
   267  	}
   268  
   269  	for _, tt := range tests {
   270  		t.Run(tt.test, func(t *testing.T) {
   271  			// Use a closure so defer statements trigger between loop iterations.
   272  			err := func() error {
   273  				kubeConfig := v1.Config{}
   274  
   275  				if tt.cluster != nil {
   276  					kubeConfig.Clusters = []v1.NamedCluster{*tt.cluster}
   277  				}
   278  
   279  				if tt.context != nil {
   280  					kubeConfig.Contexts = []v1.NamedContext{*tt.context}
   281  				}
   282  
   283  				if tt.user != nil {
   284  					kubeConfig.AuthInfos = []v1.NamedAuthInfo{*tt.user}
   285  				}
   286  
   287  				kubeConfig.CurrentContext = tt.currentContext
   288  
   289  				kubeConfigFile, err := newKubeConfigFile(kubeConfig)
   290  				if err != nil {
   291  					return err
   292  				}
   293  
   294  				defer os.Remove(kubeConfigFile)
   295  
   296  				config, err := LoadKubeconfig(kubeConfigFile, nil)
   297  				if err != nil {
   298  					return err
   299  				}
   300  
   301  				_, err = NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, config, groupVersions, retryBackoff)
   302  				return err
   303  			}()
   304  
   305  			if err == nil {
   306  				if tt.errRegex != "" {
   307  					t.Errorf("%s: expected an error", tt.test)
   308  				}
   309  			} else {
   310  				if tt.errRegex == "" {
   311  					t.Errorf("%s: unexpected error: %v", tt.test, err)
   312  				} else if !regexp.MustCompile(tt.errRegex).MatchString(err.Error()) {
   313  					t.Errorf("%s: unexpected error message to match:\n  Expected: %s\n  Actual:   %s", tt.test, tt.errRegex, err.Error())
   314  				}
   315  			}
   316  		})
   317  	}
   318  }
   319  
   320  // TestMissingKubeConfigFile ensures that a kube config path to a missing file is handled properly
   321  func TestMissingKubeConfigFile(t *testing.T) {
   322  	kubeConfigPath := "/some/missing/path"
   323  	_, err := LoadKubeconfig(kubeConfigPath, nil)
   324  
   325  	if err == nil {
   326  		t.Errorf("creating the webhook should had failed")
   327  	} else if strings.Index(err.Error(), fmt.Sprintf("stat %s", kubeConfigPath)) != 0 {
   328  		t.Errorf("unexpected error: %v", err)
   329  	}
   330  }
   331  
   332  // TestTLSConfig ensures that the TLS-based communication between client and server works as expected
   333  func TestTLSConfig(t *testing.T) {
   334  	invalidCert := []byte("invalid")
   335  	tests := []struct {
   336  		test                             string
   337  		clientCert, clientKey, clientCA  []byte
   338  		serverCert, serverKey, serverCA  []byte
   339  		errRegex                         string
   340  		increaseSANWarnCounter           bool
   341  		increaseSHA1SignatureWarnCounter bool
   342  	}{
   343  		{
   344  			test:       "invalid server CA",
   345  			clientCert: clientCert, clientKey: clientKey, clientCA: caCert,
   346  			serverCert: serverCert, serverKey: serverKey, serverCA: invalidCert,
   347  			errRegex: errBadCertificate,
   348  		},
   349  		{
   350  			test:       "invalid client certificate",
   351  			clientCert: invalidCert, clientKey: clientKey, clientCA: caCert,
   352  			serverCert: serverCert, serverKey: serverKey, serverCA: caCert,
   353  			errRegex: "tls: failed to find any PEM data in certificate input",
   354  		},
   355  		{
   356  			test:       "invalid client key",
   357  			clientCert: clientCert, clientKey: invalidCert, clientCA: caCert,
   358  			serverCert: serverCert, serverKey: serverKey, serverCA: caCert,
   359  			errRegex: "tls: failed to find any PEM data in key input",
   360  		},
   361  		{
   362  			test:       "client does not trust server",
   363  			clientCert: clientCert, clientKey: clientKey,
   364  			serverCert: serverCert, serverKey: serverKey,
   365  			errRegex: errSignedByUnknownCA,
   366  		},
   367  		{
   368  			test:       "server does not trust client",
   369  			clientCert: clientCert, clientKey: clientKey, clientCA: badCACert,
   370  			serverCert: serverCert, serverKey: serverKey, serverCA: caCert,
   371  			errRegex: errSignedByUnknownCA + " .*",
   372  		},
   373  		{
   374  			test:       "server requires auth, client provides it",
   375  			clientCert: clientCert, clientKey: clientKey, clientCA: caCert,
   376  			serverCert: serverCert, serverKey: serverKey, serverCA: caCert,
   377  			errRegex: "",
   378  		},
   379  		{
   380  			test:       "server does not require client auth",
   381  			clientCA:   caCert,
   382  			serverCert: serverCert, serverKey: serverKey,
   383  			errRegex: "",
   384  		},
   385  		{
   386  			test:       "server does not require client auth, client provides it",
   387  			clientCert: clientCert, clientKey: clientKey, clientCA: caCert,
   388  			serverCert: serverCert, serverKey: serverKey,
   389  			errRegex: "",
   390  		},
   391  		{
   392  			test:       "webhook does not support insecure servers",
   393  			serverCert: serverCert, serverKey: serverKey,
   394  			errRegex: errSignedByUnknownCA,
   395  		},
   396  		{
   397  			// this will fail when GODEBUG is set to x509ignoreCN=0 with
   398  			// expected err, but the SAN counter gets increased
   399  			test:       "server cert does not have SAN extension",
   400  			clientCA:   caCert,
   401  			serverCert: serverCertNoSAN, serverKey: serverKey,
   402  			errRegex:               "x509: certificate relies on legacy Common Name field",
   403  			increaseSANWarnCounter: true,
   404  		},
   405  		{
   406  			test:       "server cert with SHA1 signature",
   407  			clientCA:   caCert,
   408  			serverCert: append(append(sha1ServerCertInter, byte('\n')), caCertInter...), serverKey: serverKey,
   409  			errRegex:                         "x509: cannot verify signature: insecure algorithm SHA1-RSA \\(temporarily override with GODEBUG=x509sha1=1\\)",
   410  			increaseSHA1SignatureWarnCounter: true,
   411  		},
   412  		{
   413  			test:       "server cert signed by an intermediate CA with SHA1 signature",
   414  			clientCA:   caCert,
   415  			serverCert: append(append(serverCertInterSHA1, byte('\n')), caCertInterSHA1...), serverKey: serverKey,
   416  			errRegex:                         "x509: cannot verify signature: insecure algorithm SHA1-RSA \\(temporarily override with GODEBUG=x509sha1=1\\)",
   417  			increaseSHA1SignatureWarnCounter: true,
   418  		},
   419  	}
   420  
   421  	lastSHA1SigCounter := 0
   422  	for _, tt := range tests {
   423  		// Use a closure so defer statements trigger between loop iterations.
   424  		func() {
   425  			// Create and start a simple HTTPS server
   426  			server, err := newTestServer(tt.serverCert, tt.serverKey, tt.serverCA, nil)
   427  			if err != nil {
   428  				t.Errorf("%s: failed to create server: %v", tt.test, err)
   429  				return
   430  			}
   431  
   432  			serverURL, err := url.Parse(server.URL)
   433  			if err != nil {
   434  				t.Errorf("%s: failed to parse the testserver URL: %v", tt.test, err)
   435  				return
   436  			}
   437  			serverURL.Host = net.JoinHostPort("localhost", serverURL.Port())
   438  
   439  			defer server.Close()
   440  
   441  			// Create a Kubernetes client configuration file
   442  			configFile, err := newKubeConfigFile(v1.Config{
   443  				Clusters: []v1.NamedCluster{
   444  					{
   445  						Cluster: v1.Cluster{
   446  							Server:                   serverURL.String(),
   447  							CertificateAuthorityData: tt.clientCA,
   448  						},
   449  					},
   450  				},
   451  				AuthInfos: []v1.NamedAuthInfo{
   452  					{
   453  						AuthInfo: v1.AuthInfo{
   454  							ClientCertificateData: tt.clientCert,
   455  							ClientKeyData:         tt.clientKey,
   456  						},
   457  					},
   458  				},
   459  			})
   460  
   461  			if err != nil {
   462  				t.Errorf("%s: %v", tt.test, err)
   463  				return
   464  			}
   465  
   466  			defer os.Remove(configFile)
   467  
   468  			config, err := LoadKubeconfig(configFile, nil)
   469  			if err != nil {
   470  				t.Fatal(err)
   471  			}
   472  
   473  			wh, err := NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, config, groupVersions, retryBackoff)
   474  
   475  			if err == nil {
   476  				err = wh.RestClient.Get().Do(context.TODO()).Error()
   477  			}
   478  
   479  			if err == nil {
   480  				if tt.errRegex != "" {
   481  					t.Errorf("%s: expected an error", tt.test)
   482  				}
   483  			} else {
   484  				if tt.errRegex == "" {
   485  					t.Errorf("%s: unexpected error: %v", tt.test, err)
   486  				} else if !regexp.MustCompile(tt.errRegex).MatchString(err.Error()) {
   487  					t.Errorf("%s: unexpected error message mismatch:\n  Expected: %s\n  Actual:   %s", tt.test, tt.errRegex, err.Error())
   488  				}
   489  			}
   490  
   491  			if tt.increaseSANWarnCounter {
   492  				errorCounter := getSingleCounterValueFromRegistry(t, legacyregistry.DefaultGatherer, "apiserver_webhooks_x509_missing_san_total")
   493  
   494  				if errorCounter == -1 {
   495  					t.Errorf("failed to get the x509_common_name_error_count metrics: %v", err)
   496  				}
   497  				if int(errorCounter) != 1 {
   498  					t.Errorf("expected the x509_common_name_error_count to be 1, but it's %d", errorCounter)
   499  				}
   500  			}
   501  
   502  			if tt.increaseSHA1SignatureWarnCounter {
   503  				errorCounter := getSingleCounterValueFromRegistry(t, legacyregistry.DefaultGatherer, "apiserver_webhooks_x509_insecure_sha1_total")
   504  
   505  				if errorCounter == -1 {
   506  					t.Errorf("failed to get the apiserver_webhooks_x509_insecure_sha1_total metrics: %v", err)
   507  				}
   508  
   509  				if int(errorCounter) != lastSHA1SigCounter+1 {
   510  					t.Errorf("expected the apiserver_webhooks_x509_insecure_sha1_total counter to be 1, but it's %d", errorCounter)
   511  				}
   512  
   513  				lastSHA1SigCounter++
   514  			}
   515  		}()
   516  	}
   517  }
   518  
   519  func TestRequestTimeout(t *testing.T) {
   520  	done := make(chan struct{})
   521  
   522  	handler := func(w http.ResponseWriter, r *http.Request) {
   523  		<-done
   524  	}
   525  
   526  	// Create and start a simple HTTPS server
   527  	server, err := newTestServer(clientCert, clientKey, caCert, handler)
   528  	if err != nil {
   529  		t.Errorf("failed to create server: %v", err)
   530  		return
   531  	}
   532  	defer server.Close()
   533  	defer close(done) // done channel must be closed before server is.
   534  
   535  	// Create a Kubernetes client configuration file
   536  	configFile, err := newKubeConfigFile(v1.Config{
   537  		Clusters: []v1.NamedCluster{
   538  			{
   539  				Cluster: v1.Cluster{
   540  					Server:                   server.URL,
   541  					CertificateAuthorityData: caCert,
   542  				},
   543  			},
   544  		},
   545  		AuthInfos: []v1.NamedAuthInfo{
   546  			{
   547  				AuthInfo: v1.AuthInfo{
   548  					ClientCertificateData: clientCert,
   549  					ClientKeyData:         clientKey,
   550  				},
   551  			},
   552  		},
   553  	})
   554  	if err != nil {
   555  		t.Errorf("failed to create the client config file: %v", err)
   556  		return
   557  	}
   558  	defer os.Remove(configFile)
   559  
   560  	var requestTimeout = 10 * time.Millisecond
   561  
   562  	config, err := LoadKubeconfig(configFile, nil)
   563  	if err != nil {
   564  		t.Fatal(err)
   565  	}
   566  
   567  	config.Timeout = requestTimeout
   568  
   569  	wh, err := NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, config, groupVersions, retryBackoff)
   570  	if err != nil {
   571  		t.Fatalf("failed to create the webhook: %v", err)
   572  	}
   573  
   574  	resultCh := make(chan rest.Result)
   575  
   576  	go func() { resultCh <- wh.RestClient.Get().Do(context.TODO()) }()
   577  	select {
   578  	case <-time.After(time.Second * 5):
   579  		t.Errorf("expected request to timeout after %s", requestTimeout)
   580  	case <-resultCh:
   581  	}
   582  }
   583  
   584  // TestWithExponentialBackoff ensures that the webhook's exponential backoff support works as expected
   585  func TestWithExponentialBackoff(t *testing.T) {
   586  	count := 0 // To keep track of the requests
   587  	gr := schema.GroupResource{
   588  		Group:    "webhook.util.k8s.io",
   589  		Resource: "test",
   590  	}
   591  
   592  	// Handler that will handle all backoff CONDITIONS
   593  	ebHandler := func(w http.ResponseWriter, r *http.Request) {
   594  		w.Header().Set("Content-Type", "application/json")
   595  
   596  		switch count++; count {
   597  		case 1:
   598  			// Timeout error with retry supplied
   599  			w.WriteHeader(http.StatusGatewayTimeout)
   600  			json.NewEncoder(w).Encode(apierrors.NewServerTimeout(gr, "get", 2))
   601  		case 2:
   602  			// Internal server error
   603  			w.WriteHeader(http.StatusInternalServerError)
   604  			json.NewEncoder(w).Encode(apierrors.NewInternalError(fmt.Errorf("nope")))
   605  		case 3:
   606  			// HTTP error that is not retryable
   607  			w.WriteHeader(http.StatusNotAcceptable)
   608  			json.NewEncoder(w).Encode(apierrors.NewGenericServerResponse(http.StatusNotAcceptable, "get", gr, "testing", "nope", 0, false))
   609  		case 4:
   610  			// Successful request
   611  			w.WriteHeader(http.StatusOK)
   612  			json.NewEncoder(w).Encode(map[string]string{
   613  				"status": "OK",
   614  			})
   615  		}
   616  	}
   617  
   618  	// Create and start a simple HTTPS server
   619  	server, err := newTestServer(clientCert, clientKey, caCert, ebHandler)
   620  
   621  	if err != nil {
   622  		t.Errorf("failed to create server: %v", err)
   623  		return
   624  	}
   625  
   626  	defer server.Close()
   627  
   628  	// Create a Kubernetes client configuration file
   629  	configFile, err := newKubeConfigFile(v1.Config{
   630  		Clusters: []v1.NamedCluster{
   631  			{
   632  				Cluster: v1.Cluster{
   633  					Server:                   server.URL,
   634  					CertificateAuthorityData: caCert,
   635  				},
   636  			},
   637  		},
   638  		AuthInfos: []v1.NamedAuthInfo{
   639  			{
   640  				AuthInfo: v1.AuthInfo{
   641  					ClientCertificateData: clientCert,
   642  					ClientKeyData:         clientKey,
   643  				},
   644  			},
   645  		},
   646  	})
   647  
   648  	if err != nil {
   649  		t.Errorf("failed to create the client config file: %v", err)
   650  		return
   651  	}
   652  
   653  	defer os.Remove(configFile)
   654  
   655  	config, err := LoadKubeconfig(configFile, nil)
   656  	if err != nil {
   657  		t.Fatal(err)
   658  	}
   659  
   660  	wh, err := NewGenericWebhook(runtime.NewScheme(), scheme.Codecs, config, groupVersions, retryBackoff)
   661  
   662  	if err != nil {
   663  		t.Fatalf("failed to create the webhook: %v", err)
   664  	}
   665  
   666  	result := wh.WithExponentialBackoff(context.Background(), func() rest.Result {
   667  		return wh.RestClient.Get().Do(context.TODO())
   668  	})
   669  
   670  	var statusCode int
   671  
   672  	result.StatusCode(&statusCode)
   673  
   674  	if statusCode != http.StatusNotAcceptable {
   675  		t.Errorf("unexpected status code: %d", statusCode)
   676  	}
   677  
   678  	result = wh.WithExponentialBackoff(context.Background(), func() rest.Result {
   679  		return wh.RestClient.Get().Do(context.TODO())
   680  	})
   681  
   682  	result.StatusCode(&statusCode)
   683  
   684  	if statusCode != http.StatusOK {
   685  		t.Errorf("unexpected status code: %d", statusCode)
   686  	}
   687  }
   688  
   689  func bootstrapTestDir(t *testing.T) string {
   690  	dir, err := os.MkdirTemp("", "")
   691  
   692  	if err != nil {
   693  		t.Fatal(err)
   694  	}
   695  
   696  	// The certificates needed on disk for the tests
   697  	files := map[string][]byte{
   698  		"ca.pem":         caCert,
   699  		"client.pem":     clientCert,
   700  		"client-key.pem": clientKey,
   701  	}
   702  
   703  	// Write the certificate files to disk or fail
   704  	for fileName, fileData := range files {
   705  		if err := os.WriteFile(filepath.Join(dir, fileName), fileData, 0400); err != nil {
   706  			os.RemoveAll(dir)
   707  			t.Fatal(err)
   708  		}
   709  	}
   710  
   711  	return dir
   712  }
   713  
   714  func newKubeConfigFile(config v1.Config) (string, error) {
   715  	configFile, err := os.CreateTemp("", "")
   716  	if err != nil {
   717  		return "", err
   718  	}
   719  	defer configFile.Close()
   720  
   721  	if err != nil {
   722  		return "", fmt.Errorf("unable to create the Kubernetes client config file: %v", err)
   723  	}
   724  
   725  	if err = json.NewEncoder(configFile).Encode(config); err != nil {
   726  		return "", fmt.Errorf("unable to write the Kubernetes client configuration to disk: %v", err)
   727  	}
   728  
   729  	return configFile.Name(), nil
   730  }
   731  
   732  func newTestServer(clientCert, clientKey, caCert []byte, handler func(http.ResponseWriter, *http.Request)) (*httptest.Server, error) {
   733  	var tlsConfig *tls.Config
   734  
   735  	if clientCert != nil {
   736  		cert, err := tls.X509KeyPair(clientCert, clientKey)
   737  
   738  		if err != nil {
   739  			return nil, err
   740  		}
   741  
   742  		tlsConfig = &tls.Config{
   743  			Certificates: []tls.Certificate{cert},
   744  		}
   745  	}
   746  
   747  	if caCert != nil {
   748  		rootCAs := x509.NewCertPool()
   749  
   750  		rootCAs.AppendCertsFromPEM(caCert)
   751  
   752  		if tlsConfig == nil {
   753  			tlsConfig = &tls.Config{}
   754  		}
   755  
   756  		tlsConfig.ClientCAs = rootCAs
   757  		tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
   758  	}
   759  
   760  	if handler == nil {
   761  		handler = func(w http.ResponseWriter, r *http.Request) {
   762  			w.Write([]byte("OK"))
   763  		}
   764  	}
   765  
   766  	server := httptest.NewUnstartedServer(http.HandlerFunc(handler))
   767  
   768  	server.TLS = tlsConfig
   769  	server.StartTLS()
   770  
   771  	return server, nil
   772  }
   773  
   774  func TestWithExponentialBackoffContextIsAlreadyCanceled(t *testing.T) {
   775  	alwaysRetry := func(e error) bool {
   776  		return true
   777  	}
   778  
   779  	attemptsGot := 0
   780  	webhookFunc := func() error {
   781  		attemptsGot++
   782  		return nil
   783  	}
   784  
   785  	ctx, cancel := context.WithCancel(context.TODO())
   786  	cancel()
   787  
   788  	// We don't expect the webhook function to be called since the context is already canceled.
   789  	retryBackoff := wait.Backoff{Steps: 5}
   790  	err := WithExponentialBackoff(ctx, retryBackoff, webhookFunc, alwaysRetry)
   791  
   792  	errExpected := fmt.Errorf("webhook call failed: %s", context.Canceled)
   793  	if errExpected.Error() != err.Error() {
   794  		t.Errorf("expected error: %v, but got: %v", errExpected, err)
   795  	}
   796  	if attemptsGot != 0 {
   797  		t.Errorf("expected %d webhook attempts, but got: %d", 0, attemptsGot)
   798  	}
   799  }
   800  
   801  func TestWithExponentialBackoffWebhookErrorIsMostImportant(t *testing.T) {
   802  	alwaysRetry := func(e error) bool {
   803  		return true
   804  	}
   805  
   806  	ctx, cancel := context.WithCancel(context.TODO())
   807  	attemptsGot := 0
   808  	errExpected := errors.New("webhook not available")
   809  	webhookFunc := func() error {
   810  		attemptsGot++
   811  
   812  		// after the first attempt, the context is canceled
   813  		cancel()
   814  
   815  		return errExpected
   816  	}
   817  
   818  	// webhook err has higher priority than ctx error. we expect the webhook error to be returned.
   819  	retryBackoff := wait.Backoff{Steps: 5}
   820  	err := WithExponentialBackoff(ctx, retryBackoff, webhookFunc, alwaysRetry)
   821  
   822  	if attemptsGot != 1 {
   823  		t.Errorf("expected %d webhook attempts, but got: %d", 1, attemptsGot)
   824  	}
   825  	if errExpected != err {
   826  		t.Errorf("expected error: %v, but got: %v", errExpected, err)
   827  	}
   828  }
   829  
   830  func TestWithExponentialBackoffWithRetryExhaustedWhileContextIsNotCanceled(t *testing.T) {
   831  	alwaysRetry := func(e error) bool {
   832  		return true
   833  	}
   834  
   835  	ctx, cancel := context.WithCancel(context.TODO())
   836  	defer cancel()
   837  
   838  	attemptsGot := 0
   839  	errExpected := errors.New("webhook not available")
   840  	webhookFunc := func() error {
   841  		attemptsGot++
   842  		return errExpected
   843  	}
   844  
   845  	// webhook err has higher priority than ctx error. we expect the webhook error to be returned.
   846  	retryBackoff := wait.Backoff{Steps: 5}
   847  	err := WithExponentialBackoff(ctx, retryBackoff, webhookFunc, alwaysRetry)
   848  
   849  	if attemptsGot != 5 {
   850  		t.Errorf("expected %d webhook attempts, but got: %d", 1, attemptsGot)
   851  	}
   852  	if errExpected != err {
   853  		t.Errorf("expected error: %v, but got: %v", errExpected, err)
   854  	}
   855  }
   856  
   857  func TestWithExponentialBackoffParametersNotSet(t *testing.T) {
   858  	alwaysRetry := func(e error) bool {
   859  		return true
   860  	}
   861  
   862  	attemptsGot := 0
   863  	webhookFunc := func() error {
   864  		attemptsGot++
   865  		return nil
   866  	}
   867  
   868  	err := WithExponentialBackoff(context.TODO(), wait.Backoff{}, webhookFunc, alwaysRetry)
   869  
   870  	errExpected := fmt.Errorf("webhook call failed: %s", wait.ErrWaitTimeout)
   871  	if errExpected.Error() != err.Error() {
   872  		t.Errorf("expected error: %v, but got: %v", errExpected, err)
   873  	}
   874  	if attemptsGot != 0 {
   875  		t.Errorf("expected %d webhook attempts, but got: %d", 0, attemptsGot)
   876  	}
   877  }
   878  
   879  func TestGenericWebhookWithExponentialBackoff(t *testing.T) {
   880  	attemptsPerCallExpected := 5
   881  	webhook := &GenericWebhook{
   882  		RetryBackoff: wait.Backoff{
   883  			Duration: time.Millisecond,
   884  			Factor:   1.5,
   885  			Jitter:   0.2,
   886  			Steps:    attemptsPerCallExpected,
   887  		},
   888  
   889  		ShouldRetry: func(e error) bool {
   890  			return true
   891  		},
   892  	}
   893  
   894  	attemptsGot := 0
   895  	webhookFunc := func() rest.Result {
   896  		attemptsGot++
   897  		return rest.Result{}
   898  	}
   899  
   900  	// number of retries should always be local to each call.
   901  	totalAttemptsExpected := attemptsPerCallExpected * 2
   902  	webhook.WithExponentialBackoff(context.TODO(), webhookFunc)
   903  	webhook.WithExponentialBackoff(context.TODO(), webhookFunc)
   904  
   905  	if totalAttemptsExpected != attemptsGot {
   906  		t.Errorf("expected a total of %d webhook attempts but got: %d", totalAttemptsExpected, attemptsGot)
   907  	}
   908  }
   909  
   910  func getSingleCounterValueFromRegistry(t *testing.T, r metrics.Gatherer, name string) int {
   911  	mfs, err := r.Gather()
   912  	if err != nil {
   913  		t.Logf("failed to gather local registry metrics: %v", err)
   914  		return -1
   915  	}
   916  
   917  	for _, mf := range mfs {
   918  		if mf.Name != nil && *mf.Name == name {
   919  			mfMetric := mf.GetMetric()
   920  			for _, m := range mfMetric {
   921  				if m.GetCounter() != nil {
   922  					return int(m.GetCounter().GetValue())
   923  				}
   924  			}
   925  		}
   926  	}
   927  
   928  	return -1
   929  }