k8s.io/apiserver@v0.31.1/pkg/server/graceful_shutdown_test.go (about)

     1  /*
     2  Copyright 2021 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package server
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"crypto/x509"
    23  	"fmt"
    24  	"io"
    25  	"net"
    26  	"net/http"
    27  	"net/http/httptest"
    28  	"sync"
    29  	"testing"
    30  	"time"
    31  
    32  	"golang.org/x/net/http2"
    33  )
    34  
    35  var (
    36  	// startServerShutdown is a signal from the backend after receiving all (25) requests
    37  	// after which the test shuts down the HTTP server
    38  	startServerShutdown = make(chan struct{})
    39  
    40  	// handlerLock used in the backendHTTPHandler to count the number of requests and signal the test that the termination can start
    41  	handlerLock = sync.Mutex{}
    42  )
    43  
    44  var backendCrt = []byte(`-----BEGIN CERTIFICATE-----
    45  MIIDTjCCAjagAwIBAgIJANYWBFaLyBC/MA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV
    46  BAYTAlBMMQ8wDQYDVQQIDAZQb2xhbmQxDzANBgNVBAcMBkdkYW5zazELMAkGA1UE
    47  CgwCU0sxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0yMDEyMTExMDI0MzBaFw0zMDEy
    48  MDkxMDI0MzBaMFAxCzAJBgNVBAYTAlBMMQ8wDQYDVQQIDAZQb2xhbmQxDzANBgNV
    49  BAcMBkdkYW5zazELMAkGA1UECgwCU0sxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIw
    50  DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMYax2q/m/N237UFMFKZsox4EyKq
    51  De+mbaRGeKqnI7Gi9Ai3b7BPCIa7RFJ2ntpGUd5GyL+HCQHG8/f6DjsbUuhZnmn7
    52  F7ZJeih2DP2acKkODdGbXA52kABCMdDs2DMYhR2UwECY2t+DLpxqJqE2ab8pI9Xd
    53  BZ3pCNodS03yHXzfeJV44lCjxoDOi9ynXLjd3w3+FowomHMEBunTepiqnbgoYtnn
    54  RW9tQyQQK5g6+/j/O1M8o71s/0loBT3vKSqNSrdlMOEGrj4yyL/Cw1NmQf1V1sGf
    55  w1QAW5xk7Br5oh8h1D+oflGWV3Y3zluuZQnA9D+vFpjL0969oFedsgr4UU8CAwEA
    56  AaMrMCkwCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwDwYDVR0RBAgwBocEfwAAATAN
    57  BgkqhkiG9w0BAQsFAAOCAQEAWbOF7TOfGiC59S50okfcS7M4gwz2kcbqOftWzcA1
    58  lT1qX6TWj7A4bVIOMAFK2tWNd4Omk6bnIAxTJdHB7b1hrBjkpt2krEGH1S8xeRRz
    59  Gs62KQwehM3fMhLvYSEqOQMETZn9AjEigYm6ohCO5obG9Gkfz7uvuv9rbIetbAmm
    60  YE9HdDv6qhCqtynpP2yad3v53idlrDnCIe9e4eKUD5uR/MIp9mEFgnMXR1m43/ya
    61  DnmddSsjtzamVvI/+2Cqjb8qT8dMHZrCBK64UwSaJsUKzSeF6yNvZKQ1yfA/NrfV
    62  P6gNULDOqtPgXFP4j+Z402gjYox1bGHjeDHh1OVSnr9jVw==
    63  -----END CERTIFICATE-----`)
    64  
    65  var backendKey = []byte(`-----BEGIN PRIVATE KEY-----
    66  MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGGsdqv5vzdt+1
    67  BTBSmbKMeBMiqg3vpm2kRniqpyOxovQIt2+wTwiGu0RSdp7aRlHeRsi/hwkBxvP3
    68  +g47G1LoWZ5p+xe2SXoodgz9mnCpDg3Rm1wOdpAAQjHQ7NgzGIUdlMBAmNrfgy6c
    69  aiahNmm/KSPV3QWd6QjaHUtN8h1833iVeOJQo8aAzovcp1y43d8N/haMKJhzBAbp
    70  03qYqp24KGLZ50VvbUMkECuYOvv4/ztTPKO9bP9JaAU97ykqjUq3ZTDhBq4+Msi/
    71  wsNTZkH9VdbBn8NUAFucZOwa+aIfIdQ/qH5Rlld2N85brmUJwPQ/rxaYy9PevaBX
    72  nbIK+FFPAgMBAAECggEAKmdZABR7gSWUxN6TdVrIySB6mBTmXsG0/lDHS1/zV/aV
    73  XbhGA+sm3BABk9UoM3iR1Y45MiXpW6QGXLH9kdFLccidC/pfHPmlWDvMlAwWyVjk
    74  xFUI41+leyiwGRRZQrag57ALZshRMT6XH4vpMODAydY4gXKJ3T8gUe+rSsfkX/Hl
    75  Ce59c8pDsV3NDy4WKy00lYZfTqBqHu10qy9W8/eVYf+RUt53nrygCesnFfmJx/P8
    76  GnHnN06QbZdpgVgbU49u+BujkjFgKH/60Ct9A19o34upXvkPOaKbABZ4dL1lUrbo
    77  e3L3vnSdgXh1oOsy/JyICmDG5M2b68h33YNa+qUEgQKBgQDs1rf1+hw75o7iDlnx
    78  E46CPC+9DkDuisWLgbUyW5KHPgropPl80uqnRxmaWpYGU/Fgyml08orpduHIWxtU
    79  0tMRKm2HoFRM010fAp3xWc/B4pt2pdRMMSjMle//4FmoNlcJ8+owmD+2eook9Qjm
    80  qN1UsQllkSoH4zx4iI+HhDJnHwKBgQDWIdGmlZqaYGhsndkco9yK+gve6W80ik4J
    81  qnjnv9ux28SBrlORn2zzfGcu5LkJw8Dp9yjZzVUiFT8VFsWVNNuJyFba227Qxrwz
    82  Hb/qvd5l2DfXHk4poyMZThzg7cxkxlVaWUIBMoGynDxQZIOypc6WmTeEG5+9W4+w
    83  NCuTKt6/0QKBgQCOgALftUUXpXmC+i+TpbixE5WFovXekRCbB8gGLKLVTLczk0+p
    84  kx4s19LH1Ik/9XHeUutwuh5qqmTfMDIZr1/fjC+q0wTl1KbK6cAuX2NpvPbdRJmf
    85  3lQ2BGELC+nmFAv6qQ/XfUOYf9JuuiBI6IGDW6HTwqwPYuIXg9MYLqpE8QKBgA/2
    86  2YCH6szTnzVp10PxW4Ho/nWSBb5vCT5jPTxZ63EpJ09bxdM3hZHplm/CkaEOvRU0
    87  XhFO46f02Y0i83waQrvU+dS7Q1nBV0qgTyybFzeUlSUulzk3dmhukGycjf59YuOn
    88  f+pC77R3PW/o7oClJ+/GYIMy5AfkCaRjX1RLf+vhAoGBANJBi0ARkhwOWbnD2urA
    89  0tPMURSYIZ+JW7ghMspbm1XV1NTreCB/llLNqUGQ7zLAmH+KyqJK8O37/oh3VHrV
    90  6jp9pqrqmibtGEIpQi4D9IM8Zo9mc8GexCf0x+11mamC+ZXjT+bvLQzbcJGnG5CL
    91  W+S7SneWTL09leh5ATNhog6s
    92  -----END PRIVATE KEY-----`)
    93  
    94  // TestGracefulShutdownForActiveHTTP2Streams checks if graceful shut down of HTTP2 server works.
    95  // It expects that all active connections will be finished (without any errors) before the server exits.
    96  //
    97  // The test sends 25 requests to the target server in parallel. Each request is held by the target server for 60s.
    98  // As soon as the target server receives the last request the test calls backendServer.Config.Shutdown which gracefully shuts down the server without interrupting any active connections.
    99  //
   100  // See more at: https://github.com/golang/go/issues/39776
   101  //
   102  // Note this test will fail on upstream golang 1.15
   103  func TestGracefulShutdownForActiveHTTP2Streams(t *testing.T) {
   104  	// set up the backend server
   105  	backendHandler := &backendHTTPHandler{}
   106  	backendServer := httptest.NewUnstartedServer(backendHandler)
   107  	backendCert, err := tls.X509KeyPair(backendCrt, backendKey)
   108  	if err != nil {
   109  		t.Fatalf("backend: invalid x509/key pair: %v", err)
   110  	}
   111  	backendServer.TLS = &tls.Config{
   112  		Certificates: []tls.Certificate{backendCert},
   113  		NextProtos:   []string{http2.NextProtoTLS},
   114  	}
   115  	backendServer.StartTLS()
   116  	defer backendServer.Close()
   117  
   118  	// set up the client
   119  	clientCACertPool := x509.NewCertPool()
   120  	clientCACertPool.AppendCertsFromPEM(backendCrt)
   121  	clientTLSConfig := &tls.Config{
   122  		RootCAs:    clientCACertPool,
   123  		NextProtos: []string{http2.NextProtoTLS},
   124  	}
   125  	client := &http.Client{}
   126  	client.Transport = &http2.Transport{
   127  		TLSClientConfig: clientTLSConfig,
   128  	}
   129  
   130  	// client request
   131  	sendRequest := func(wg *sync.WaitGroup) {
   132  		defer func() {
   133  			wg.Done()
   134  		}()
   135  
   136  		// act
   137  		resp, err := client.Get(fmt.Sprintf("https://127.0.0.1:%d", backendServer.Listener.Addr().(*net.TCPAddr).Port))
   138  		if err != nil {
   139  			t.Errorf("%v", err)
   140  			return
   141  		}
   142  
   143  		// validate
   144  		defer resp.Body.Close()
   145  		_, err = io.ReadAll(resp.Body)
   146  		if err != nil {
   147  			t.Errorf("%v", err)
   148  		}
   149  		if resp.StatusCode != 200 {
   150  			t.Errorf("unexpected HTTP staus: %v, expected: 200", resp.StatusCode)
   151  		}
   152  		expectedProto := "HTTP/2.0"
   153  		if resp.Proto != expectedProto {
   154  			t.Errorf("unexpected response proto: %v, expected: %v", resp.Proto, expectedProto)
   155  		}
   156  	}
   157  
   158  	// this function starts the graceful shutdown
   159  	go func() {
   160  		<-startServerShutdown // signal from the backend after receiving all (25) requests
   161  
   162  		backendServer.Config.Shutdown(context.Background())
   163  	}()
   164  
   165  	wg := sync.WaitGroup{}
   166  	wg.Add(25)
   167  	for i := 0; i < 25; i++ {
   168  		go sendRequest(&wg)
   169  	}
   170  	wg.Wait()
   171  
   172  	// validate backendHandler
   173  	if backendHandler.counter != 25 {
   174  		t.Errorf("the target server haven't received all expected requests, expected 25, it got %d", backendHandler.counter)
   175  	}
   176  }
   177  
   178  type backendHTTPHandler struct {
   179  	counter int
   180  }
   181  
   182  func (b *backendHTTPHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
   183  	handlerLock.Lock()
   184  	b.counter++
   185  	if b.counter == 25 {
   186  		startServerShutdown <- struct{}{}
   187  	}
   188  	handlerLock.Unlock()
   189  
   190  	time.Sleep(60 * time.Second)
   191  
   192  	w.Write([]byte("hello from the backend"))
   193  	w.WriteHeader(http.StatusOK)
   194  }