k8s.io/apiserver@v0.31.1/pkg/server/filters/timeout.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package filters
    18  
    19  import (
    20  	"bufio"
    21  	"encoding/json"
    22  	"fmt"
    23  	"net"
    24  	"net/http"
    25  	"runtime"
    26  	"sync"
    27  	"time"
    28  
    29  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    30  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    31  	"k8s.io/apiserver/pkg/endpoints/metrics"
    32  	apirequest "k8s.io/apiserver/pkg/endpoints/request"
    33  	"k8s.io/apiserver/pkg/endpoints/responsewriter"
    34  	"k8s.io/apiserver/pkg/server/httplog"
    35  )
    36  
    37  // WithTimeoutForNonLongRunningRequests times out non-long-running requests after the time given by timeout.
    38  func WithTimeoutForNonLongRunningRequests(handler http.Handler, longRunning apirequest.LongRunningRequestCheck) http.Handler {
    39  	if longRunning == nil {
    40  		return handler
    41  	}
    42  	timeoutFunc := func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) {
    43  		// TODO unify this with apiserver.MaxInFlightLimit
    44  		ctx := req.Context()
    45  
    46  		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
    47  		if !ok {
    48  			// if this happens, the handler chain isn't setup correctly because there is no request info
    49  			return req, false, func() {}, apierrors.NewInternalError(fmt.Errorf("no request info found for request during timeout"))
    50  		}
    51  
    52  		if longRunning(req, requestInfo) {
    53  			return req, true, nil, nil
    54  		}
    55  
    56  		postTimeoutFn := func() {
    57  			metrics.RecordRequestTermination(req, requestInfo, metrics.APIServerComponent, http.StatusGatewayTimeout)
    58  		}
    59  		return req, false, postTimeoutFn, apierrors.NewTimeoutError("request did not complete within the allotted timeout", 0)
    60  	}
    61  	return WithTimeout(handler, timeoutFunc)
    62  }
    63  
    64  type timeoutFunc = func(*http.Request) (req *http.Request, longRunning bool, postTimeoutFunc func(), err *apierrors.StatusError)
    65  
    66  // WithTimeout returns an http.Handler that runs h with a timeout
    67  // determined by timeoutFunc. The new http.Handler calls h.ServeHTTP to handle
    68  // each request, but if a call runs for longer than its time limit, the
    69  // handler responds with a 504 Gateway Timeout error and the message
    70  // provided. (If msg is empty, a suitable default message will be sent.) After
    71  // the handler times out, writes by h to its http.ResponseWriter will return
    72  // http.ErrHandlerTimeout. If timeoutFunc returns a nil timeout channel, no
    73  // timeout will be enforced. recordFn is a function that will be invoked whenever
    74  // a timeout happens.
    75  func WithTimeout(h http.Handler, timeoutFunc timeoutFunc) http.Handler {
    76  	return &timeoutHandler{h, timeoutFunc}
    77  }
    78  
    79  type timeoutHandler struct {
    80  	handler http.Handler
    81  	timeout timeoutFunc
    82  }
    83  
    84  func (t *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
    85  	r, longRunning, postTimeoutFn, err := t.timeout(r)
    86  	if longRunning {
    87  		t.handler.ServeHTTP(w, r)
    88  		return
    89  	}
    90  
    91  	timeoutCh := r.Context().Done()
    92  
    93  	// resultCh is used as both errCh and stopCh
    94  	resultCh := make(chan interface{})
    95  	var tw timeoutWriter
    96  	tw, w = newTimeoutWriter(w)
    97  
    98  	// Make a copy of request and work on it in new goroutine
    99  	// to avoid race condition when accessing/modifying request (e.g. headers)
   100  	rCopy := r.Clone(r.Context())
   101  	go func() {
   102  		defer func() {
   103  			err := recover()
   104  			// do not wrap the sentinel ErrAbortHandler panic value
   105  			if err != nil && err != http.ErrAbortHandler {
   106  				// Same as stdlib http server code. Manually allocate stack
   107  				// trace buffer size to prevent excessively large logs
   108  				const size = 64 << 10
   109  				buf := make([]byte, size)
   110  				buf = buf[:runtime.Stack(buf, false)]
   111  				err = fmt.Sprintf("%v\n%s", err, buf)
   112  			}
   113  			resultCh <- err
   114  		}()
   115  		t.handler.ServeHTTP(w, rCopy)
   116  	}()
   117  	select {
   118  	case err := <-resultCh:
   119  		// panic if error occurs; stop otherwise
   120  		if err != nil {
   121  			panic(err)
   122  		}
   123  		return
   124  	case <-timeoutCh:
   125  		defer func() {
   126  			// resultCh needs to have a reader, since the function doing
   127  			// the work needs to send to it. This is defer'd to ensure it runs
   128  			// ever if the post timeout work itself panics.
   129  			go func() {
   130  				timedOutAt := time.Now()
   131  				res := <-resultCh
   132  
   133  				status := metrics.PostTimeoutHandlerOK
   134  				if res != nil {
   135  					// a non nil res indicates that there was a panic.
   136  					status = metrics.PostTimeoutHandlerPanic
   137  				}
   138  
   139  				metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceTimeoutHandler, status)
   140  				utilruntime.HandleErrorWithContext(r.Context(), nil, "Post-timeout activity", "timeElapsed", time.Since(timedOutAt), "method", r.Method, "path", r.URL.Path, "result", res)
   141  			}()
   142  		}()
   143  		httplog.SetStacktracePredicate(r.Context(), func(status int) bool {
   144  			return false
   145  		})
   146  		defer postTimeoutFn()
   147  		tw.timeout(err)
   148  	}
   149  }
   150  
   151  type timeoutWriter interface {
   152  	http.ResponseWriter
   153  	timeout(*apierrors.StatusError)
   154  }
   155  
   156  func newTimeoutWriter(w http.ResponseWriter) (timeoutWriter, http.ResponseWriter) {
   157  	base := &baseTimeoutWriter{w: w, handlerHeaders: w.Header().Clone()}
   158  	wrapped := responsewriter.WrapForHTTP1Or2(base)
   159  
   160  	return base, wrapped
   161  }
   162  
   163  var _ http.ResponseWriter = &baseTimeoutWriter{}
   164  var _ responsewriter.UserProvidedDecorator = &baseTimeoutWriter{}
   165  
   166  type baseTimeoutWriter struct {
   167  	w http.ResponseWriter
   168  
   169  	// headers written by the normal handler
   170  	handlerHeaders http.Header
   171  
   172  	mu sync.Mutex
   173  	// if the timeout handler has timeout
   174  	timedOut bool
   175  	// if this timeout writer has wrote header
   176  	wroteHeader bool
   177  	// if this timeout writer has been hijacked
   178  	hijacked bool
   179  }
   180  
   181  func (tw *baseTimeoutWriter) Unwrap() http.ResponseWriter {
   182  	return tw.w
   183  }
   184  
   185  func (tw *baseTimeoutWriter) Header() http.Header {
   186  	tw.mu.Lock()
   187  	defer tw.mu.Unlock()
   188  
   189  	if tw.timedOut {
   190  		return http.Header{}
   191  	}
   192  
   193  	return tw.handlerHeaders
   194  }
   195  
   196  func (tw *baseTimeoutWriter) Write(p []byte) (int, error) {
   197  	tw.mu.Lock()
   198  	defer tw.mu.Unlock()
   199  
   200  	if tw.timedOut {
   201  		return 0, http.ErrHandlerTimeout
   202  	}
   203  	if tw.hijacked {
   204  		return 0, http.ErrHijacked
   205  	}
   206  
   207  	if !tw.wroteHeader {
   208  		copyHeaders(tw.w.Header(), tw.handlerHeaders)
   209  		tw.wroteHeader = true
   210  	}
   211  	return tw.w.Write(p)
   212  }
   213  
   214  func (tw *baseTimeoutWriter) Flush() {
   215  	tw.mu.Lock()
   216  	defer tw.mu.Unlock()
   217  
   218  	if tw.timedOut {
   219  		return
   220  	}
   221  
   222  	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
   223  	// http.Flusher if the inner object (tw.w) implements http.Flusher.
   224  	tw.w.(http.Flusher).Flush()
   225  }
   226  
   227  func (tw *baseTimeoutWriter) WriteHeader(code int) {
   228  	tw.mu.Lock()
   229  	defer tw.mu.Unlock()
   230  
   231  	if tw.timedOut || tw.wroteHeader || tw.hijacked {
   232  		return
   233  	}
   234  
   235  	copyHeaders(tw.w.Header(), tw.handlerHeaders)
   236  	tw.wroteHeader = true
   237  	tw.w.WriteHeader(code)
   238  }
   239  
   240  func copyHeaders(dst, src http.Header) {
   241  	for k, v := range src {
   242  		dst[k] = v
   243  	}
   244  }
   245  
   246  func (tw *baseTimeoutWriter) timeout(err *apierrors.StatusError) {
   247  	tw.mu.Lock()
   248  	defer tw.mu.Unlock()
   249  
   250  	tw.timedOut = true
   251  
   252  	// The timeout writer has not been used by the inner handler.
   253  	// We can safely timeout the HTTP request by sending by a timeout
   254  	// handler
   255  	if !tw.wroteHeader && !tw.hijacked {
   256  		tw.w.WriteHeader(http.StatusGatewayTimeout)
   257  		enc := json.NewEncoder(tw.w)
   258  		enc.Encode(&err.ErrStatus)
   259  	} else {
   260  		// The timeout writer has been used by the inner handler. There is
   261  		// no way to timeout the HTTP request at the point. We have to shutdown
   262  		// the connection for HTTP1 or reset stream for HTTP2.
   263  		//
   264  		// Note from the golang's docs:
   265  		// If ServeHTTP panics, the server (the caller of ServeHTTP) assumes
   266  		// that the effect of the panic was isolated to the active request.
   267  		// It recovers the panic, logs a stack trace to the server error log,
   268  		// and either closes the network connection or sends an HTTP/2
   269  		// RST_STREAM, depending on the HTTP protocol. To abort a handler so
   270  		// the client sees an interrupted response but the server doesn't log
   271  		// an error, panic with the value ErrAbortHandler.
   272  		//
   273  		// We are throwing http.ErrAbortHandler deliberately so that a client is notified and to suppress a not helpful stacktrace in the logs
   274  		panic(http.ErrAbortHandler)
   275  	}
   276  }
   277  
   278  func (tw *baseTimeoutWriter) CloseNotify() <-chan bool {
   279  	tw.mu.Lock()
   280  	defer tw.mu.Unlock()
   281  
   282  	if tw.timedOut {
   283  		done := make(chan bool)
   284  		close(done)
   285  		return done
   286  	}
   287  
   288  	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
   289  	// http.CloseNotifier if the inner object (tw.w) implements http.CloseNotifier.
   290  	return tw.w.(http.CloseNotifier).CloseNotify()
   291  }
   292  
   293  func (tw *baseTimeoutWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
   294  	tw.mu.Lock()
   295  	defer tw.mu.Unlock()
   296  
   297  	if tw.timedOut {
   298  		return nil, nil, http.ErrHandlerTimeout
   299  	}
   300  
   301  	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
   302  	// http.Hijacker if the inner object (tw.w) implements http.Hijacker.
   303  	conn, rw, err := tw.w.(http.Hijacker).Hijack()
   304  	if err == nil {
   305  		tw.hijacked = true
   306  	}
   307  	return conn, rw, err
   308  }