k8s.io/apiserver@v0.31.1/pkg/server/filters/timeout.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package filters 18 19 import ( 20 "bufio" 21 "encoding/json" 22 "fmt" 23 "net" 24 "net/http" 25 "runtime" 26 "sync" 27 "time" 28 29 apierrors "k8s.io/apimachinery/pkg/api/errors" 30 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 31 "k8s.io/apiserver/pkg/endpoints/metrics" 32 apirequest "k8s.io/apiserver/pkg/endpoints/request" 33 "k8s.io/apiserver/pkg/endpoints/responsewriter" 34 "k8s.io/apiserver/pkg/server/httplog" 35 ) 36 37 // WithTimeoutForNonLongRunningRequests times out non-long-running requests after the time given by timeout. 38 func WithTimeoutForNonLongRunningRequests(handler http.Handler, longRunning apirequest.LongRunningRequestCheck) http.Handler { 39 if longRunning == nil { 40 return handler 41 } 42 timeoutFunc := func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) { 43 // TODO unify this with apiserver.MaxInFlightLimit 44 ctx := req.Context() 45 46 requestInfo, ok := apirequest.RequestInfoFrom(ctx) 47 if !ok { 48 // if this happens, the handler chain isn't setup correctly because there is no request info 49 return req, false, func() {}, apierrors.NewInternalError(fmt.Errorf("no request info found for request during timeout")) 50 } 51 52 if longRunning(req, requestInfo) { 53 return req, true, nil, nil 54 } 55 56 postTimeoutFn := func() { 57 metrics.RecordRequestTermination(req, requestInfo, metrics.APIServerComponent, http.StatusGatewayTimeout) 58 } 59 return req, false, postTimeoutFn, apierrors.NewTimeoutError("request did not complete within the allotted timeout", 0) 60 } 61 return WithTimeout(handler, timeoutFunc) 62 } 63 64 type timeoutFunc = func(*http.Request) (req *http.Request, longRunning bool, postTimeoutFunc func(), err *apierrors.StatusError) 65 66 // WithTimeout returns an http.Handler that runs h with a timeout 67 // determined by timeoutFunc. The new http.Handler calls h.ServeHTTP to handle 68 // each request, but if a call runs for longer than its time limit, the 69 // handler responds with a 504 Gateway Timeout error and the message 70 // provided. (If msg is empty, a suitable default message will be sent.) After 71 // the handler times out, writes by h to its http.ResponseWriter will return 72 // http.ErrHandlerTimeout. If timeoutFunc returns a nil timeout channel, no 73 // timeout will be enforced. recordFn is a function that will be invoked whenever 74 // a timeout happens. 75 func WithTimeout(h http.Handler, timeoutFunc timeoutFunc) http.Handler { 76 return &timeoutHandler{h, timeoutFunc} 77 } 78 79 type timeoutHandler struct { 80 handler http.Handler 81 timeout timeoutFunc 82 } 83 84 func (t *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 85 r, longRunning, postTimeoutFn, err := t.timeout(r) 86 if longRunning { 87 t.handler.ServeHTTP(w, r) 88 return 89 } 90 91 timeoutCh := r.Context().Done() 92 93 // resultCh is used as both errCh and stopCh 94 resultCh := make(chan interface{}) 95 var tw timeoutWriter 96 tw, w = newTimeoutWriter(w) 97 98 // Make a copy of request and work on it in new goroutine 99 // to avoid race condition when accessing/modifying request (e.g. headers) 100 rCopy := r.Clone(r.Context()) 101 go func() { 102 defer func() { 103 err := recover() 104 // do not wrap the sentinel ErrAbortHandler panic value 105 if err != nil && err != http.ErrAbortHandler { 106 // Same as stdlib http server code. Manually allocate stack 107 // trace buffer size to prevent excessively large logs 108 const size = 64 << 10 109 buf := make([]byte, size) 110 buf = buf[:runtime.Stack(buf, false)] 111 err = fmt.Sprintf("%v\n%s", err, buf) 112 } 113 resultCh <- err 114 }() 115 t.handler.ServeHTTP(w, rCopy) 116 }() 117 select { 118 case err := <-resultCh: 119 // panic if error occurs; stop otherwise 120 if err != nil { 121 panic(err) 122 } 123 return 124 case <-timeoutCh: 125 defer func() { 126 // resultCh needs to have a reader, since the function doing 127 // the work needs to send to it. This is defer'd to ensure it runs 128 // ever if the post timeout work itself panics. 129 go func() { 130 timedOutAt := time.Now() 131 res := <-resultCh 132 133 status := metrics.PostTimeoutHandlerOK 134 if res != nil { 135 // a non nil res indicates that there was a panic. 136 status = metrics.PostTimeoutHandlerPanic 137 } 138 139 metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceTimeoutHandler, status) 140 utilruntime.HandleErrorWithContext(r.Context(), nil, "Post-timeout activity", "timeElapsed", time.Since(timedOutAt), "method", r.Method, "path", r.URL.Path, "result", res) 141 }() 142 }() 143 httplog.SetStacktracePredicate(r.Context(), func(status int) bool { 144 return false 145 }) 146 defer postTimeoutFn() 147 tw.timeout(err) 148 } 149 } 150 151 type timeoutWriter interface { 152 http.ResponseWriter 153 timeout(*apierrors.StatusError) 154 } 155 156 func newTimeoutWriter(w http.ResponseWriter) (timeoutWriter, http.ResponseWriter) { 157 base := &baseTimeoutWriter{w: w, handlerHeaders: w.Header().Clone()} 158 wrapped := responsewriter.WrapForHTTP1Or2(base) 159 160 return base, wrapped 161 } 162 163 var _ http.ResponseWriter = &baseTimeoutWriter{} 164 var _ responsewriter.UserProvidedDecorator = &baseTimeoutWriter{} 165 166 type baseTimeoutWriter struct { 167 w http.ResponseWriter 168 169 // headers written by the normal handler 170 handlerHeaders http.Header 171 172 mu sync.Mutex 173 // if the timeout handler has timeout 174 timedOut bool 175 // if this timeout writer has wrote header 176 wroteHeader bool 177 // if this timeout writer has been hijacked 178 hijacked bool 179 } 180 181 func (tw *baseTimeoutWriter) Unwrap() http.ResponseWriter { 182 return tw.w 183 } 184 185 func (tw *baseTimeoutWriter) Header() http.Header { 186 tw.mu.Lock() 187 defer tw.mu.Unlock() 188 189 if tw.timedOut { 190 return http.Header{} 191 } 192 193 return tw.handlerHeaders 194 } 195 196 func (tw *baseTimeoutWriter) Write(p []byte) (int, error) { 197 tw.mu.Lock() 198 defer tw.mu.Unlock() 199 200 if tw.timedOut { 201 return 0, http.ErrHandlerTimeout 202 } 203 if tw.hijacked { 204 return 0, http.ErrHijacked 205 } 206 207 if !tw.wroteHeader { 208 copyHeaders(tw.w.Header(), tw.handlerHeaders) 209 tw.wroteHeader = true 210 } 211 return tw.w.Write(p) 212 } 213 214 func (tw *baseTimeoutWriter) Flush() { 215 tw.mu.Lock() 216 defer tw.mu.Unlock() 217 218 if tw.timedOut { 219 return 220 } 221 222 // the outer ResponseWriter object returned by WrapForHTTP1Or2 implements 223 // http.Flusher if the inner object (tw.w) implements http.Flusher. 224 tw.w.(http.Flusher).Flush() 225 } 226 227 func (tw *baseTimeoutWriter) WriteHeader(code int) { 228 tw.mu.Lock() 229 defer tw.mu.Unlock() 230 231 if tw.timedOut || tw.wroteHeader || tw.hijacked { 232 return 233 } 234 235 copyHeaders(tw.w.Header(), tw.handlerHeaders) 236 tw.wroteHeader = true 237 tw.w.WriteHeader(code) 238 } 239 240 func copyHeaders(dst, src http.Header) { 241 for k, v := range src { 242 dst[k] = v 243 } 244 } 245 246 func (tw *baseTimeoutWriter) timeout(err *apierrors.StatusError) { 247 tw.mu.Lock() 248 defer tw.mu.Unlock() 249 250 tw.timedOut = true 251 252 // The timeout writer has not been used by the inner handler. 253 // We can safely timeout the HTTP request by sending by a timeout 254 // handler 255 if !tw.wroteHeader && !tw.hijacked { 256 tw.w.WriteHeader(http.StatusGatewayTimeout) 257 enc := json.NewEncoder(tw.w) 258 enc.Encode(&err.ErrStatus) 259 } else { 260 // The timeout writer has been used by the inner handler. There is 261 // no way to timeout the HTTP request at the point. We have to shutdown 262 // the connection for HTTP1 or reset stream for HTTP2. 263 // 264 // Note from the golang's docs: 265 // If ServeHTTP panics, the server (the caller of ServeHTTP) assumes 266 // that the effect of the panic was isolated to the active request. 267 // It recovers the panic, logs a stack trace to the server error log, 268 // and either closes the network connection or sends an HTTP/2 269 // RST_STREAM, depending on the HTTP protocol. To abort a handler so 270 // the client sees an interrupted response but the server doesn't log 271 // an error, panic with the value ErrAbortHandler. 272 // 273 // We are throwing http.ErrAbortHandler deliberately so that a client is notified and to suppress a not helpful stacktrace in the logs 274 panic(http.ErrAbortHandler) 275 } 276 } 277 278 func (tw *baseTimeoutWriter) CloseNotify() <-chan bool { 279 tw.mu.Lock() 280 defer tw.mu.Unlock() 281 282 if tw.timedOut { 283 done := make(chan bool) 284 close(done) 285 return done 286 } 287 288 // the outer ResponseWriter object returned by WrapForHTTP1Or2 implements 289 // http.CloseNotifier if the inner object (tw.w) implements http.CloseNotifier. 290 return tw.w.(http.CloseNotifier).CloseNotify() 291 } 292 293 func (tw *baseTimeoutWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { 294 tw.mu.Lock() 295 defer tw.mu.Unlock() 296 297 if tw.timedOut { 298 return nil, nil, http.ErrHandlerTimeout 299 } 300 301 // the outer ResponseWriter object returned by WrapForHTTP1Or2 implements 302 // http.Hijacker if the inner object (tw.w) implements http.Hijacker. 303 conn, rw, err := tw.w.(http.Hijacker).Hijack() 304 if err == nil { 305 tw.hijacked = true 306 } 307 return conn, rw, err 308 }