k8s.io/apiserver@v0.31.1/pkg/util/proxy/streamtunnel.go (about) 1 /* 2 Copyright 2024 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package proxy 18 19 import ( 20 "bufio" 21 "bytes" 22 "context" 23 "errors" 24 "fmt" 25 "net" 26 "net/http" 27 "strconv" 28 "strings" 29 "sync" 30 "time" 31 32 gwebsocket "github.com/gorilla/websocket" 33 34 "k8s.io/apimachinery/pkg/util/httpstream" 35 "k8s.io/apimachinery/pkg/util/httpstream/spdy" 36 "k8s.io/apimachinery/pkg/util/httpstream/wsstream" 37 utilnet "k8s.io/apimachinery/pkg/util/net" 38 constants "k8s.io/apimachinery/pkg/util/portforward" 39 "k8s.io/apiserver/pkg/util/proxy/metrics" 40 "k8s.io/client-go/tools/portforward" 41 "k8s.io/klog/v2" 42 ) 43 44 // TunnelingHandler is a handler which tunnels SPDY through WebSockets. 45 type TunnelingHandler struct { 46 // Used to communicate between upstream SPDY and downstream tunnel. 47 upgradeHandler http.Handler 48 } 49 50 // NewTunnelingHandler is used to create the tunnel between an upstream 51 // SPDY connection and a downstream tunneling connection through the stored 52 // UpgradeAwareProxy. 53 func NewTunnelingHandler(upgradeHandler http.Handler) *TunnelingHandler { 54 return &TunnelingHandler{upgradeHandler: upgradeHandler} 55 } 56 57 // ServeHTTP uses the upgradeHandler to tunnel between a downstream tunneling 58 // connection and an upstream SPDY connection. The tunneling connection is 59 // a wrapped WebSockets connection which communicates SPDY framed data. In the 60 // case the upstream upgrade fails, we delegate communication to the passed 61 // in "w" ResponseWriter. 62 func (h *TunnelingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { 63 klog.V(4).Infoln("TunnelingHandler ServeHTTP") 64 65 spdyProtocols := spdyProtocolsFromWebsocketProtocols(req) 66 if len(spdyProtocols) == 0 { 67 metrics.IncStreamTunnelRequest(req.Context(), strconv.Itoa(http.StatusBadRequest)) 68 http.Error(w, "unable to upgrade: no tunneling spdy protocols provided", http.StatusBadRequest) 69 return 70 } 71 72 spdyRequest := createSPDYRequest(req, spdyProtocols...) 73 74 // The fields "w" and "conn" are mutually exclusive. Either a successful upgrade occurs 75 // and the "conn" is hijacked and used in the subsequent upgradeHandler, or 76 // the upgrade failed, and "w" is the delegate used for the non-upgrade response. 77 writer := &tunnelingResponseWriter{ 78 // "w" is used in the non-upgrade error cases called in the upgradeHandler. 79 w: w, 80 // "conn" is returned in the successful upgrade case when hijacked in the upgradeHandler. 81 conn: &headerInterceptingConn{ 82 initializableConn: &tunnelingWebsocketUpgraderConn{ 83 w: w, 84 req: req, 85 }, 86 }, 87 } 88 89 klog.V(4).Infoln("Tunnel spdy through websockets using the UpgradeAwareProxy") 90 h.upgradeHandler.ServeHTTP(writer, spdyRequest) 91 } 92 93 // createSPDYRequest modifies the passed request to remove 94 // WebSockets headers and add SPDY upgrade information, including 95 // spdy protocols acceptable to the client. 96 func createSPDYRequest(req *http.Request, spdyProtocols ...string) *http.Request { 97 clone := utilnet.CloneRequest(req) 98 // Clean up the websocket headers from the http request. 99 clone.Header.Del(wsstream.WebSocketProtocolHeader) 100 clone.Header.Del("Sec-Websocket-Key") 101 clone.Header.Del("Sec-Websocket-Version") 102 clone.Header.Del(httpstream.HeaderUpgrade) 103 // Update the http request for an upstream SPDY upgrade. 104 clone.Method = "POST" 105 clone.Body = nil // Remove the request body which is unused. 106 clone.Header.Set(httpstream.HeaderUpgrade, spdy.HeaderSpdy31) 107 clone.Header.Del(httpstream.HeaderProtocolVersion) 108 for i := range spdyProtocols { 109 clone.Header.Add(httpstream.HeaderProtocolVersion, spdyProtocols[i]) 110 } 111 return clone 112 } 113 114 // spdyProtocolsFromWebsocketProtocols returns a list of spdy protocols by filtering 115 // to Kubernetes websocket subprotocols prefixed with "SPDY/3.1+", then removing the prefix 116 func spdyProtocolsFromWebsocketProtocols(req *http.Request) []string { 117 var spdyProtocols []string 118 for _, protocol := range gwebsocket.Subprotocols(req) { 119 if strings.HasPrefix(protocol, constants.WebsocketsSPDYTunnelingPrefix) && strings.HasSuffix(protocol, constants.KubernetesSuffix) { 120 spdyProtocols = append(spdyProtocols, strings.TrimPrefix(protocol, constants.WebsocketsSPDYTunnelingPrefix)) 121 } 122 } 123 return spdyProtocols 124 } 125 126 var _ http.ResponseWriter = &tunnelingResponseWriter{} 127 var _ http.Hijacker = &tunnelingResponseWriter{} 128 129 // tunnelingResponseWriter implements the http.ResponseWriter and http.Hijacker interfaces. 130 // Only non-upgrade responses can be written using WriteHeader() and Write(). 131 // Once Write or WriteHeader is called, Hijack returns an error. 132 // Once Hijack is called, Write, WriteHeader, and Hijack return errors. 133 type tunnelingResponseWriter struct { 134 // w is used to delegate Header(), WriteHeader(), and Write() calls 135 w http.ResponseWriter 136 // conn is returned from Hijack() 137 conn net.Conn 138 // mu guards writes 139 mu sync.Mutex 140 // wrote tracks whether WriteHeader or Write has been called 141 written bool 142 // hijacked tracks whether Hijack has been called 143 hijacked bool 144 } 145 146 // Hijack returns a delegate "net.Conn". 147 // An error is returned if Write(), WriteHeader(), or Hijack() was previously called. 148 // The returned bufio.ReadWriter is always nil. 149 func (w *tunnelingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { 150 w.mu.Lock() 151 defer w.mu.Unlock() 152 if w.written { 153 klog.Errorf("Hijack called after write") 154 return nil, nil, errors.New("connection has already been written to") 155 } 156 if w.hijacked { 157 klog.Errorf("Hijack called after hijack") 158 return nil, nil, errors.New("connection has already been hijacked") 159 } 160 w.hijacked = true 161 klog.V(6).Infof("Hijack returning websocket tunneling net.Conn") 162 return w.conn, nil, nil 163 } 164 165 // Header is delegated to the stored "http.ResponseWriter". 166 func (w *tunnelingResponseWriter) Header() http.Header { 167 return w.w.Header() 168 } 169 170 // Write is delegated to the stored "http.ResponseWriter". 171 func (w *tunnelingResponseWriter) Write(p []byte) (int, error) { 172 w.mu.Lock() 173 defer w.mu.Unlock() 174 if w.hijacked { 175 klog.Errorf("Write called after hijack") 176 return 0, http.ErrHijacked 177 } 178 w.written = true 179 return w.w.Write(p) 180 } 181 182 // WriteHeader is delegated to the stored "http.ResponseWriter". 183 func (w *tunnelingResponseWriter) WriteHeader(statusCode int) { 184 w.mu.Lock() 185 defer w.mu.Unlock() 186 if w.written { 187 klog.Errorf("WriteHeader called after write") 188 return 189 } 190 if w.hijacked { 191 klog.Errorf("WriteHeader called after hijack") 192 return 193 } 194 w.written = true 195 196 if statusCode == http.StatusSwitchingProtocols { 197 // 101 upgrade responses must come via the hijacked connection, not WriteHeader 198 klog.Errorf("WriteHeader called with 101 upgrade") 199 http.Error(w.w, "unexpected upgrade", http.StatusInternalServerError) 200 return 201 } 202 203 // pass through non-upgrade responses we don't need to translate 204 w.w.WriteHeader(statusCode) 205 } 206 207 // headerInterceptingConn wraps the tunneling "net.Conn" to drain the 208 // HTTP response status/headers from the upstream SPDY connection, then use 209 // that to decide how to initialize the delegate connection for writes. 210 type headerInterceptingConn struct { 211 // initializableConn is delegated to for all net.Conn methods. 212 // initializableConn.Write() is not called until response headers have been read 213 // and initializableConn#InitializeWrite() has been called with the result. 214 initializableConn 215 216 lock sync.Mutex 217 headerBuffer []byte 218 initialized bool 219 initializeErr error 220 } 221 222 // initializableConn is a connection that will be initialized before any calls to Write are made 223 type initializableConn interface { 224 net.Conn 225 // InitializeWrite is called when the backend response headers have been read. 226 // backendResponse contains the parsed headers. 227 // backendResponseBytes are the raw bytes the headers were parsed from. 228 InitializeWrite(backendResponse *http.Response, backendResponseBytes []byte) error 229 } 230 231 const maxHeaderBytes = 1 << 20 232 233 // token for normal header / body separation (\r\n\r\n, but go tolerates the leading \r being absent) 234 var lfCRLF = []byte("\n\r\n") 235 236 // token for header / body separation without \r (which go tolerates) 237 var lfLF = []byte("\n\n") 238 239 // Write intercepts to initially swallow the HTTP response, then 240 // delegate to the tunneling "net.Conn" once the response has been 241 // seen and processed. 242 func (h *headerInterceptingConn) Write(b []byte) (int, error) { 243 h.lock.Lock() 244 defer h.lock.Unlock() 245 246 if h.initializeErr != nil { 247 return 0, h.initializeErr 248 } 249 if h.initialized { 250 return h.initializableConn.Write(b) 251 } 252 253 // Guard against excessive buffering 254 if len(h.headerBuffer)+len(b) > maxHeaderBytes { 255 return 0, fmt.Errorf("header size limit exceeded") 256 } 257 258 // Accumulate into headerBuffer 259 h.headerBuffer = append(h.headerBuffer, b...) 260 261 // Attempt to parse http response headers 262 var headerBytes, bodyBytes []byte 263 if i := bytes.Index(h.headerBuffer, lfCRLF); i != -1 { 264 // headers terminated with \n\r\n 265 headerBytes = h.headerBuffer[0 : i+len(lfCRLF)] 266 bodyBytes = h.headerBuffer[i+len(lfCRLF):] 267 } else if i := bytes.Index(h.headerBuffer, lfLF); i != -1 { 268 // headers terminated with \n\n (which go tolerates) 269 headerBytes = h.headerBuffer[0 : i+len(lfLF)] 270 bodyBytes = h.headerBuffer[i+len(lfLF):] 271 } else { 272 // don't yet have a complete set of headers yet 273 return len(b), nil 274 } 275 resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(headerBytes)), nil) 276 if err != nil { 277 klog.Errorf("invalid headers: %v", err) 278 h.initializeErr = err 279 return len(b), err 280 } 281 resp.Body.Close() //nolint:errcheck 282 283 h.headerBuffer = nil 284 h.initialized = true 285 h.initializeErr = h.initializableConn.InitializeWrite(resp, headerBytes) 286 if h.initializeErr != nil { 287 return len(b), h.initializeErr 288 } 289 if len(bodyBytes) > 0 { 290 _, err = h.initializableConn.Write(bodyBytes) 291 } 292 return len(b), err 293 } 294 295 type tunnelingWebsocketUpgraderConn struct { 296 // req is the websocket request, used for upgrading 297 req *http.Request 298 // w is the websocket writer, used for upgrading and writing error responses 299 w http.ResponseWriter 300 301 // lock guards conn and err 302 lock sync.RWMutex 303 // if conn is non-nil, InitializeWrite succeeded 304 conn net.Conn 305 // if err is non-nil, InitializeWrite failed or Close was called before InitializeWrite 306 err error 307 } 308 309 func (u *tunnelingWebsocketUpgraderConn) InitializeWrite(backendResponse *http.Response, backendResponseBytes []byte) (err error) { 310 // make sure we close a connection we open in error cases 311 var conn net.Conn 312 defer func() { 313 if err != nil && conn != nil { 314 conn.Close() //nolint:errcheck 315 } 316 }() 317 318 u.lock.Lock() 319 defer u.lock.Unlock() 320 if u.conn != nil { 321 return fmt.Errorf("InitializeWrite already called") 322 } 323 if u.err != nil { 324 return u.err 325 } 326 327 if backendResponse.StatusCode == http.StatusSwitchingProtocols { 328 connectionHeader := strings.ToLower(backendResponse.Header.Get(httpstream.HeaderConnection)) 329 upgradeHeader := strings.ToLower(backendResponse.Header.Get(httpstream.HeaderUpgrade)) 330 if !strings.Contains(connectionHeader, strings.ToLower(httpstream.HeaderUpgrade)) || !strings.Contains(upgradeHeader, strings.ToLower(spdy.HeaderSpdy31)) { 331 klog.Errorf("unable to upgrade: missing upgrade headers in response: %#v", backendResponse.Header) 332 u.err = fmt.Errorf("unable to upgrade: missing upgrade headers in response") 333 metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(http.StatusInternalServerError)) 334 http.Error(u.w, u.err.Error(), http.StatusInternalServerError) 335 return u.err 336 } 337 338 // Translate the server's chosen SPDY protocol into the tunneled websocket protocol for the handshake 339 var serverWebsocketProtocols []string 340 if backendSPDYProtocol := strings.TrimSpace(backendResponse.Header.Get(httpstream.HeaderProtocolVersion)); backendSPDYProtocol != "" { 341 serverWebsocketProtocols = []string{constants.WebsocketsSPDYTunnelingPrefix + backendSPDYProtocol} 342 } else { 343 serverWebsocketProtocols = []string{} 344 } 345 346 // Try to upgrade the websocket connection. 347 // Beyond this point, we don't need to write errors to the response. 348 var upgrader = gwebsocket.Upgrader{ 349 CheckOrigin: func(r *http.Request) bool { return true }, 350 Subprotocols: serverWebsocketProtocols, 351 } 352 conn, err := upgrader.Upgrade(u.w, u.req, nil) 353 if err != nil { 354 klog.Errorf("error upgrading websocket connection: %v", err) 355 metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(http.StatusInternalServerError)) 356 u.err = err 357 return u.err 358 } 359 360 klog.V(4).Infof("websocket connection created: %s", conn.Subprotocol()) 361 metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(http.StatusSwitchingProtocols)) 362 u.conn = portforward.NewTunnelingConnection("server", conn) 363 return nil 364 } 365 366 // anything other than an upgrade should pass through the backend response 367 klog.Errorf("SPDY upgrade failed: %s", backendResponse.Status) 368 metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(backendResponse.StatusCode)) 369 370 // try to hijack 371 conn, _, err = u.w.(http.Hijacker).Hijack() 372 if err != nil { 373 klog.Errorf("Unable to hijack response: %v", err) 374 u.err = err 375 return u.err 376 } 377 // replay the backend response bytes to the hijacked conn 378 conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) //nolint:errcheck 379 _, err = conn.Write(backendResponseBytes) 380 if err != nil { 381 u.err = err 382 return u.err 383 } 384 u.conn = conn 385 return nil 386 } 387 388 func (u *tunnelingWebsocketUpgraderConn) Read(b []byte) (n int, err error) { 389 u.lock.RLock() 390 defer u.lock.RUnlock() 391 if u.conn != nil { 392 return u.conn.Read(b) 393 } 394 if u.err != nil { 395 return 0, u.err 396 } 397 // return empty read without blocking until we are initialized 398 return 0, nil 399 } 400 func (u *tunnelingWebsocketUpgraderConn) Write(b []byte) (n int, err error) { 401 u.lock.RLock() 402 defer u.lock.RUnlock() 403 if u.conn != nil { 404 return u.conn.Write(b) 405 } 406 if u.err != nil { 407 return 0, u.err 408 } 409 return 0, fmt.Errorf("Write called before Initialize") 410 } 411 func (u *tunnelingWebsocketUpgraderConn) Close() error { 412 u.lock.Lock() 413 defer u.lock.Unlock() 414 if u.conn != nil { 415 return u.conn.Close() 416 } 417 if u.err != nil { 418 return u.err 419 } 420 // record that we closed so we don't write again or try to initialize 421 u.err = fmt.Errorf("connection closed") 422 // write a response 423 http.Error(u.w, u.err.Error(), http.StatusInternalServerError) 424 return nil 425 } 426 func (u *tunnelingWebsocketUpgraderConn) LocalAddr() net.Addr { 427 u.lock.RLock() 428 defer u.lock.RUnlock() 429 if u.conn != nil { 430 return u.conn.LocalAddr() 431 } 432 return noopAddr{} 433 } 434 func (u *tunnelingWebsocketUpgraderConn) RemoteAddr() net.Addr { 435 u.lock.RLock() 436 defer u.lock.RUnlock() 437 if u.conn != nil { 438 return u.conn.RemoteAddr() 439 } 440 return noopAddr{} 441 } 442 func (u *tunnelingWebsocketUpgraderConn) SetDeadline(t time.Time) error { 443 u.lock.RLock() 444 defer u.lock.RUnlock() 445 if u.conn != nil { 446 return u.conn.SetDeadline(t) 447 } 448 return nil 449 } 450 func (u *tunnelingWebsocketUpgraderConn) SetReadDeadline(t time.Time) error { 451 u.lock.RLock() 452 defer u.lock.RUnlock() 453 if u.conn != nil { 454 return u.conn.SetReadDeadline(t) 455 } 456 return nil 457 } 458 func (u *tunnelingWebsocketUpgraderConn) SetWriteDeadline(t time.Time) error { 459 u.lock.RLock() 460 defer u.lock.RUnlock() 461 if u.conn != nil { 462 return u.conn.SetWriteDeadline(t) 463 } 464 return nil 465 } 466 467 type noopAddr struct{} 468 469 func (n noopAddr) Network() string { return "" } 470 func (n noopAddr) String() string { return "" }