k8s.io/apiserver@v0.31.1/pkg/util/proxy/streamtunnel.go (about)

     1  /*
     2  Copyright 2024 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package proxy
    18  
    19  import (
    20  	"bufio"
    21  	"bytes"
    22  	"context"
    23  	"errors"
    24  	"fmt"
    25  	"net"
    26  	"net/http"
    27  	"strconv"
    28  	"strings"
    29  	"sync"
    30  	"time"
    31  
    32  	gwebsocket "github.com/gorilla/websocket"
    33  
    34  	"k8s.io/apimachinery/pkg/util/httpstream"
    35  	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
    36  	"k8s.io/apimachinery/pkg/util/httpstream/wsstream"
    37  	utilnet "k8s.io/apimachinery/pkg/util/net"
    38  	constants "k8s.io/apimachinery/pkg/util/portforward"
    39  	"k8s.io/apiserver/pkg/util/proxy/metrics"
    40  	"k8s.io/client-go/tools/portforward"
    41  	"k8s.io/klog/v2"
    42  )
    43  
    44  // TunnelingHandler is a handler which tunnels SPDY through WebSockets.
    45  type TunnelingHandler struct {
    46  	// Used to communicate between upstream SPDY and downstream tunnel.
    47  	upgradeHandler http.Handler
    48  }
    49  
    50  // NewTunnelingHandler is used to create the tunnel between an upstream
    51  // SPDY connection and a downstream tunneling connection through the stored
    52  // UpgradeAwareProxy.
    53  func NewTunnelingHandler(upgradeHandler http.Handler) *TunnelingHandler {
    54  	return &TunnelingHandler{upgradeHandler: upgradeHandler}
    55  }
    56  
    57  // ServeHTTP uses the upgradeHandler to tunnel between a downstream tunneling
    58  // connection and an upstream SPDY connection. The tunneling connection is
    59  // a wrapped WebSockets connection which communicates SPDY framed data. In the
    60  // case the upstream upgrade fails, we delegate communication to the passed
    61  // in "w" ResponseWriter.
    62  func (h *TunnelingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
    63  	klog.V(4).Infoln("TunnelingHandler ServeHTTP")
    64  
    65  	spdyProtocols := spdyProtocolsFromWebsocketProtocols(req)
    66  	if len(spdyProtocols) == 0 {
    67  		metrics.IncStreamTunnelRequest(req.Context(), strconv.Itoa(http.StatusBadRequest))
    68  		http.Error(w, "unable to upgrade: no tunneling spdy protocols provided", http.StatusBadRequest)
    69  		return
    70  	}
    71  
    72  	spdyRequest := createSPDYRequest(req, spdyProtocols...)
    73  
    74  	// The fields "w" and "conn" are mutually exclusive. Either a successful upgrade occurs
    75  	// and the "conn" is hijacked and used in the subsequent upgradeHandler, or
    76  	// the upgrade failed, and "w" is the delegate used for the non-upgrade response.
    77  	writer := &tunnelingResponseWriter{
    78  		// "w" is used in the non-upgrade error cases called in the upgradeHandler.
    79  		w: w,
    80  		// "conn" is returned in the successful upgrade case when hijacked in the upgradeHandler.
    81  		conn: &headerInterceptingConn{
    82  			initializableConn: &tunnelingWebsocketUpgraderConn{
    83  				w:   w,
    84  				req: req,
    85  			},
    86  		},
    87  	}
    88  
    89  	klog.V(4).Infoln("Tunnel spdy through websockets using the UpgradeAwareProxy")
    90  	h.upgradeHandler.ServeHTTP(writer, spdyRequest)
    91  }
    92  
    93  // createSPDYRequest modifies the passed request to remove
    94  // WebSockets headers and add SPDY upgrade information, including
    95  // spdy protocols acceptable to the client.
    96  func createSPDYRequest(req *http.Request, spdyProtocols ...string) *http.Request {
    97  	clone := utilnet.CloneRequest(req)
    98  	// Clean up the websocket headers from the http request.
    99  	clone.Header.Del(wsstream.WebSocketProtocolHeader)
   100  	clone.Header.Del("Sec-Websocket-Key")
   101  	clone.Header.Del("Sec-Websocket-Version")
   102  	clone.Header.Del(httpstream.HeaderUpgrade)
   103  	// Update the http request for an upstream SPDY upgrade.
   104  	clone.Method = "POST"
   105  	clone.Body = nil // Remove the request body which is unused.
   106  	clone.Header.Set(httpstream.HeaderUpgrade, spdy.HeaderSpdy31)
   107  	clone.Header.Del(httpstream.HeaderProtocolVersion)
   108  	for i := range spdyProtocols {
   109  		clone.Header.Add(httpstream.HeaderProtocolVersion, spdyProtocols[i])
   110  	}
   111  	return clone
   112  }
   113  
   114  // spdyProtocolsFromWebsocketProtocols returns a list of spdy protocols by filtering
   115  // to Kubernetes websocket subprotocols prefixed with "SPDY/3.1+", then removing the prefix
   116  func spdyProtocolsFromWebsocketProtocols(req *http.Request) []string {
   117  	var spdyProtocols []string
   118  	for _, protocol := range gwebsocket.Subprotocols(req) {
   119  		if strings.HasPrefix(protocol, constants.WebsocketsSPDYTunnelingPrefix) && strings.HasSuffix(protocol, constants.KubernetesSuffix) {
   120  			spdyProtocols = append(spdyProtocols, strings.TrimPrefix(protocol, constants.WebsocketsSPDYTunnelingPrefix))
   121  		}
   122  	}
   123  	return spdyProtocols
   124  }
   125  
   126  var _ http.ResponseWriter = &tunnelingResponseWriter{}
   127  var _ http.Hijacker = &tunnelingResponseWriter{}
   128  
   129  // tunnelingResponseWriter implements the http.ResponseWriter and http.Hijacker interfaces.
   130  // Only non-upgrade responses can be written using WriteHeader() and Write().
   131  // Once Write or WriteHeader is called, Hijack returns an error.
   132  // Once Hijack is called, Write, WriteHeader, and Hijack return errors.
   133  type tunnelingResponseWriter struct {
   134  	// w is used to delegate Header(), WriteHeader(), and Write() calls
   135  	w http.ResponseWriter
   136  	// conn is returned from Hijack()
   137  	conn net.Conn
   138  	// mu guards writes
   139  	mu sync.Mutex
   140  	// wrote tracks whether WriteHeader or Write has been called
   141  	written bool
   142  	// hijacked tracks whether Hijack has been called
   143  	hijacked bool
   144  }
   145  
   146  // Hijack returns a delegate "net.Conn".
   147  // An error is returned if Write(), WriteHeader(), or Hijack() was previously called.
   148  // The returned bufio.ReadWriter is always nil.
   149  func (w *tunnelingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
   150  	w.mu.Lock()
   151  	defer w.mu.Unlock()
   152  	if w.written {
   153  		klog.Errorf("Hijack called after write")
   154  		return nil, nil, errors.New("connection has already been written to")
   155  	}
   156  	if w.hijacked {
   157  		klog.Errorf("Hijack called after hijack")
   158  		return nil, nil, errors.New("connection has already been hijacked")
   159  	}
   160  	w.hijacked = true
   161  	klog.V(6).Infof("Hijack returning websocket tunneling net.Conn")
   162  	return w.conn, nil, nil
   163  }
   164  
   165  // Header is delegated to the stored "http.ResponseWriter".
   166  func (w *tunnelingResponseWriter) Header() http.Header {
   167  	return w.w.Header()
   168  }
   169  
   170  // Write is delegated to the stored "http.ResponseWriter".
   171  func (w *tunnelingResponseWriter) Write(p []byte) (int, error) {
   172  	w.mu.Lock()
   173  	defer w.mu.Unlock()
   174  	if w.hijacked {
   175  		klog.Errorf("Write called after hijack")
   176  		return 0, http.ErrHijacked
   177  	}
   178  	w.written = true
   179  	return w.w.Write(p)
   180  }
   181  
   182  // WriteHeader is delegated to the stored "http.ResponseWriter".
   183  func (w *tunnelingResponseWriter) WriteHeader(statusCode int) {
   184  	w.mu.Lock()
   185  	defer w.mu.Unlock()
   186  	if w.written {
   187  		klog.Errorf("WriteHeader called after write")
   188  		return
   189  	}
   190  	if w.hijacked {
   191  		klog.Errorf("WriteHeader called after hijack")
   192  		return
   193  	}
   194  	w.written = true
   195  
   196  	if statusCode == http.StatusSwitchingProtocols {
   197  		// 101 upgrade responses must come via the hijacked connection, not WriteHeader
   198  		klog.Errorf("WriteHeader called with 101 upgrade")
   199  		http.Error(w.w, "unexpected upgrade", http.StatusInternalServerError)
   200  		return
   201  	}
   202  
   203  	// pass through non-upgrade responses we don't need to translate
   204  	w.w.WriteHeader(statusCode)
   205  }
   206  
   207  // headerInterceptingConn wraps the tunneling "net.Conn" to drain the
   208  // HTTP response status/headers from the upstream SPDY connection, then use
   209  // that to decide how to initialize the delegate connection for writes.
   210  type headerInterceptingConn struct {
   211  	// initializableConn is delegated to for all net.Conn methods.
   212  	// initializableConn.Write() is not called until response headers have been read
   213  	// and initializableConn#InitializeWrite() has been called with the result.
   214  	initializableConn
   215  
   216  	lock          sync.Mutex
   217  	headerBuffer  []byte
   218  	initialized   bool
   219  	initializeErr error
   220  }
   221  
   222  // initializableConn is a connection that will be initialized before any calls to Write are made
   223  type initializableConn interface {
   224  	net.Conn
   225  	// InitializeWrite is called when the backend response headers have been read.
   226  	// backendResponse contains the parsed headers.
   227  	// backendResponseBytes are the raw bytes the headers were parsed from.
   228  	InitializeWrite(backendResponse *http.Response, backendResponseBytes []byte) error
   229  }
   230  
   231  const maxHeaderBytes = 1 << 20
   232  
   233  // token for normal header / body separation (\r\n\r\n, but go tolerates the leading \r being absent)
   234  var lfCRLF = []byte("\n\r\n")
   235  
   236  // token for header / body separation without \r (which go tolerates)
   237  var lfLF = []byte("\n\n")
   238  
   239  // Write intercepts to initially swallow the HTTP response, then
   240  // delegate to the tunneling "net.Conn" once the response has been
   241  // seen and processed.
   242  func (h *headerInterceptingConn) Write(b []byte) (int, error) {
   243  	h.lock.Lock()
   244  	defer h.lock.Unlock()
   245  
   246  	if h.initializeErr != nil {
   247  		return 0, h.initializeErr
   248  	}
   249  	if h.initialized {
   250  		return h.initializableConn.Write(b)
   251  	}
   252  
   253  	// Guard against excessive buffering
   254  	if len(h.headerBuffer)+len(b) > maxHeaderBytes {
   255  		return 0, fmt.Errorf("header size limit exceeded")
   256  	}
   257  
   258  	// Accumulate into headerBuffer
   259  	h.headerBuffer = append(h.headerBuffer, b...)
   260  
   261  	// Attempt to parse http response headers
   262  	var headerBytes, bodyBytes []byte
   263  	if i := bytes.Index(h.headerBuffer, lfCRLF); i != -1 {
   264  		// headers terminated with \n\r\n
   265  		headerBytes = h.headerBuffer[0 : i+len(lfCRLF)]
   266  		bodyBytes = h.headerBuffer[i+len(lfCRLF):]
   267  	} else if i := bytes.Index(h.headerBuffer, lfLF); i != -1 {
   268  		// headers terminated with \n\n (which go tolerates)
   269  		headerBytes = h.headerBuffer[0 : i+len(lfLF)]
   270  		bodyBytes = h.headerBuffer[i+len(lfLF):]
   271  	} else {
   272  		// don't yet have a complete set of headers yet
   273  		return len(b), nil
   274  	}
   275  	resp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(headerBytes)), nil)
   276  	if err != nil {
   277  		klog.Errorf("invalid headers: %v", err)
   278  		h.initializeErr = err
   279  		return len(b), err
   280  	}
   281  	resp.Body.Close() //nolint:errcheck
   282  
   283  	h.headerBuffer = nil
   284  	h.initialized = true
   285  	h.initializeErr = h.initializableConn.InitializeWrite(resp, headerBytes)
   286  	if h.initializeErr != nil {
   287  		return len(b), h.initializeErr
   288  	}
   289  	if len(bodyBytes) > 0 {
   290  		_, err = h.initializableConn.Write(bodyBytes)
   291  	}
   292  	return len(b), err
   293  }
   294  
   295  type tunnelingWebsocketUpgraderConn struct {
   296  	// req is the websocket request, used for upgrading
   297  	req *http.Request
   298  	// w is the websocket writer, used for upgrading and writing error responses
   299  	w http.ResponseWriter
   300  
   301  	// lock guards conn and err
   302  	lock sync.RWMutex
   303  	// if conn is non-nil, InitializeWrite succeeded
   304  	conn net.Conn
   305  	// if err is non-nil, InitializeWrite failed or Close was called before InitializeWrite
   306  	err error
   307  }
   308  
   309  func (u *tunnelingWebsocketUpgraderConn) InitializeWrite(backendResponse *http.Response, backendResponseBytes []byte) (err error) {
   310  	// make sure we close a connection we open in error cases
   311  	var conn net.Conn
   312  	defer func() {
   313  		if err != nil && conn != nil {
   314  			conn.Close() //nolint:errcheck
   315  		}
   316  	}()
   317  
   318  	u.lock.Lock()
   319  	defer u.lock.Unlock()
   320  	if u.conn != nil {
   321  		return fmt.Errorf("InitializeWrite already called")
   322  	}
   323  	if u.err != nil {
   324  		return u.err
   325  	}
   326  
   327  	if backendResponse.StatusCode == http.StatusSwitchingProtocols {
   328  		connectionHeader := strings.ToLower(backendResponse.Header.Get(httpstream.HeaderConnection))
   329  		upgradeHeader := strings.ToLower(backendResponse.Header.Get(httpstream.HeaderUpgrade))
   330  		if !strings.Contains(connectionHeader, strings.ToLower(httpstream.HeaderUpgrade)) || !strings.Contains(upgradeHeader, strings.ToLower(spdy.HeaderSpdy31)) {
   331  			klog.Errorf("unable to upgrade: missing upgrade headers in response: %#v", backendResponse.Header)
   332  			u.err = fmt.Errorf("unable to upgrade: missing upgrade headers in response")
   333  			metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(http.StatusInternalServerError))
   334  			http.Error(u.w, u.err.Error(), http.StatusInternalServerError)
   335  			return u.err
   336  		}
   337  
   338  		// Translate the server's chosen SPDY protocol into the tunneled websocket protocol for the handshake
   339  		var serverWebsocketProtocols []string
   340  		if backendSPDYProtocol := strings.TrimSpace(backendResponse.Header.Get(httpstream.HeaderProtocolVersion)); backendSPDYProtocol != "" {
   341  			serverWebsocketProtocols = []string{constants.WebsocketsSPDYTunnelingPrefix + backendSPDYProtocol}
   342  		} else {
   343  			serverWebsocketProtocols = []string{}
   344  		}
   345  
   346  		// Try to upgrade the websocket connection.
   347  		// Beyond this point, we don't need to write errors to the response.
   348  		var upgrader = gwebsocket.Upgrader{
   349  			CheckOrigin:  func(r *http.Request) bool { return true },
   350  			Subprotocols: serverWebsocketProtocols,
   351  		}
   352  		conn, err := upgrader.Upgrade(u.w, u.req, nil)
   353  		if err != nil {
   354  			klog.Errorf("error upgrading websocket connection: %v", err)
   355  			metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(http.StatusInternalServerError))
   356  			u.err = err
   357  			return u.err
   358  		}
   359  
   360  		klog.V(4).Infof("websocket connection created: %s", conn.Subprotocol())
   361  		metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(http.StatusSwitchingProtocols))
   362  		u.conn = portforward.NewTunnelingConnection("server", conn)
   363  		return nil
   364  	}
   365  
   366  	// anything other than an upgrade should pass through the backend response
   367  	klog.Errorf("SPDY upgrade failed: %s", backendResponse.Status)
   368  	metrics.IncStreamTunnelRequest(context.Background(), strconv.Itoa(backendResponse.StatusCode))
   369  
   370  	// try to hijack
   371  	conn, _, err = u.w.(http.Hijacker).Hijack()
   372  	if err != nil {
   373  		klog.Errorf("Unable to hijack response: %v", err)
   374  		u.err = err
   375  		return u.err
   376  	}
   377  	// replay the backend response bytes to the hijacked conn
   378  	conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) //nolint:errcheck
   379  	_, err = conn.Write(backendResponseBytes)
   380  	if err != nil {
   381  		u.err = err
   382  		return u.err
   383  	}
   384  	u.conn = conn
   385  	return nil
   386  }
   387  
   388  func (u *tunnelingWebsocketUpgraderConn) Read(b []byte) (n int, err error) {
   389  	u.lock.RLock()
   390  	defer u.lock.RUnlock()
   391  	if u.conn != nil {
   392  		return u.conn.Read(b)
   393  	}
   394  	if u.err != nil {
   395  		return 0, u.err
   396  	}
   397  	// return empty read without blocking until we are initialized
   398  	return 0, nil
   399  }
   400  func (u *tunnelingWebsocketUpgraderConn) Write(b []byte) (n int, err error) {
   401  	u.lock.RLock()
   402  	defer u.lock.RUnlock()
   403  	if u.conn != nil {
   404  		return u.conn.Write(b)
   405  	}
   406  	if u.err != nil {
   407  		return 0, u.err
   408  	}
   409  	return 0, fmt.Errorf("Write called before Initialize")
   410  }
   411  func (u *tunnelingWebsocketUpgraderConn) Close() error {
   412  	u.lock.Lock()
   413  	defer u.lock.Unlock()
   414  	if u.conn != nil {
   415  		return u.conn.Close()
   416  	}
   417  	if u.err != nil {
   418  		return u.err
   419  	}
   420  	// record that we closed so we don't write again or try to initialize
   421  	u.err = fmt.Errorf("connection closed")
   422  	// write a response
   423  	http.Error(u.w, u.err.Error(), http.StatusInternalServerError)
   424  	return nil
   425  }
   426  func (u *tunnelingWebsocketUpgraderConn) LocalAddr() net.Addr {
   427  	u.lock.RLock()
   428  	defer u.lock.RUnlock()
   429  	if u.conn != nil {
   430  		return u.conn.LocalAddr()
   431  	}
   432  	return noopAddr{}
   433  }
   434  func (u *tunnelingWebsocketUpgraderConn) RemoteAddr() net.Addr {
   435  	u.lock.RLock()
   436  	defer u.lock.RUnlock()
   437  	if u.conn != nil {
   438  		return u.conn.RemoteAddr()
   439  	}
   440  	return noopAddr{}
   441  }
   442  func (u *tunnelingWebsocketUpgraderConn) SetDeadline(t time.Time) error {
   443  	u.lock.RLock()
   444  	defer u.lock.RUnlock()
   445  	if u.conn != nil {
   446  		return u.conn.SetDeadline(t)
   447  	}
   448  	return nil
   449  }
   450  func (u *tunnelingWebsocketUpgraderConn) SetReadDeadline(t time.Time) error {
   451  	u.lock.RLock()
   452  	defer u.lock.RUnlock()
   453  	if u.conn != nil {
   454  		return u.conn.SetReadDeadline(t)
   455  	}
   456  	return nil
   457  }
   458  func (u *tunnelingWebsocketUpgraderConn) SetWriteDeadline(t time.Time) error {
   459  	u.lock.RLock()
   460  	defer u.lock.RUnlock()
   461  	if u.conn != nil {
   462  		return u.conn.SetWriteDeadline(t)
   463  	}
   464  	return nil
   465  }
   466  
   467  type noopAddr struct{}
   468  
   469  func (n noopAddr) Network() string { return "" }
   470  func (n noopAddr) String() string  { return "" }