k8s.io/apiserver@v0.31.1/pkg/server/egressselector/egress_selector.go (about)

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package egressselector
    18  
    19  import (
    20  	"bufio"
    21  	"context"
    22  	"crypto/tls"
    23  	"crypto/x509"
    24  	"fmt"
    25  	"net"
    26  	"net/http"
    27  	"net/url"
    28  	"os"
    29  	"strings"
    30  	"time"
    31  
    32  	"go.opentelemetry.io/otel/attribute"
    33  	"google.golang.org/grpc"
    34  	"google.golang.org/grpc/credentials/insecure"
    35  
    36  	utilnet "k8s.io/apimachinery/pkg/util/net"
    37  	"k8s.io/apiserver/pkg/apis/apiserver"
    38  	egressmetrics "k8s.io/apiserver/pkg/server/egressselector/metrics"
    39  	"k8s.io/component-base/metrics/legacyregistry"
    40  	"k8s.io/component-base/tracing"
    41  	"k8s.io/klog/v2"
    42  	client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client"
    43  )
    44  
    45  var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext
    46  
    47  func init() {
    48  	client.Metrics.RegisterMetrics(legacyregistry.Registerer())
    49  }
    50  
    51  // EgressSelector is the map of network context type to context dialer, for network egress.
    52  type EgressSelector struct {
    53  	egressToDialer map[EgressType]utilnet.DialFunc
    54  }
    55  
    56  // EgressType is an indicator of which egress selection should be used for sending traffic.
    57  // See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/1281-network-proxy/README.md#network-context
    58  type EgressType int
    59  
    60  const (
    61  	// ControlPlane is the EgressType for traffic intended to go to the control plane.
    62  	ControlPlane EgressType = iota
    63  	// Etcd is the EgressType for traffic intended to go to Kubernetes persistence store.
    64  	Etcd
    65  	// Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes.
    66  	Cluster
    67  )
    68  
    69  // NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent.
    70  type NetworkContext struct {
    71  	// EgressSelectionName is the unique name of the
    72  	// EgressSelectorConfiguration which determines
    73  	// the network we route the traffic to.
    74  	EgressSelectionName EgressType
    75  }
    76  
    77  // Lookup is the interface to get the dialer function for the network context.
    78  type Lookup func(networkContext NetworkContext) (utilnet.DialFunc, error)
    79  
    80  // String returns the canonical string representation of the egress type
    81  func (s EgressType) String() string {
    82  	switch s {
    83  	case ControlPlane:
    84  		return "controlplane"
    85  	case Etcd:
    86  		return "etcd"
    87  	case Cluster:
    88  		return "cluster"
    89  	default:
    90  		return "invalid"
    91  	}
    92  }
    93  
    94  // AsNetworkContext is a helper function to make it easy to get the basic NetworkContext objects.
    95  func (s EgressType) AsNetworkContext() NetworkContext {
    96  	return NetworkContext{EgressSelectionName: s}
    97  }
    98  
    99  func lookupServiceName(name string) (EgressType, error) {
   100  	switch strings.ToLower(name) {
   101  	case "controlplane":
   102  		return ControlPlane, nil
   103  	case "etcd":
   104  		return Etcd, nil
   105  	case "cluster":
   106  		return Cluster, nil
   107  	}
   108  	return -1, fmt.Errorf("unrecognized service name %s", name)
   109  }
   110  
   111  func tunnelHTTPConnect(proxyConn net.Conn, proxyAddress, addr string) (net.Conn, error) {
   112  	fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1")
   113  	br := bufio.NewReader(proxyConn)
   114  	res, err := http.ReadResponse(br, nil)
   115  	if err != nil {
   116  		proxyConn.Close()
   117  		return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v",
   118  			addr, proxyAddress, err)
   119  	}
   120  	if res.StatusCode != 200 {
   121  		proxyConn.Close()
   122  		return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v",
   123  			proxyAddress, addr, res.StatusCode, res.Status)
   124  	}
   125  
   126  	// It's safe to discard the bufio.Reader here and return the
   127  	// original TCP conn directly because we only use this for
   128  	// TLS, and in TLS the client speaks first, so we know there's
   129  	// no unbuffered data. But we can double-check.
   130  	if br.Buffered() > 0 {
   131  		proxyConn.Close()
   132  		return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q",
   133  			br.Buffered(), proxyAddress)
   134  	}
   135  	return proxyConn, nil
   136  }
   137  
   138  type proxier interface {
   139  	// proxy returns a connection to addr.
   140  	proxy(ctx context.Context, addr string) (net.Conn, error)
   141  }
   142  
   143  var _ proxier = &httpConnectProxier{}
   144  
   145  type httpConnectProxier struct {
   146  	conn         net.Conn
   147  	proxyAddress string
   148  }
   149  
   150  func (t *httpConnectProxier) proxy(ctx context.Context, addr string) (net.Conn, error) {
   151  	return tunnelHTTPConnect(t.conn, t.proxyAddress, addr)
   152  }
   153  
   154  var _ proxier = &grpcProxier{}
   155  
   156  type grpcProxier struct {
   157  	tunnel client.Tunnel
   158  }
   159  
   160  func (g *grpcProxier) proxy(ctx context.Context, addr string) (net.Conn, error) {
   161  	return g.tunnel.DialContext(ctx, "tcp", addr)
   162  }
   163  
   164  type proxyServerConnector interface {
   165  	// connect establishes connection to the proxy server, and returns a
   166  	// proxier based on the connection.
   167  	//
   168  	// The provided Context must be non-nil. The context is used for connecting to the proxy only.
   169  	// If the context expires before the connection is complete, an error is returned.
   170  	// Once successfully connected to the proxy, any expiration of the context will not affect the connection.
   171  	connect(context.Context) (proxier, error)
   172  }
   173  
   174  type tcpHTTPConnectConnector struct {
   175  	proxyAddress string
   176  	tlsConfig    *tls.Config
   177  }
   178  
   179  func (t *tcpHTTPConnectConnector) connect(ctx context.Context) (proxier, error) {
   180  	d := tls.Dialer{
   181  		Config: t.tlsConfig,
   182  	}
   183  	conn, err := d.DialContext(ctx, "tcp", t.proxyAddress)
   184  	if err != nil {
   185  		return nil, err
   186  	}
   187  	return &httpConnectProxier{conn: conn, proxyAddress: t.proxyAddress}, nil
   188  }
   189  
   190  type udsHTTPConnectConnector struct {
   191  	udsName string
   192  }
   193  
   194  func (u *udsHTTPConnectConnector) connect(ctx context.Context) (proxier, error) {
   195  	var d net.Dialer
   196  	conn, err := d.DialContext(ctx, "unix", u.udsName)
   197  	if err != nil {
   198  		return nil, err
   199  	}
   200  	return &httpConnectProxier{conn: conn, proxyAddress: u.udsName}, nil
   201  }
   202  
   203  type udsGRPCConnector struct {
   204  	udsName string
   205  }
   206  
   207  // connect establishes a connection to a proxy over gRPC.
   208  // TODO At the moment, it does not use the provided context.
   209  func (u *udsGRPCConnector) connect(_ context.Context) (proxier, error) {
   210  	udsName := u.udsName
   211  	dialOption := grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
   212  		var d net.Dialer
   213  		c, err := d.DialContext(ctx, "unix", udsName)
   214  		if err != nil {
   215  			klog.Errorf("failed to create connection to uds name %s, error: %v", udsName, err)
   216  		}
   217  		return c, err
   218  	})
   219  
   220  	// CreateSingleUseGrpcTunnel() unfortunately couples dial and connection contexts. Because of that,
   221  	// we cannot use ctx just for dialing and control the connection lifetime separately.
   222  	// See https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/357.
   223  	tunnelCtx := context.TODO()
   224  	tunnel, err := client.CreateSingleUseGrpcTunnel(tunnelCtx, udsName, dialOption,
   225  		grpc.WithBlock(),
   226  		grpc.WithReturnConnectionError(),
   227  		grpc.WithTimeout(30*time.Second), // matches http.DefaultTransport dial timeout
   228  		grpc.WithTransportCredentials(insecure.NewCredentials()))
   229  	if err != nil {
   230  		return nil, err
   231  	}
   232  	return &grpcProxier{tunnel: tunnel}, nil
   233  }
   234  
   235  type dialerCreator struct {
   236  	connector proxyServerConnector
   237  	direct    bool
   238  	options   metricsOptions
   239  }
   240  
   241  type metricsOptions struct {
   242  	transport string
   243  	protocol  string
   244  }
   245  
   246  func (d *dialerCreator) createDialer() utilnet.DialFunc {
   247  	if d.direct {
   248  		return directDialer
   249  	}
   250  	return func(ctx context.Context, network, addr string) (net.Conn, error) {
   251  		ctx, span := tracing.Start(ctx, fmt.Sprintf("Proxy via %s protocol over %s", d.options.protocol, d.options.transport), attribute.String("address", addr))
   252  		defer span.End(500 * time.Millisecond)
   253  		start := egressmetrics.Metrics.Clock().Now()
   254  		egressmetrics.Metrics.ObserveDialStart(d.options.protocol, d.options.transport)
   255  		proxier, err := d.connector.connect(ctx)
   256  		if err != nil {
   257  			egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageConnect)
   258  			return nil, err
   259  		}
   260  		conn, err := proxier.proxy(ctx, addr)
   261  		if err != nil {
   262  			egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageProxy)
   263  			return nil, err
   264  		}
   265  		egressmetrics.Metrics.ObserveDialLatency(egressmetrics.Metrics.Clock().Now().Sub(start), d.options.protocol, d.options.transport)
   266  		return conn, nil
   267  	}
   268  }
   269  
   270  func getTLSConfig(t *apiserver.TLSConfig) (*tls.Config, error) {
   271  	clientCert := t.ClientCert
   272  	clientKey := t.ClientKey
   273  	caCert := t.CABundle
   274  	clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey)
   275  	if err != nil {
   276  		return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err)
   277  	}
   278  	certPool := x509.NewCertPool()
   279  	if caCert != "" {
   280  		certBytes, err := os.ReadFile(caCert)
   281  		if err != nil {
   282  			return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err)
   283  		}
   284  		ok := certPool.AppendCertsFromPEM(certBytes)
   285  		if !ok {
   286  			return nil, fmt.Errorf("failed to append CA cert to the cert pool")
   287  		}
   288  	} else {
   289  		// Use host's root CA set instead of providing our own
   290  		certPool = nil
   291  	}
   292  	return &tls.Config{
   293  		Certificates: []tls.Certificate{clientCerts},
   294  		RootCAs:      certPool,
   295  	}, nil
   296  }
   297  
   298  func getProxyAddress(urlString string) (string, error) {
   299  	proxyURL, err := url.Parse(urlString)
   300  	if err != nil {
   301  		return "", fmt.Errorf("invalid proxy server url %q: %v", urlString, err)
   302  	}
   303  	return proxyURL.Host, nil
   304  }
   305  
   306  func connectionToDialerCreator(c apiserver.Connection) (*dialerCreator, error) {
   307  	switch c.ProxyProtocol {
   308  
   309  	case apiserver.ProtocolHTTPConnect:
   310  		if c.Transport.UDS != nil {
   311  			return &dialerCreator{
   312  				connector: &udsHTTPConnectConnector{
   313  					udsName: c.Transport.UDS.UDSName,
   314  				},
   315  				options: metricsOptions{
   316  					transport: egressmetrics.TransportUDS,
   317  					protocol:  egressmetrics.ProtocolHTTPConnect,
   318  				},
   319  			}, nil
   320  		} else if c.Transport.TCP != nil {
   321  			tlsConfig, err := getTLSConfig(c.Transport.TCP.TLSConfig)
   322  			if err != nil {
   323  				return nil, err
   324  			}
   325  			proxyAddress, err := getProxyAddress(c.Transport.TCP.URL)
   326  			if err != nil {
   327  				return nil, err
   328  			}
   329  			return &dialerCreator{
   330  				connector: &tcpHTTPConnectConnector{
   331  					tlsConfig:    tlsConfig,
   332  					proxyAddress: proxyAddress,
   333  				},
   334  				options: metricsOptions{
   335  					transport: egressmetrics.TransportTCP,
   336  					protocol:  egressmetrics.ProtocolHTTPConnect,
   337  				},
   338  			}, nil
   339  		} else {
   340  			return nil, fmt.Errorf("Either a TCP or UDS transport must be specified")
   341  		}
   342  	case apiserver.ProtocolGRPC:
   343  		if c.Transport.UDS != nil {
   344  			return &dialerCreator{
   345  				connector: &udsGRPCConnector{
   346  					udsName: c.Transport.UDS.UDSName,
   347  				},
   348  				options: metricsOptions{
   349  					transport: egressmetrics.TransportUDS,
   350  					protocol:  egressmetrics.ProtocolGRPC,
   351  				},
   352  			}, nil
   353  		}
   354  		return nil, fmt.Errorf("UDS transport must be specified for GRPC")
   355  	case apiserver.ProtocolDirect:
   356  		return &dialerCreator{direct: true}, nil
   357  	default:
   358  		return nil, fmt.Errorf("unrecognized service connection protocol %q", c.ProxyProtocol)
   359  	}
   360  
   361  }
   362  
   363  // NewEgressSelector configures lookup mechanism for Lookup.
   364  // It does so based on a EgressSelectorConfiguration which was read at startup.
   365  func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) {
   366  	if config == nil || config.EgressSelections == nil {
   367  		// No Connection Services configured, leaving the serviceMap empty, will return default dialer.
   368  		return nil, nil
   369  	}
   370  	cs := &EgressSelector{
   371  		egressToDialer: make(map[EgressType]utilnet.DialFunc),
   372  	}
   373  	for _, service := range config.EgressSelections {
   374  		name, err := lookupServiceName(service.Name)
   375  		if err != nil {
   376  			return nil, err
   377  		}
   378  		dialerCreator, err := connectionToDialerCreator(service.Connection)
   379  		if err != nil {
   380  			return nil, fmt.Errorf("failed to create dialer for egressSelection %q: %v", name, err)
   381  		}
   382  		cs.egressToDialer[name] = dialerCreator.createDialer()
   383  	}
   384  	return cs, nil
   385  }
   386  
   387  // NewEgressSelectorWithMap returns a EgressSelector with the supplied EgressType to DialFunc map.
   388  func NewEgressSelectorWithMap(m map[EgressType]utilnet.DialFunc) *EgressSelector {
   389  	if m == nil {
   390  		m = make(map[EgressType]utilnet.DialFunc)
   391  	}
   392  	return &EgressSelector{
   393  		egressToDialer: m,
   394  	}
   395  }
   396  
   397  // Lookup gets the dialer function for the network context.
   398  // This is configured for the Kubernetes API Server at startup.
   399  func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) {
   400  	if cs.egressToDialer == nil {
   401  		// The round trip wrapper will over-ride the dialContext method appropriately
   402  		return nil, nil
   403  	}
   404  
   405  	return cs.egressToDialer[networkContext.EgressSelectionName], nil
   406  }