k8s.io/apiserver@v0.31.1/pkg/server/egressselector/egress_selector.go (about) 1 /* 2 Copyright 2019 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package egressselector 18 19 import ( 20 "bufio" 21 "context" 22 "crypto/tls" 23 "crypto/x509" 24 "fmt" 25 "net" 26 "net/http" 27 "net/url" 28 "os" 29 "strings" 30 "time" 31 32 "go.opentelemetry.io/otel/attribute" 33 "google.golang.org/grpc" 34 "google.golang.org/grpc/credentials/insecure" 35 36 utilnet "k8s.io/apimachinery/pkg/util/net" 37 "k8s.io/apiserver/pkg/apis/apiserver" 38 egressmetrics "k8s.io/apiserver/pkg/server/egressselector/metrics" 39 "k8s.io/component-base/metrics/legacyregistry" 40 "k8s.io/component-base/tracing" 41 "k8s.io/klog/v2" 42 client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" 43 ) 44 45 var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext 46 47 func init() { 48 client.Metrics.RegisterMetrics(legacyregistry.Registerer()) 49 } 50 51 // EgressSelector is the map of network context type to context dialer, for network egress. 52 type EgressSelector struct { 53 egressToDialer map[EgressType]utilnet.DialFunc 54 } 55 56 // EgressType is an indicator of which egress selection should be used for sending traffic. 57 // See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/1281-network-proxy/README.md#network-context 58 type EgressType int 59 60 const ( 61 // ControlPlane is the EgressType for traffic intended to go to the control plane. 62 ControlPlane EgressType = iota 63 // Etcd is the EgressType for traffic intended to go to Kubernetes persistence store. 64 Etcd 65 // Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes. 66 Cluster 67 ) 68 69 // NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent. 70 type NetworkContext struct { 71 // EgressSelectionName is the unique name of the 72 // EgressSelectorConfiguration which determines 73 // the network we route the traffic to. 74 EgressSelectionName EgressType 75 } 76 77 // Lookup is the interface to get the dialer function for the network context. 78 type Lookup func(networkContext NetworkContext) (utilnet.DialFunc, error) 79 80 // String returns the canonical string representation of the egress type 81 func (s EgressType) String() string { 82 switch s { 83 case ControlPlane: 84 return "controlplane" 85 case Etcd: 86 return "etcd" 87 case Cluster: 88 return "cluster" 89 default: 90 return "invalid" 91 } 92 } 93 94 // AsNetworkContext is a helper function to make it easy to get the basic NetworkContext objects. 95 func (s EgressType) AsNetworkContext() NetworkContext { 96 return NetworkContext{EgressSelectionName: s} 97 } 98 99 func lookupServiceName(name string) (EgressType, error) { 100 switch strings.ToLower(name) { 101 case "controlplane": 102 return ControlPlane, nil 103 case "etcd": 104 return Etcd, nil 105 case "cluster": 106 return Cluster, nil 107 } 108 return -1, fmt.Errorf("unrecognized service name %s", name) 109 } 110 111 func tunnelHTTPConnect(proxyConn net.Conn, proxyAddress, addr string) (net.Conn, error) { 112 fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1") 113 br := bufio.NewReader(proxyConn) 114 res, err := http.ReadResponse(br, nil) 115 if err != nil { 116 proxyConn.Close() 117 return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v", 118 addr, proxyAddress, err) 119 } 120 if res.StatusCode != 200 { 121 proxyConn.Close() 122 return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v", 123 proxyAddress, addr, res.StatusCode, res.Status) 124 } 125 126 // It's safe to discard the bufio.Reader here and return the 127 // original TCP conn directly because we only use this for 128 // TLS, and in TLS the client speaks first, so we know there's 129 // no unbuffered data. But we can double-check. 130 if br.Buffered() > 0 { 131 proxyConn.Close() 132 return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q", 133 br.Buffered(), proxyAddress) 134 } 135 return proxyConn, nil 136 } 137 138 type proxier interface { 139 // proxy returns a connection to addr. 140 proxy(ctx context.Context, addr string) (net.Conn, error) 141 } 142 143 var _ proxier = &httpConnectProxier{} 144 145 type httpConnectProxier struct { 146 conn net.Conn 147 proxyAddress string 148 } 149 150 func (t *httpConnectProxier) proxy(ctx context.Context, addr string) (net.Conn, error) { 151 return tunnelHTTPConnect(t.conn, t.proxyAddress, addr) 152 } 153 154 var _ proxier = &grpcProxier{} 155 156 type grpcProxier struct { 157 tunnel client.Tunnel 158 } 159 160 func (g *grpcProxier) proxy(ctx context.Context, addr string) (net.Conn, error) { 161 return g.tunnel.DialContext(ctx, "tcp", addr) 162 } 163 164 type proxyServerConnector interface { 165 // connect establishes connection to the proxy server, and returns a 166 // proxier based on the connection. 167 // 168 // The provided Context must be non-nil. The context is used for connecting to the proxy only. 169 // If the context expires before the connection is complete, an error is returned. 170 // Once successfully connected to the proxy, any expiration of the context will not affect the connection. 171 connect(context.Context) (proxier, error) 172 } 173 174 type tcpHTTPConnectConnector struct { 175 proxyAddress string 176 tlsConfig *tls.Config 177 } 178 179 func (t *tcpHTTPConnectConnector) connect(ctx context.Context) (proxier, error) { 180 d := tls.Dialer{ 181 Config: t.tlsConfig, 182 } 183 conn, err := d.DialContext(ctx, "tcp", t.proxyAddress) 184 if err != nil { 185 return nil, err 186 } 187 return &httpConnectProxier{conn: conn, proxyAddress: t.proxyAddress}, nil 188 } 189 190 type udsHTTPConnectConnector struct { 191 udsName string 192 } 193 194 func (u *udsHTTPConnectConnector) connect(ctx context.Context) (proxier, error) { 195 var d net.Dialer 196 conn, err := d.DialContext(ctx, "unix", u.udsName) 197 if err != nil { 198 return nil, err 199 } 200 return &httpConnectProxier{conn: conn, proxyAddress: u.udsName}, nil 201 } 202 203 type udsGRPCConnector struct { 204 udsName string 205 } 206 207 // connect establishes a connection to a proxy over gRPC. 208 // TODO At the moment, it does not use the provided context. 209 func (u *udsGRPCConnector) connect(_ context.Context) (proxier, error) { 210 udsName := u.udsName 211 dialOption := grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { 212 var d net.Dialer 213 c, err := d.DialContext(ctx, "unix", udsName) 214 if err != nil { 215 klog.Errorf("failed to create connection to uds name %s, error: %v", udsName, err) 216 } 217 return c, err 218 }) 219 220 // CreateSingleUseGrpcTunnel() unfortunately couples dial and connection contexts. Because of that, 221 // we cannot use ctx just for dialing and control the connection lifetime separately. 222 // See https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/357. 223 tunnelCtx := context.TODO() 224 tunnel, err := client.CreateSingleUseGrpcTunnel(tunnelCtx, udsName, dialOption, 225 grpc.WithBlock(), 226 grpc.WithReturnConnectionError(), 227 grpc.WithTimeout(30*time.Second), // matches http.DefaultTransport dial timeout 228 grpc.WithTransportCredentials(insecure.NewCredentials())) 229 if err != nil { 230 return nil, err 231 } 232 return &grpcProxier{tunnel: tunnel}, nil 233 } 234 235 type dialerCreator struct { 236 connector proxyServerConnector 237 direct bool 238 options metricsOptions 239 } 240 241 type metricsOptions struct { 242 transport string 243 protocol string 244 } 245 246 func (d *dialerCreator) createDialer() utilnet.DialFunc { 247 if d.direct { 248 return directDialer 249 } 250 return func(ctx context.Context, network, addr string) (net.Conn, error) { 251 ctx, span := tracing.Start(ctx, fmt.Sprintf("Proxy via %s protocol over %s", d.options.protocol, d.options.transport), attribute.String("address", addr)) 252 defer span.End(500 * time.Millisecond) 253 start := egressmetrics.Metrics.Clock().Now() 254 egressmetrics.Metrics.ObserveDialStart(d.options.protocol, d.options.transport) 255 proxier, err := d.connector.connect(ctx) 256 if err != nil { 257 egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageConnect) 258 return nil, err 259 } 260 conn, err := proxier.proxy(ctx, addr) 261 if err != nil { 262 egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageProxy) 263 return nil, err 264 } 265 egressmetrics.Metrics.ObserveDialLatency(egressmetrics.Metrics.Clock().Now().Sub(start), d.options.protocol, d.options.transport) 266 return conn, nil 267 } 268 } 269 270 func getTLSConfig(t *apiserver.TLSConfig) (*tls.Config, error) { 271 clientCert := t.ClientCert 272 clientKey := t.ClientKey 273 caCert := t.CABundle 274 clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey) 275 if err != nil { 276 return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err) 277 } 278 certPool := x509.NewCertPool() 279 if caCert != "" { 280 certBytes, err := os.ReadFile(caCert) 281 if err != nil { 282 return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err) 283 } 284 ok := certPool.AppendCertsFromPEM(certBytes) 285 if !ok { 286 return nil, fmt.Errorf("failed to append CA cert to the cert pool") 287 } 288 } else { 289 // Use host's root CA set instead of providing our own 290 certPool = nil 291 } 292 return &tls.Config{ 293 Certificates: []tls.Certificate{clientCerts}, 294 RootCAs: certPool, 295 }, nil 296 } 297 298 func getProxyAddress(urlString string) (string, error) { 299 proxyURL, err := url.Parse(urlString) 300 if err != nil { 301 return "", fmt.Errorf("invalid proxy server url %q: %v", urlString, err) 302 } 303 return proxyURL.Host, nil 304 } 305 306 func connectionToDialerCreator(c apiserver.Connection) (*dialerCreator, error) { 307 switch c.ProxyProtocol { 308 309 case apiserver.ProtocolHTTPConnect: 310 if c.Transport.UDS != nil { 311 return &dialerCreator{ 312 connector: &udsHTTPConnectConnector{ 313 udsName: c.Transport.UDS.UDSName, 314 }, 315 options: metricsOptions{ 316 transport: egressmetrics.TransportUDS, 317 protocol: egressmetrics.ProtocolHTTPConnect, 318 }, 319 }, nil 320 } else if c.Transport.TCP != nil { 321 tlsConfig, err := getTLSConfig(c.Transport.TCP.TLSConfig) 322 if err != nil { 323 return nil, err 324 } 325 proxyAddress, err := getProxyAddress(c.Transport.TCP.URL) 326 if err != nil { 327 return nil, err 328 } 329 return &dialerCreator{ 330 connector: &tcpHTTPConnectConnector{ 331 tlsConfig: tlsConfig, 332 proxyAddress: proxyAddress, 333 }, 334 options: metricsOptions{ 335 transport: egressmetrics.TransportTCP, 336 protocol: egressmetrics.ProtocolHTTPConnect, 337 }, 338 }, nil 339 } else { 340 return nil, fmt.Errorf("Either a TCP or UDS transport must be specified") 341 } 342 case apiserver.ProtocolGRPC: 343 if c.Transport.UDS != nil { 344 return &dialerCreator{ 345 connector: &udsGRPCConnector{ 346 udsName: c.Transport.UDS.UDSName, 347 }, 348 options: metricsOptions{ 349 transport: egressmetrics.TransportUDS, 350 protocol: egressmetrics.ProtocolGRPC, 351 }, 352 }, nil 353 } 354 return nil, fmt.Errorf("UDS transport must be specified for GRPC") 355 case apiserver.ProtocolDirect: 356 return &dialerCreator{direct: true}, nil 357 default: 358 return nil, fmt.Errorf("unrecognized service connection protocol %q", c.ProxyProtocol) 359 } 360 361 } 362 363 // NewEgressSelector configures lookup mechanism for Lookup. 364 // It does so based on a EgressSelectorConfiguration which was read at startup. 365 func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) { 366 if config == nil || config.EgressSelections == nil { 367 // No Connection Services configured, leaving the serviceMap empty, will return default dialer. 368 return nil, nil 369 } 370 cs := &EgressSelector{ 371 egressToDialer: make(map[EgressType]utilnet.DialFunc), 372 } 373 for _, service := range config.EgressSelections { 374 name, err := lookupServiceName(service.Name) 375 if err != nil { 376 return nil, err 377 } 378 dialerCreator, err := connectionToDialerCreator(service.Connection) 379 if err != nil { 380 return nil, fmt.Errorf("failed to create dialer for egressSelection %q: %v", name, err) 381 } 382 cs.egressToDialer[name] = dialerCreator.createDialer() 383 } 384 return cs, nil 385 } 386 387 // NewEgressSelectorWithMap returns a EgressSelector with the supplied EgressType to DialFunc map. 388 func NewEgressSelectorWithMap(m map[EgressType]utilnet.DialFunc) *EgressSelector { 389 if m == nil { 390 m = make(map[EgressType]utilnet.DialFunc) 391 } 392 return &EgressSelector{ 393 egressToDialer: m, 394 } 395 } 396 397 // Lookup gets the dialer function for the network context. 398 // This is configured for the Kubernetes API Server at startup. 399 func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) { 400 if cs.egressToDialer == nil { 401 // The round trip wrapper will over-ride the dialContext method appropriately 402 return nil, nil 403 } 404 405 return cs.egressToDialer[networkContext.EgressSelectionName], nil 406 }