k8s.io/apiserver@v0.31.1/pkg/util/peerproxy/peerproxy_handler.go (about)

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package peerproxy
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"math/rand"
    24  	"net"
    25  	"net/http"
    26  	"net/url"
    27  	"strconv"
    28  	"strings"
    29  	"sync"
    30  	"sync/atomic"
    31  
    32  	"k8s.io/api/apiserverinternal/v1alpha1"
    33  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    34  	"k8s.io/apimachinery/pkg/runtime"
    35  	schema "k8s.io/apimachinery/pkg/runtime/schema"
    36  	"k8s.io/apimachinery/pkg/util/proxy"
    37  	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
    38  	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
    39  	apirequest "k8s.io/apiserver/pkg/endpoints/request"
    40  	"k8s.io/apiserver/pkg/endpoints/responsewriter"
    41  	"k8s.io/apiserver/pkg/reconcilers"
    42  	"k8s.io/apiserver/pkg/storageversion"
    43  	"k8s.io/apiserver/pkg/util/peerproxy/metrics"
    44  	apiserverproxyutil "k8s.io/apiserver/pkg/util/proxy"
    45  	"k8s.io/client-go/tools/cache"
    46  	"k8s.io/client-go/transport"
    47  	"k8s.io/klog/v2"
    48  )
    49  
    50  const (
    51  	PeerProxiedHeader = "x-kubernetes-peer-proxied"
    52  )
    53  
    54  type peerProxyHandler struct {
    55  	name string
    56  	// StorageVersion informer used to fetch apiserver ids than can serve a resource
    57  	storageversionInformer cache.SharedIndexInformer
    58  
    59  	// StorageVersion manager used to ensure it has finished updating storageversions before
    60  	// we start handling external requests
    61  	storageversionManager storageversion.Manager
    62  
    63  	// proxy transport
    64  	proxyTransport http.RoundTripper
    65  
    66  	// identity for this server
    67  	serverId string
    68  
    69  	// reconciler that is used to fetch host port of peer apiserver when proxying request to a peer
    70  	reconciler reconcilers.PeerEndpointLeaseReconciler
    71  
    72  	serializer runtime.NegotiatedSerializer
    73  
    74  	// SyncMap for storing an up to date copy of the storageversions and apiservers that can serve them
    75  	// This map is populated using the StorageVersion informer
    76  	// This map has key set to GVR and value being another SyncMap
    77  	// The nested SyncMap has key set to apiserver id and value set to boolean
    78  	// The nested maps are created to have a "Set" like structure to store unique apiserver ids
    79  	// for a given GVR
    80  	svMap sync.Map
    81  
    82  	finishedSync atomic.Bool
    83  }
    84  
    85  type serviceableByResponse struct {
    86  	locallyServiceable            bool
    87  	errorFetchingAddressFromLease bool
    88  	peerEndpoints                 []string
    89  }
    90  
    91  // responder implements rest.Responder for assisting a connector in writing objects or errors.
    92  type responder struct {
    93  	w   http.ResponseWriter
    94  	ctx context.Context
    95  }
    96  
    97  func (h *peerProxyHandler) HasFinishedSync() bool {
    98  	return h.finishedSync.Load()
    99  }
   100  
   101  func (h *peerProxyHandler) WaitForCacheSync(stopCh <-chan struct{}) error {
   102  
   103  	ok := cache.WaitForNamedCacheSync("unknown-version-proxy", stopCh, h.storageversionInformer.HasSynced, h.storageversionManager.Completed)
   104  	if !ok {
   105  		return fmt.Errorf("error while waiting for initial cache sync")
   106  	}
   107  	klog.V(3).Infof("setting finishedSync to true")
   108  	h.finishedSync.Store(true)
   109  	return nil
   110  }
   111  
   112  // WrapHandler will fetch the apiservers that can serve the request and either serve it locally
   113  // or route it to a peer
   114  func (h *peerProxyHandler) WrapHandler(handler http.Handler) http.Handler {
   115  	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   116  		ctx := r.Context()
   117  		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
   118  
   119  		if !ok {
   120  			responsewriters.InternalError(w, r, errors.New("no RequestInfo found in the context"))
   121  			return
   122  		}
   123  
   124  		// Allow non-resource requests
   125  		if !requestInfo.IsResourceRequest {
   126  			klog.V(3).Infof("Not a resource request skipping proxying")
   127  			handler.ServeHTTP(w, r)
   128  			return
   129  		}
   130  
   131  		// Request has already been proxied once, it must be served locally
   132  		if r.Header.Get(PeerProxiedHeader) == "true" {
   133  			klog.V(3).Infof("Already rerouted once, skipping proxying to peer")
   134  			handler.ServeHTTP(w, r)
   135  			return
   136  		}
   137  
   138  		// StorageVersion Informers and/or StorageVersionManager is not synced yet, pass request to next handler
   139  		// This will happen for self requests from the kube-apiserver because we have a poststarthook
   140  		// to ensure that external requests are not served until the StorageVersion Informer and
   141  		// StorageVersionManager has synced
   142  		if !h.HasFinishedSync() {
   143  			handler.ServeHTTP(w, r)
   144  			return
   145  		}
   146  
   147  		gvr := schema.GroupVersionResource{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion, Resource: requestInfo.Resource}
   148  		if requestInfo.APIGroup == "" {
   149  			gvr.Group = "core"
   150  		}
   151  
   152  		// find servers that are capable of serving this request
   153  		serviceableByResp, err := h.findServiceableByServers(gvr, h.serverId, h.reconciler)
   154  		if err != nil {
   155  			// this means that resource is an aggregated API or a CR since it wasn't found in SV informer cache, pass as it is
   156  			handler.ServeHTTP(w, r)
   157  			return
   158  		}
   159  		// found the gvr locally, pass request to the next handler in local apiserver
   160  		if serviceableByResp.locallyServiceable {
   161  			handler.ServeHTTP(w, r)
   162  			return
   163  		}
   164  
   165  		gv := schema.GroupVersion{Group: gvr.Group, Version: gvr.Version}
   166  
   167  		if serviceableByResp.errorFetchingAddressFromLease {
   168  			klog.ErrorS(err, "error fetching ip and port of remote server while proxying")
   169  			responsewriters.ErrorNegotiated(apierrors.NewServiceUnavailable("Error getting ip and port info of the remote server while proxying"), h.serializer, gv, w, r)
   170  			return
   171  		}
   172  
   173  		// no apiservers were found that could serve the request, pass request to
   174  		// next handler, that should eventually serve 404
   175  
   176  		// TODO: maintain locally serviceable GVRs somewhere so that we dont have to
   177  		// consult the storageversion-informed map for those
   178  		if len(serviceableByResp.peerEndpoints) == 0 {
   179  			klog.Errorf(fmt.Sprintf("GVR %v is not served by anything in this cluster", gvr))
   180  			handler.ServeHTTP(w, r)
   181  			return
   182  		}
   183  
   184  		// otherwise, randomly select an apiserver and proxy request to it
   185  		rand := rand.Intn(len(serviceableByResp.peerEndpoints))
   186  		destServerHostPort := serviceableByResp.peerEndpoints[rand]
   187  		h.proxyRequestToDestinationAPIServer(r, w, destServerHostPort)
   188  
   189  	})
   190  }
   191  
   192  func (h *peerProxyHandler) findServiceableByServers(gvr schema.GroupVersionResource, localAPIServerId string, reconciler reconcilers.PeerEndpointLeaseReconciler) (serviceableByResponse, error) {
   193  
   194  	apiserversi, ok := h.svMap.Load(gvr)
   195  
   196  	// no value found for the requested gvr in svMap
   197  	if !ok || apiserversi == nil {
   198  		return serviceableByResponse{}, fmt.Errorf("no StorageVersions found for the GVR: %v", gvr)
   199  	}
   200  	apiservers := apiserversi.(*sync.Map)
   201  	response := serviceableByResponse{}
   202  	var peerServerEndpoints []string
   203  	apiservers.Range(func(key, value interface{}) bool {
   204  		apiserverKey := key.(string)
   205  		if apiserverKey == localAPIServerId {
   206  			response.errorFetchingAddressFromLease = true
   207  			response.locallyServiceable = true
   208  			// stop iteration
   209  			return false
   210  		}
   211  
   212  		hostPort, err := reconciler.GetEndpoint(apiserverKey)
   213  		if err != nil {
   214  			response.errorFetchingAddressFromLease = true
   215  			klog.Errorf("failed to get peer ip from storage lease for server %s", apiserverKey)
   216  			// continue with iteration
   217  			return true
   218  		}
   219  		// check ip format
   220  		_, _, err = net.SplitHostPort(hostPort)
   221  		if err != nil {
   222  			response.errorFetchingAddressFromLease = true
   223  			klog.Errorf("invalid address found for server %s", apiserverKey)
   224  			// continue with iteration
   225  			return true
   226  		}
   227  		peerServerEndpoints = append(peerServerEndpoints, hostPort)
   228  		// continue with iteration
   229  		return true
   230  	})
   231  
   232  	response.peerEndpoints = peerServerEndpoints
   233  	return response, nil
   234  }
   235  
   236  func (h *peerProxyHandler) proxyRequestToDestinationAPIServer(req *http.Request, rw http.ResponseWriter, host string) {
   237  	user, ok := apirequest.UserFrom(req.Context())
   238  	if !ok {
   239  		klog.Errorf("failed to get user info from request")
   240  		return
   241  	}
   242  
   243  	// write a new location based on the existing request pointed at the target service
   244  	location := &url.URL{}
   245  	location.Scheme = "https"
   246  	location.Host = host
   247  	location.Path = req.URL.Path
   248  	location.RawQuery = req.URL.Query().Encode()
   249  
   250  	newReq, cancelFn := apiserverproxyutil.NewRequestForProxy(location, req)
   251  	newReq.Header.Add(PeerProxiedHeader, "true")
   252  	defer cancelFn()
   253  
   254  	proxyRoundTripper := transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), h.proxyTransport)
   255  
   256  	delegate := &epmetrics.ResponseWriterDelegator{ResponseWriter: rw}
   257  	w := responsewriter.WrapForHTTP1Or2(delegate)
   258  
   259  	handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, false, &responder{w: w, ctx: req.Context()})
   260  	handler.ServeHTTP(w, newReq)
   261  	// Increment the count of proxied requests
   262  	metrics.IncPeerProxiedRequest(req.Context(), strconv.Itoa(delegate.Status()))
   263  }
   264  
   265  func (r *responder) Error(w http.ResponseWriter, req *http.Request, err error) {
   266  	klog.Errorf("Error while proxying request to destination apiserver: %v", err)
   267  	http.Error(w, err.Error(), http.StatusServiceUnavailable)
   268  }
   269  
   270  // Adds a storageversion object to SVMap
   271  func (h *peerProxyHandler) addSV(obj interface{}) {
   272  	sv, ok := obj.(*v1alpha1.StorageVersion)
   273  	if !ok {
   274  		klog.Errorf("Invalid StorageVersion provided to addSV()")
   275  		return
   276  	}
   277  	h.updateSVMap(nil, sv)
   278  }
   279  
   280  // Updates the SVMap to delete old storageversion and add new storageversion
   281  func (h *peerProxyHandler) updateSV(oldObj interface{}, newObj interface{}) {
   282  	oldSV, ok := oldObj.(*v1alpha1.StorageVersion)
   283  	if !ok {
   284  		klog.Errorf("Invalid StorageVersion provided to updateSV()")
   285  		return
   286  	}
   287  	newSV, ok := newObj.(*v1alpha1.StorageVersion)
   288  	if !ok {
   289  		klog.Errorf("Invalid StorageVersion provided to updateSV()")
   290  		return
   291  	}
   292  	h.updateSVMap(oldSV, newSV)
   293  }
   294  
   295  // Deletes a storageversion object from SVMap
   296  func (h *peerProxyHandler) deleteSV(obj interface{}) {
   297  	sv, ok := obj.(*v1alpha1.StorageVersion)
   298  	if !ok {
   299  		klog.Errorf("Invalid StorageVersion provided to deleteSV()")
   300  		return
   301  	}
   302  	h.updateSVMap(sv, nil)
   303  }
   304  
   305  // Delete old storageversion, add new storagversion
   306  func (h *peerProxyHandler) updateSVMap(oldSV *v1alpha1.StorageVersion, newSV *v1alpha1.StorageVersion) {
   307  	if oldSV != nil {
   308  		// delete old SV entries
   309  		h.deleteSVFromMap(oldSV)
   310  	}
   311  	if newSV != nil {
   312  		// add new SV entries
   313  		h.addSVToMap(newSV)
   314  	}
   315  }
   316  
   317  func (h *peerProxyHandler) deleteSVFromMap(sv *v1alpha1.StorageVersion) {
   318  	// The name of storageversion is <group>.<resource>
   319  	splitInd := strings.LastIndex(sv.Name, ".")
   320  	group := sv.Name[:splitInd]
   321  	resource := sv.Name[splitInd+1:]
   322  
   323  	gvr := schema.GroupVersionResource{Group: group, Resource: resource}
   324  	for _, gr := range sv.Status.StorageVersions {
   325  		for _, version := range gr.ServedVersions {
   326  			versionSplit := strings.Split(version, "/")
   327  			if len(versionSplit) == 2 {
   328  				version = versionSplit[1]
   329  			}
   330  			gvr.Version = version
   331  			h.svMap.Delete(gvr)
   332  		}
   333  	}
   334  }
   335  
   336  func (h *peerProxyHandler) addSVToMap(sv *v1alpha1.StorageVersion) {
   337  	// The name of storageversion is <group>.<resource>
   338  	splitInd := strings.LastIndex(sv.Name, ".")
   339  	group := sv.Name[:splitInd]
   340  	resource := sv.Name[splitInd+1:]
   341  
   342  	gvr := schema.GroupVersionResource{Group: group, Resource: resource}
   343  	for _, gr := range sv.Status.StorageVersions {
   344  		for _, version := range gr.ServedVersions {
   345  
   346  			// some versions have groups included in them, so get rid of the groups
   347  			versionSplit := strings.Split(version, "/")
   348  			if len(versionSplit) == 2 {
   349  				version = versionSplit[1]
   350  			}
   351  			gvr.Version = version
   352  			apiserversi, _ := h.svMap.LoadOrStore(gvr, &sync.Map{})
   353  			apiservers := apiserversi.(*sync.Map)
   354  			apiservers.Store(gr.APIServerID, true)
   355  		}
   356  	}
   357  }