k8s.io/apiserver@v0.31.1/pkg/util/peerproxy/peerproxy_handler.go (about) 1 /* 2 Copyright 2023 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package peerproxy 18 19 import ( 20 "context" 21 "errors" 22 "fmt" 23 "math/rand" 24 "net" 25 "net/http" 26 "net/url" 27 "strconv" 28 "strings" 29 "sync" 30 "sync/atomic" 31 32 "k8s.io/api/apiserverinternal/v1alpha1" 33 apierrors "k8s.io/apimachinery/pkg/api/errors" 34 "k8s.io/apimachinery/pkg/runtime" 35 schema "k8s.io/apimachinery/pkg/runtime/schema" 36 "k8s.io/apimachinery/pkg/util/proxy" 37 "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" 38 epmetrics "k8s.io/apiserver/pkg/endpoints/metrics" 39 apirequest "k8s.io/apiserver/pkg/endpoints/request" 40 "k8s.io/apiserver/pkg/endpoints/responsewriter" 41 "k8s.io/apiserver/pkg/reconcilers" 42 "k8s.io/apiserver/pkg/storageversion" 43 "k8s.io/apiserver/pkg/util/peerproxy/metrics" 44 apiserverproxyutil "k8s.io/apiserver/pkg/util/proxy" 45 "k8s.io/client-go/tools/cache" 46 "k8s.io/client-go/transport" 47 "k8s.io/klog/v2" 48 ) 49 50 const ( 51 PeerProxiedHeader = "x-kubernetes-peer-proxied" 52 ) 53 54 type peerProxyHandler struct { 55 name string 56 // StorageVersion informer used to fetch apiserver ids than can serve a resource 57 storageversionInformer cache.SharedIndexInformer 58 59 // StorageVersion manager used to ensure it has finished updating storageversions before 60 // we start handling external requests 61 storageversionManager storageversion.Manager 62 63 // proxy transport 64 proxyTransport http.RoundTripper 65 66 // identity for this server 67 serverId string 68 69 // reconciler that is used to fetch host port of peer apiserver when proxying request to a peer 70 reconciler reconcilers.PeerEndpointLeaseReconciler 71 72 serializer runtime.NegotiatedSerializer 73 74 // SyncMap for storing an up to date copy of the storageversions and apiservers that can serve them 75 // This map is populated using the StorageVersion informer 76 // This map has key set to GVR and value being another SyncMap 77 // The nested SyncMap has key set to apiserver id and value set to boolean 78 // The nested maps are created to have a "Set" like structure to store unique apiserver ids 79 // for a given GVR 80 svMap sync.Map 81 82 finishedSync atomic.Bool 83 } 84 85 type serviceableByResponse struct { 86 locallyServiceable bool 87 errorFetchingAddressFromLease bool 88 peerEndpoints []string 89 } 90 91 // responder implements rest.Responder for assisting a connector in writing objects or errors. 92 type responder struct { 93 w http.ResponseWriter 94 ctx context.Context 95 } 96 97 func (h *peerProxyHandler) HasFinishedSync() bool { 98 return h.finishedSync.Load() 99 } 100 101 func (h *peerProxyHandler) WaitForCacheSync(stopCh <-chan struct{}) error { 102 103 ok := cache.WaitForNamedCacheSync("unknown-version-proxy", stopCh, h.storageversionInformer.HasSynced, h.storageversionManager.Completed) 104 if !ok { 105 return fmt.Errorf("error while waiting for initial cache sync") 106 } 107 klog.V(3).Infof("setting finishedSync to true") 108 h.finishedSync.Store(true) 109 return nil 110 } 111 112 // WrapHandler will fetch the apiservers that can serve the request and either serve it locally 113 // or route it to a peer 114 func (h *peerProxyHandler) WrapHandler(handler http.Handler) http.Handler { 115 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 116 ctx := r.Context() 117 requestInfo, ok := apirequest.RequestInfoFrom(ctx) 118 119 if !ok { 120 responsewriters.InternalError(w, r, errors.New("no RequestInfo found in the context")) 121 return 122 } 123 124 // Allow non-resource requests 125 if !requestInfo.IsResourceRequest { 126 klog.V(3).Infof("Not a resource request skipping proxying") 127 handler.ServeHTTP(w, r) 128 return 129 } 130 131 // Request has already been proxied once, it must be served locally 132 if r.Header.Get(PeerProxiedHeader) == "true" { 133 klog.V(3).Infof("Already rerouted once, skipping proxying to peer") 134 handler.ServeHTTP(w, r) 135 return 136 } 137 138 // StorageVersion Informers and/or StorageVersionManager is not synced yet, pass request to next handler 139 // This will happen for self requests from the kube-apiserver because we have a poststarthook 140 // to ensure that external requests are not served until the StorageVersion Informer and 141 // StorageVersionManager has synced 142 if !h.HasFinishedSync() { 143 handler.ServeHTTP(w, r) 144 return 145 } 146 147 gvr := schema.GroupVersionResource{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion, Resource: requestInfo.Resource} 148 if requestInfo.APIGroup == "" { 149 gvr.Group = "core" 150 } 151 152 // find servers that are capable of serving this request 153 serviceableByResp, err := h.findServiceableByServers(gvr, h.serverId, h.reconciler) 154 if err != nil { 155 // this means that resource is an aggregated API or a CR since it wasn't found in SV informer cache, pass as it is 156 handler.ServeHTTP(w, r) 157 return 158 } 159 // found the gvr locally, pass request to the next handler in local apiserver 160 if serviceableByResp.locallyServiceable { 161 handler.ServeHTTP(w, r) 162 return 163 } 164 165 gv := schema.GroupVersion{Group: gvr.Group, Version: gvr.Version} 166 167 if serviceableByResp.errorFetchingAddressFromLease { 168 klog.ErrorS(err, "error fetching ip and port of remote server while proxying") 169 responsewriters.ErrorNegotiated(apierrors.NewServiceUnavailable("Error getting ip and port info of the remote server while proxying"), h.serializer, gv, w, r) 170 return 171 } 172 173 // no apiservers were found that could serve the request, pass request to 174 // next handler, that should eventually serve 404 175 176 // TODO: maintain locally serviceable GVRs somewhere so that we dont have to 177 // consult the storageversion-informed map for those 178 if len(serviceableByResp.peerEndpoints) == 0 { 179 klog.Errorf(fmt.Sprintf("GVR %v is not served by anything in this cluster", gvr)) 180 handler.ServeHTTP(w, r) 181 return 182 } 183 184 // otherwise, randomly select an apiserver and proxy request to it 185 rand := rand.Intn(len(serviceableByResp.peerEndpoints)) 186 destServerHostPort := serviceableByResp.peerEndpoints[rand] 187 h.proxyRequestToDestinationAPIServer(r, w, destServerHostPort) 188 189 }) 190 } 191 192 func (h *peerProxyHandler) findServiceableByServers(gvr schema.GroupVersionResource, localAPIServerId string, reconciler reconcilers.PeerEndpointLeaseReconciler) (serviceableByResponse, error) { 193 194 apiserversi, ok := h.svMap.Load(gvr) 195 196 // no value found for the requested gvr in svMap 197 if !ok || apiserversi == nil { 198 return serviceableByResponse{}, fmt.Errorf("no StorageVersions found for the GVR: %v", gvr) 199 } 200 apiservers := apiserversi.(*sync.Map) 201 response := serviceableByResponse{} 202 var peerServerEndpoints []string 203 apiservers.Range(func(key, value interface{}) bool { 204 apiserverKey := key.(string) 205 if apiserverKey == localAPIServerId { 206 response.errorFetchingAddressFromLease = true 207 response.locallyServiceable = true 208 // stop iteration 209 return false 210 } 211 212 hostPort, err := reconciler.GetEndpoint(apiserverKey) 213 if err != nil { 214 response.errorFetchingAddressFromLease = true 215 klog.Errorf("failed to get peer ip from storage lease for server %s", apiserverKey) 216 // continue with iteration 217 return true 218 } 219 // check ip format 220 _, _, err = net.SplitHostPort(hostPort) 221 if err != nil { 222 response.errorFetchingAddressFromLease = true 223 klog.Errorf("invalid address found for server %s", apiserverKey) 224 // continue with iteration 225 return true 226 } 227 peerServerEndpoints = append(peerServerEndpoints, hostPort) 228 // continue with iteration 229 return true 230 }) 231 232 response.peerEndpoints = peerServerEndpoints 233 return response, nil 234 } 235 236 func (h *peerProxyHandler) proxyRequestToDestinationAPIServer(req *http.Request, rw http.ResponseWriter, host string) { 237 user, ok := apirequest.UserFrom(req.Context()) 238 if !ok { 239 klog.Errorf("failed to get user info from request") 240 return 241 } 242 243 // write a new location based on the existing request pointed at the target service 244 location := &url.URL{} 245 location.Scheme = "https" 246 location.Host = host 247 location.Path = req.URL.Path 248 location.RawQuery = req.URL.Query().Encode() 249 250 newReq, cancelFn := apiserverproxyutil.NewRequestForProxy(location, req) 251 newReq.Header.Add(PeerProxiedHeader, "true") 252 defer cancelFn() 253 254 proxyRoundTripper := transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), h.proxyTransport) 255 256 delegate := &epmetrics.ResponseWriterDelegator{ResponseWriter: rw} 257 w := responsewriter.WrapForHTTP1Or2(delegate) 258 259 handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, false, &responder{w: w, ctx: req.Context()}) 260 handler.ServeHTTP(w, newReq) 261 // Increment the count of proxied requests 262 metrics.IncPeerProxiedRequest(req.Context(), strconv.Itoa(delegate.Status())) 263 } 264 265 func (r *responder) Error(w http.ResponseWriter, req *http.Request, err error) { 266 klog.Errorf("Error while proxying request to destination apiserver: %v", err) 267 http.Error(w, err.Error(), http.StatusServiceUnavailable) 268 } 269 270 // Adds a storageversion object to SVMap 271 func (h *peerProxyHandler) addSV(obj interface{}) { 272 sv, ok := obj.(*v1alpha1.StorageVersion) 273 if !ok { 274 klog.Errorf("Invalid StorageVersion provided to addSV()") 275 return 276 } 277 h.updateSVMap(nil, sv) 278 } 279 280 // Updates the SVMap to delete old storageversion and add new storageversion 281 func (h *peerProxyHandler) updateSV(oldObj interface{}, newObj interface{}) { 282 oldSV, ok := oldObj.(*v1alpha1.StorageVersion) 283 if !ok { 284 klog.Errorf("Invalid StorageVersion provided to updateSV()") 285 return 286 } 287 newSV, ok := newObj.(*v1alpha1.StorageVersion) 288 if !ok { 289 klog.Errorf("Invalid StorageVersion provided to updateSV()") 290 return 291 } 292 h.updateSVMap(oldSV, newSV) 293 } 294 295 // Deletes a storageversion object from SVMap 296 func (h *peerProxyHandler) deleteSV(obj interface{}) { 297 sv, ok := obj.(*v1alpha1.StorageVersion) 298 if !ok { 299 klog.Errorf("Invalid StorageVersion provided to deleteSV()") 300 return 301 } 302 h.updateSVMap(sv, nil) 303 } 304 305 // Delete old storageversion, add new storagversion 306 func (h *peerProxyHandler) updateSVMap(oldSV *v1alpha1.StorageVersion, newSV *v1alpha1.StorageVersion) { 307 if oldSV != nil { 308 // delete old SV entries 309 h.deleteSVFromMap(oldSV) 310 } 311 if newSV != nil { 312 // add new SV entries 313 h.addSVToMap(newSV) 314 } 315 } 316 317 func (h *peerProxyHandler) deleteSVFromMap(sv *v1alpha1.StorageVersion) { 318 // The name of storageversion is <group>.<resource> 319 splitInd := strings.LastIndex(sv.Name, ".") 320 group := sv.Name[:splitInd] 321 resource := sv.Name[splitInd+1:] 322 323 gvr := schema.GroupVersionResource{Group: group, Resource: resource} 324 for _, gr := range sv.Status.StorageVersions { 325 for _, version := range gr.ServedVersions { 326 versionSplit := strings.Split(version, "/") 327 if len(versionSplit) == 2 { 328 version = versionSplit[1] 329 } 330 gvr.Version = version 331 h.svMap.Delete(gvr) 332 } 333 } 334 } 335 336 func (h *peerProxyHandler) addSVToMap(sv *v1alpha1.StorageVersion) { 337 // The name of storageversion is <group>.<resource> 338 splitInd := strings.LastIndex(sv.Name, ".") 339 group := sv.Name[:splitInd] 340 resource := sv.Name[splitInd+1:] 341 342 gvr := schema.GroupVersionResource{Group: group, Resource: resource} 343 for _, gr := range sv.Status.StorageVersions { 344 for _, version := range gr.ServedVersions { 345 346 // some versions have groups included in them, so get rid of the groups 347 versionSplit := strings.Split(version, "/") 348 if len(versionSplit) == 2 { 349 version = versionSplit[1] 350 } 351 gvr.Version = version 352 apiserversi, _ := h.svMap.LoadOrStore(gvr, &sync.Map{}) 353 apiservers := apiserversi.(*sync.Map) 354 apiservers.Store(gr.APIServerID, true) 355 } 356 } 357 }