k8s.io/kubernetes@v1.29.3/test/integration/apiserver/admissionwebhook/load_balance_test.go (about) 1 /* 2 Copyright 2019 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package admissionwebhook 18 19 import ( 20 "context" 21 "crypto/tls" 22 "crypto/x509" 23 "encoding/json" 24 "fmt" 25 "io" 26 "net" 27 "net/http" 28 "sync" 29 "sync/atomic" 30 "testing" 31 "time" 32 33 "k8s.io/api/admission/v1beta1" 34 admissionregistrationv1 "k8s.io/api/admissionregistration/v1" 35 corev1 "k8s.io/api/core/v1" 36 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 37 "k8s.io/apimachinery/pkg/types" 38 "k8s.io/apimachinery/pkg/util/wait" 39 clientset "k8s.io/client-go/kubernetes" 40 "k8s.io/client-go/rest" 41 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 42 "k8s.io/kubernetes/test/integration/framework" 43 ) 44 45 const ( 46 testLoadBalanceClientUsername = "webhook-balance-integration-client" 47 ) 48 49 // TestWebhookLoadBalance ensures that the admission webhook opens multiple connections to backends to satisfy concurrent requests 50 func TestWebhookLoadBalance(t *testing.T) { 51 52 roots := x509.NewCertPool() 53 if !roots.AppendCertsFromPEM(localhostCert) { 54 t.Fatal("Failed to append Cert from PEM") 55 } 56 cert, err := tls.X509KeyPair(localhostCert, localhostKey) 57 if err != nil { 58 t.Fatalf("Failed to build cert with error: %+v", err) 59 } 60 61 localListener, err := net.Listen("tcp", "127.0.0.1:0") 62 if err != nil { 63 if localListener, err = net.Listen("tcp6", "[::1]:0"); err != nil { 64 t.Fatal(err) 65 } 66 } 67 trackingListener := &connectionTrackingListener{delegate: localListener} 68 69 recorder := &connectionRecorder{} 70 handler := newLoadBalanceWebhookHandler(recorder) 71 httpServer := &http.Server{ 72 Handler: handler, 73 TLSConfig: &tls.Config{ 74 RootCAs: roots, 75 Certificates: []tls.Certificate{cert}, 76 }, 77 } 78 go func() { 79 httpServer.ServeTLS(trackingListener, "", "") 80 }() 81 defer httpServer.Close() 82 83 webhookURL := "https://" + localListener.Addr().String() 84 85 s := kubeapiservertesting.StartTestServerOrDie(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{ 86 "--disable-admission-plugins=ServiceAccount", 87 }, framework.SharedEtcd()) 88 defer s.TearDownFn() 89 90 // Configure a client with a distinct user name so that it is easy to distinguish requests 91 // made by the client from requests made by controllers. We use this to filter out requests 92 // before recording them to ensure we don't accidentally mistake requests from controllers 93 // as requests made by the client. 94 clientConfig := rest.CopyConfig(s.ClientConfig) 95 clientConfig.QPS = 100 96 clientConfig.Burst = 200 97 clientConfig.Impersonate.UserName = testLoadBalanceClientUsername 98 clientConfig.Impersonate.Groups = []string{"system:masters", "system:authenticated"} 99 client, err := clientset.NewForConfig(clientConfig) 100 if err != nil { 101 t.Fatalf("unexpected error: %v", err) 102 } 103 104 _, err = client.CoreV1().Pods("default").Create(context.TODO(), loadBalanceMarkerFixture, metav1.CreateOptions{}) 105 if err != nil { 106 t.Fatal(err) 107 } 108 109 upCh := recorder.Reset() 110 ns := "load-balance" 111 _, err = client.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, metav1.CreateOptions{}) 112 if err != nil { 113 t.Fatal(err) 114 } 115 116 fail := admissionregistrationv1.Fail 117 mutatingCfg, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), &admissionregistrationv1.MutatingWebhookConfiguration{ 118 ObjectMeta: metav1.ObjectMeta{Name: "admission.integration.test"}, 119 Webhooks: []admissionregistrationv1.MutatingWebhook{{ 120 Name: "admission.integration.test", 121 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 122 URL: &webhookURL, 123 CABundle: localhostCert, 124 }, 125 Rules: []admissionregistrationv1.RuleWithOperations{{ 126 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll}, 127 Rule: admissionregistrationv1.Rule{APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"pods"}}, 128 }}, 129 FailurePolicy: &fail, 130 AdmissionReviewVersions: []string{"v1beta1"}, 131 SideEffects: &noSideEffects, 132 }}, 133 }, metav1.CreateOptions{}) 134 if err != nil { 135 t.Fatal(err) 136 } 137 defer func() { 138 err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), mutatingCfg.GetName(), metav1.DeleteOptions{}) 139 if err != nil { 140 t.Fatal(err) 141 } 142 }() 143 144 // wait until new webhook is called the first time 145 if err := wait.PollImmediate(time.Millisecond*5, wait.ForeverTestTimeout, func() (bool, error) { 146 _, err = client.CoreV1().Pods("default").Patch(context.TODO(), loadBalanceMarkerFixture.Name, types.JSONPatchType, []byte("[]"), metav1.PatchOptions{}) 147 select { 148 case <-upCh: 149 return true, nil 150 default: 151 t.Logf("Waiting for webhook to become effective, getting marker object: %v", err) 152 return false, nil 153 } 154 }); err != nil { 155 t.Fatal(err) 156 } 157 158 pod := func() *corev1.Pod { 159 return &corev1.Pod{ 160 ObjectMeta: metav1.ObjectMeta{ 161 Namespace: ns, 162 GenerateName: "loadbalance-", 163 }, 164 Spec: corev1.PodSpec{ 165 Containers: []corev1.Container{{ 166 Name: "fake-name", 167 Image: "fakeimage", 168 }}, 169 }, 170 } 171 } 172 173 // Submit 10 parallel requests 174 wg := &sync.WaitGroup{} 175 for i := 0; i < 10; i++ { 176 wg.Add(1) 177 go func() { 178 defer wg.Done() 179 _, err := client.CoreV1().Pods(ns).Create(context.TODO(), pod(), metav1.CreateOptions{}) 180 if err != nil { 181 t.Error(err) 182 } 183 }() 184 } 185 wg.Wait() 186 187 if actual := atomic.LoadInt64(&trackingListener.connections); actual < 10 { 188 t.Errorf("expected at least 10 connections, got %d", actual) 189 } 190 trackingListener.Reset() 191 192 // Submit 10 more parallel requests 193 wg = &sync.WaitGroup{} 194 for i := 0; i < 10; i++ { 195 wg.Add(1) 196 go func() { 197 defer wg.Done() 198 _, err := client.CoreV1().Pods(ns).Create(context.TODO(), pod(), metav1.CreateOptions{}) 199 if err != nil { 200 t.Error(err) 201 } 202 }() 203 } 204 wg.Wait() 205 206 if actual := atomic.LoadInt64(&trackingListener.connections); actual > 0 { 207 t.Errorf("expected no additional connections (reusing kept-alive connections), got %d", actual) 208 } 209 } 210 211 type connectionRecorder struct { 212 mu sync.Mutex 213 upCh chan struct{} 214 upOnce sync.Once 215 } 216 217 // Reset zeros out all counts and returns a channel that is closed when the first admission of the 218 // marker object is received. 219 func (i *connectionRecorder) Reset() chan struct{} { 220 i.mu.Lock() 221 defer i.mu.Unlock() 222 i.upCh = make(chan struct{}) 223 i.upOnce = sync.Once{} 224 return i.upCh 225 } 226 227 func (i *connectionRecorder) MarkerReceived() { 228 i.mu.Lock() 229 defer i.mu.Unlock() 230 i.upOnce.Do(func() { 231 close(i.upCh) 232 }) 233 } 234 235 func newLoadBalanceWebhookHandler(recorder *connectionRecorder) http.Handler { 236 allow := func(w http.ResponseWriter) { 237 w.Header().Set("Content-Type", "application/json") 238 json.NewEncoder(w).Encode(&v1beta1.AdmissionReview{ 239 Response: &v1beta1.AdmissionResponse{ 240 Allowed: true, 241 }, 242 }) 243 } 244 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 245 fmt.Println(r.Proto) 246 defer r.Body.Close() 247 data, err := io.ReadAll(r.Body) 248 if err != nil { 249 http.Error(w, err.Error(), 400) 250 } 251 review := v1beta1.AdmissionReview{} 252 if err := json.Unmarshal(data, &review); err != nil { 253 http.Error(w, err.Error(), 400) 254 } 255 if review.Request.UserInfo.Username != testLoadBalanceClientUsername { 256 // skip requests not originating from this integration test's client 257 allow(w) 258 return 259 } 260 261 if len(review.Request.Object.Raw) == 0 { 262 http.Error(w, err.Error(), 400) 263 } 264 pod := &corev1.Pod{} 265 if err := json.Unmarshal(review.Request.Object.Raw, pod); err != nil { 266 http.Error(w, err.Error(), 400) 267 } 268 269 // When resetting between tests, a marker object is patched until this webhook 270 // observes it, at which point it is considered ready. 271 if pod.Namespace == loadBalanceMarkerFixture.Namespace && pod.Name == loadBalanceMarkerFixture.Name { 272 recorder.MarkerReceived() 273 allow(w) 274 return 275 } 276 277 // simulate a loaded backend 278 time.Sleep(2 * time.Second) 279 allow(w) 280 }) 281 } 282 283 var loadBalanceMarkerFixture = &corev1.Pod{ 284 ObjectMeta: metav1.ObjectMeta{ 285 Namespace: "default", 286 Name: "marker", 287 }, 288 Spec: corev1.PodSpec{ 289 Containers: []corev1.Container{{ 290 Name: "fake-name", 291 Image: "fakeimage", 292 }}, 293 }, 294 } 295 296 type connectionTrackingListener struct { 297 connections int64 298 delegate net.Listener 299 } 300 301 func (c *connectionTrackingListener) Reset() { 302 atomic.StoreInt64(&c.connections, 0) 303 } 304 305 func (c *connectionTrackingListener) Accept() (net.Conn, error) { 306 conn, err := c.delegate.Accept() 307 if err == nil { 308 atomic.AddInt64(&c.connections, 1) 309 } 310 return conn, err 311 } 312 func (c *connectionTrackingListener) Close() error { 313 return c.delegate.Close() 314 } 315 func (c *connectionTrackingListener) Addr() net.Addr { 316 return c.delegate.Addr() 317 }