k8s.io/kubernetes@v1.29.3/test/integration/client/metrics/metrics_test.go (about) 1 /* 2 Copyright 2023 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package metrics 18 19 import ( 20 "context" 21 "fmt" 22 "strconv" 23 "strings" 24 "testing" 25 26 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 27 apierrors "k8s.io/apimachinery/pkg/api/errors" 28 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 "k8s.io/apiserver/pkg/util/feature" 30 featuregatetesting "k8s.io/component-base/featuregate/testing" 31 "k8s.io/component-base/metrics/legacyregistry" 32 apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" 33 aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" 34 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 35 "k8s.io/kubernetes/test/integration/framework" 36 37 // the metrics are loaded on cmd/kube-apiserver/apiserver.go 38 // so we need to load them here to be available for the test 39 _ "k8s.io/component-base/metrics/prometheus/restclient" 40 ) 41 42 // IMPORTANT: metrics are stored globally so all the test must run serially 43 // and reset the metrics. 44 45 // regression test for https://issues.k8s.io/117258 46 func TestAPIServerTransportMetrics(t *testing.T) { 47 defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)() 48 defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)() 49 50 // reset default registry metrics 51 legacyregistry.Reset() 52 53 result := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins", "ServiceAccount"}, framework.SharedEtcd()) 54 defer result.TearDownFn() 55 56 client := clientset.NewForConfigOrDie(result.ClientConfig) 57 58 // IMPORTANT: reflect the current values if the test changes 59 // client_test.go:1407: metric rest_client_transport_cache_entries 3 60 // client_test.go:1407: metric rest_client_transport_create_calls_total{result="hit"} 61 61 // client_test.go:1407: metric rest_client_transport_create_calls_total{result="miss"} 3 62 hits1, misses1, entries1 := checkTransportMetrics(t, client) 63 // hit ratio at startup depends on multiple factors 64 if (hits1*100)/(hits1+misses1) < 90 { 65 t.Fatalf("transport cache hit ratio %d lower than 90 percent", (hits1*100)/(hits1+misses1)) 66 } 67 68 aggregatorClient := aggregatorclient.NewForConfigOrDie(result.ClientConfig) 69 aggregatedAPI := &apiregistrationv1.APIService{ 70 ObjectMeta: metav1.ObjectMeta{Name: "v1alpha1.wardle.example.com"}, 71 Spec: apiregistrationv1.APIServiceSpec{ 72 Service: &apiregistrationv1.ServiceReference{ 73 Namespace: "kube-wardle", 74 Name: "api", 75 }, 76 Group: "wardle.example.com", 77 Version: "v1alpha1", 78 GroupPriorityMinimum: 200, 79 VersionPriority: 200, 80 }, 81 } 82 _, err := aggregatorClient.ApiregistrationV1().APIServices().Create(context.Background(), aggregatedAPI, metav1.CreateOptions{}) 83 if err != nil { 84 t.Fatal(err) 85 } 86 87 requests := 30 88 errors := 0 89 for i := 0; i < requests; i++ { 90 apiService, err := aggregatorClient.ApiregistrationV1().APIServices().Get(context.Background(), "v1alpha1.wardle.example.com", metav1.GetOptions{}) 91 if err != nil { 92 t.Fatal(err) 93 } 94 // mutate the object 95 apiService.Labels = map[string]string{"key": fmt.Sprintf("val%d", i)} 96 _, err = aggregatorClient.ApiregistrationV1().APIServices().Update(context.Background(), apiService, metav1.UpdateOptions{}) 97 if err != nil && !apierrors.IsConflict(err) { 98 t.Logf("unexpected error: %v", err) 99 errors++ 100 } 101 } 102 103 if (errors*100)/requests > 20 { 104 t.Fatalf("high number of errors during the test %d out of %d", errors, requests) 105 } 106 107 // IMPORTANT: reflect the current values if the test changes 108 // client_test.go:1407: metric rest_client_transport_cache_entries 4 109 // client_test.go:1407: metric rest_client_transport_create_calls_total{result="hit"} 120 110 // client_test.go:1407: metric rest_client_transport_create_calls_total{result="miss"} 4 111 hits2, misses2, entries2 := checkTransportMetrics(t, client) 112 if entries2-entries1 > 10 { 113 t.Fatalf("possible transport leak, number of new cache entries increased by %d", entries2-entries1) 114 } 115 116 // hit ratio after startup should grow since no new transports are expected 117 if (hits2*100)/(hits2+misses2) < 95 { 118 t.Fatalf("transport cache hit ratio %d lower than 95 percent", (hits2*100)/(hits2+misses2)) 119 } 120 } 121 122 func checkTransportMetrics(t *testing.T, client *clientset.Clientset) (hits int, misses int, entries int) { 123 t.Helper() 124 body, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.Background()) 125 if err != nil { 126 t.Fatal(err) 127 } 128 129 // TODO: this can be much better if there is some library that parse prometheus metrics 130 // the existing one in "k8s.io/component-base/metrics/testutil" uses the global variable 131 // but we want to parse the ones returned by the endpoint to be sure the metrics are 132 // exposed correctly 133 for _, line := range strings.Split(string(body), "\n") { 134 if !strings.HasPrefix(line, "rest_client_transport") { 135 continue 136 } 137 if strings.Contains(line, "uncacheable") { 138 t.Fatalf("detected transport that is not cacheable, please check https://issues.k8s.io/112017") 139 } 140 141 output := strings.Split(line, " ") 142 if len(output) != 2 { 143 t.Fatalf("expected metrics to be in the format name value, got %v", output) 144 } 145 name := output[0] 146 value, err := strconv.Atoi(output[1]) 147 if err != nil { 148 t.Fatalf("metric value can not be converted to integer %v", err) 149 } 150 switch name { 151 case "rest_client_transport_cache_entries": 152 entries = value 153 case `rest_client_transport_create_calls_total{result="hit"}`: 154 hits = value 155 case `rest_client_transport_create_calls_total{result="miss"}`: 156 misses = value 157 } 158 t.Logf("metric %s", line) 159 } 160 161 if misses != entries || misses == 0 { 162 t.Errorf("expected as many entries %d in the cache as misses, got %d", entries, misses) 163 } 164 165 if hits < misses { 166 t.Errorf("expected more hits %d in the cache than misses %d", hits, misses) 167 } 168 return 169 }