k8s.io/kubernetes@v1.29.3/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package monitoring 18 19 import ( 20 "context" 21 "time" 22 23 gcm "google.golang.org/api/monitoring/v3" 24 v1 "k8s.io/api/core/v1" 25 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 "k8s.io/apimachinery/pkg/labels" 27 "k8s.io/apimachinery/pkg/runtime/schema" 28 "k8s.io/apimachinery/pkg/selection" 29 "k8s.io/client-go/discovery" 30 cacheddiscovery "k8s.io/client-go/discovery/cached/memory" 31 clientset "k8s.io/client-go/kubernetes" 32 "k8s.io/client-go/restmapper" 33 "k8s.io/kubernetes/test/e2e/feature" 34 "k8s.io/kubernetes/test/e2e/framework" 35 e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" 36 instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common" 37 customclient "k8s.io/metrics/pkg/client/custom_metrics" 38 externalclient "k8s.io/metrics/pkg/client/external_metrics" 39 admissionapi "k8s.io/pod-security-admission/api" 40 41 "github.com/onsi/ginkgo/v2" 42 "golang.org/x/oauth2/google" 43 "google.golang.org/api/option" 44 ) 45 46 const ( 47 stackdriverExporterPod1 = "stackdriver-exporter-1" 48 stackdriverExporterPod2 = "stackdriver-exporter-2" 49 stackdriverExporterLabel = "stackdriver-exporter" 50 ) 51 52 var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() { 53 ginkgo.BeforeEach(func() { 54 e2eskipper.SkipUnlessProviderIs("gce", "gke") 55 }) 56 57 f := framework.NewDefaultFramework("stackdriver-monitoring") 58 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged 59 60 f.It("should run Custom Metrics - Stackdriver Adapter for old resource model", feature.StackdriverCustomMetrics, func(ctx context.Context) { 61 kubeClient := f.ClientSet 62 config, err := framework.LoadConfig() 63 if err != nil { 64 framework.Failf("Failed to load config: %s", err) 65 } 66 discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config) 67 cachedDiscoClient := cacheddiscovery.NewMemCacheClient(discoveryClient) 68 restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoClient) 69 restMapper.Reset() 70 apiVersionsGetter := customclient.NewAvailableAPIsGetter(discoveryClient) 71 customMetricsClient := customclient.NewForConfig(config, restMapper, apiVersionsGetter) 72 testCustomMetrics(ctx, f, kubeClient, customMetricsClient, discoveryClient, AdapterForOldResourceModel) 73 }) 74 75 f.It("should run Custom Metrics - Stackdriver Adapter for new resource model", feature.StackdriverCustomMetrics, func(ctx context.Context) { 76 kubeClient := f.ClientSet 77 config, err := framework.LoadConfig() 78 if err != nil { 79 framework.Failf("Failed to load config: %s", err) 80 } 81 discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config) 82 cachedDiscoClient := cacheddiscovery.NewMemCacheClient(discoveryClient) 83 restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoClient) 84 restMapper.Reset() 85 apiVersionsGetter := customclient.NewAvailableAPIsGetter(discoveryClient) 86 customMetricsClient := customclient.NewForConfig(config, restMapper, apiVersionsGetter) 87 testCustomMetrics(ctx, f, kubeClient, customMetricsClient, discoveryClient, AdapterForNewResourceModel) 88 }) 89 90 f.It("should run Custom Metrics - Stackdriver Adapter for external metrics", feature.StackdriverExternalMetrics, func(ctx context.Context) { 91 kubeClient := f.ClientSet 92 config, err := framework.LoadConfig() 93 if err != nil { 94 framework.Failf("Failed to load config: %s", err) 95 } 96 externalMetricsClient := externalclient.NewForConfigOrDie(config) 97 testExternalMetrics(ctx, f, kubeClient, externalMetricsClient) 98 }) 99 }) 100 101 func testCustomMetrics(ctx context.Context, f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient, adapterDeployment string) { 102 projectID := framework.TestContext.CloudConfig.ProjectID 103 104 client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) 105 framework.ExpectNoError(err) 106 107 gcmService, err := gcm.NewService(ctx, option.WithHTTPClient(client)) 108 if err != nil { 109 framework.Failf("Failed to create gcm service, %v", err) 110 } 111 112 // Set up a cluster: create a custom metric and set up k8s-sd adapter 113 err = CreateDescriptors(gcmService, projectID) 114 if err != nil { 115 framework.Failf("Failed to create metric descriptor: %s", err) 116 } 117 ginkgo.DeferCleanup(CleanupDescriptors, gcmService, projectID) 118 119 err = CreateAdapter(adapterDeployment) 120 if err != nil { 121 framework.Failf("Failed to set up: %s", err) 122 } 123 ginkgo.DeferCleanup(CleanupAdapter, adapterDeployment) 124 125 _, err = kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, HPAPermissions, metav1.CreateOptions{}) 126 if err != nil { 127 framework.Failf("Failed to create ClusterRoleBindings: %v", err) 128 } 129 ginkgo.DeferCleanup(kubeClient.RbacV1().ClusterRoleBindings().Delete, HPAPermissions.Name, metav1.DeleteOptions{}) 130 131 // Run application that exports the metric 132 _, err = createSDExporterPods(ctx, f, kubeClient) 133 if err != nil { 134 framework.Failf("Failed to create stackdriver-exporter pod: %s", err) 135 } 136 ginkgo.DeferCleanup(cleanupSDExporterPod, f, kubeClient) 137 138 // Wait a short amount of time to create a pod and export some metrics 139 // TODO: add some events to wait for instead of fixed amount of time 140 // i.e. pod creation, first time series exported 141 time.Sleep(60 * time.Second) 142 143 verifyResponsesFromCustomMetricsAPI(f, customMetricsClient, discoveryClient) 144 } 145 146 // TODO(kawych): migrate this test to new resource model 147 func testExternalMetrics(ctx context.Context, f *framework.Framework, kubeClient clientset.Interface, externalMetricsClient externalclient.ExternalMetricsClient) { 148 projectID := framework.TestContext.CloudConfig.ProjectID 149 150 client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) 151 framework.ExpectNoError(err) 152 153 gcmService, err := gcm.NewService(ctx, option.WithHTTPClient(client)) 154 if err != nil { 155 framework.Failf("Failed to create gcm service, %v", err) 156 } 157 158 // Set up a cluster: create a custom metric and set up k8s-sd adapter 159 err = CreateDescriptors(gcmService, projectID) 160 if err != nil { 161 framework.Failf("Failed to create metric descriptor: %s", err) 162 } 163 ginkgo.DeferCleanup(CleanupDescriptors, gcmService, projectID) 164 165 // Both deployments - for old and new resource model - expose External Metrics API. 166 err = CreateAdapter(AdapterForOldResourceModel) 167 if err != nil { 168 framework.Failf("Failed to set up: %s", err) 169 } 170 ginkgo.DeferCleanup(CleanupAdapter, AdapterForOldResourceModel) 171 172 _, err = kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, HPAPermissions, metav1.CreateOptions{}) 173 if err != nil { 174 framework.Failf("Failed to create ClusterRoleBindings: %v", err) 175 } 176 ginkgo.DeferCleanup(kubeClient.RbacV1().ClusterRoleBindings().Delete, HPAPermissions.Name, metav1.DeleteOptions{}) 177 178 // Run application that exports the metric 179 pod, err := createSDExporterPods(ctx, f, kubeClient) 180 if err != nil { 181 framework.Failf("Failed to create stackdriver-exporter pod: %s", err) 182 } 183 ginkgo.DeferCleanup(cleanupSDExporterPod, f, kubeClient) 184 185 // Wait a short amount of time to create a pod and export some metrics 186 // TODO: add some events to wait for instead of fixed amount of time 187 // i.e. pod creation, first time series exported 188 time.Sleep(60 * time.Second) 189 190 verifyResponseFromExternalMetricsAPI(f, externalMetricsClient, pod) 191 } 192 193 func verifyResponsesFromCustomMetricsAPI(f *framework.Framework, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) { 194 resources, err := discoveryClient.ServerResourcesForGroupVersion("custom.metrics.k8s.io/v1beta1") 195 if err != nil { 196 framework.Failf("Failed to retrieve a list of supported metrics: %s", err) 197 } 198 if !containsResource(resources.APIResources, "*/custom.googleapis.com|"+CustomMetricName) { 199 framework.Failf("Metric '%s' expected but not received", CustomMetricName) 200 } 201 if !containsResource(resources.APIResources, "*/custom.googleapis.com|"+UnusedMetricName) { 202 framework.Failf("Metric '%s' expected but not received", UnusedMetricName) 203 } 204 value, err := customMetricsClient.NamespacedMetrics(f.Namespace.Name).GetForObject(schema.GroupKind{Group: "", Kind: "Pod"}, stackdriverExporterPod1, CustomMetricName, labels.NewSelector()) 205 if err != nil { 206 framework.Failf("Failed query: %s", err) 207 } 208 if value.Value.Value() != CustomMetricValue { 209 framework.Failf("Unexpected metric value for metric %s: expected %v but received %v", CustomMetricName, CustomMetricValue, value.Value) 210 } 211 filter, err := labels.NewRequirement("name", selection.Equals, []string{stackdriverExporterLabel}) 212 if err != nil { 213 framework.Failf("Couldn't create a label filter") 214 } 215 values, err := customMetricsClient.NamespacedMetrics(f.Namespace.Name).GetForObjects(schema.GroupKind{Group: "", Kind: "Pod"}, labels.NewSelector().Add(*filter), CustomMetricName, labels.NewSelector()) 216 if err != nil { 217 framework.Failf("Failed query: %s", err) 218 } 219 if len(values.Items) != 1 { 220 framework.Failf("Expected results for exactly 1 pod, but %v results received", len(values.Items)) 221 } 222 if values.Items[0].DescribedObject.Name != stackdriverExporterPod1 || values.Items[0].Value.Value() != CustomMetricValue { 223 framework.Failf("Unexpected metric value for metric %s and pod %s: %v", CustomMetricName, values.Items[0].DescribedObject.Name, values.Items[0].Value.Value()) 224 } 225 } 226 227 func containsResource(resourcesList []metav1.APIResource, resourceName string) bool { 228 for _, resource := range resourcesList { 229 if resource.Name == resourceName { 230 return true 231 } 232 } 233 return false 234 } 235 236 func verifyResponseFromExternalMetricsAPI(f *framework.Framework, externalMetricsClient externalclient.ExternalMetricsClient, pod *v1.Pod) { 237 req1, _ := labels.NewRequirement("resource.type", selection.Equals, []string{"gke_container"}) 238 // It's important to filter out only metrics from the right namespace, since multiple e2e tests 239 // may run in the same project concurrently. "dummy" is added to test 240 req2, _ := labels.NewRequirement("resource.labels.pod_id", selection.In, []string{string(pod.UID), "dummy"}) 241 req3, _ := labels.NewRequirement("resource.labels.namespace_id", selection.Exists, []string{}) 242 req4, _ := labels.NewRequirement("resource.labels.zone", selection.NotEquals, []string{"dummy"}) 243 req5, _ := labels.NewRequirement("resource.labels.cluster_name", selection.NotIn, []string{"foo", "bar"}) 244 values, err := externalMetricsClient. 245 NamespacedMetrics("dummy"). 246 List("custom.googleapis.com|"+CustomMetricName, labels.NewSelector().Add(*req1, *req2, *req3, *req4, *req5)) 247 if err != nil { 248 framework.Failf("Failed query: %s", err) 249 } 250 if len(values.Items) != 1 { 251 framework.Failf("Expected exactly one external metric value, but % values received", len(values.Items)) 252 } 253 if values.Items[0].MetricName != "custom.googleapis.com|"+CustomMetricName || 254 values.Items[0].Value.Value() != CustomMetricValue || 255 // Check one label just to make sure labels are included 256 values.Items[0].MetricLabels["resource.labels.pod_id"] != string(pod.UID) { 257 framework.Failf("Unexpected result for metric %s: %v", CustomMetricName, values.Items[0]) 258 } 259 } 260 261 func cleanupSDExporterPod(ctx context.Context, f *framework.Framework, cs clientset.Interface) { 262 err := cs.CoreV1().Pods(f.Namespace.Name).Delete(ctx, stackdriverExporterPod1, metav1.DeleteOptions{}) 263 if err != nil { 264 framework.Logf("Failed to delete %s pod: %v", stackdriverExporterPod1, err) 265 } 266 err = cs.CoreV1().Pods(f.Namespace.Name).Delete(ctx, stackdriverExporterPod2, metav1.DeleteOptions{}) 267 if err != nil { 268 framework.Logf("Failed to delete %s pod: %v", stackdriverExporterPod2, err) 269 } 270 } 271 272 func createSDExporterPods(ctx context.Context, f *framework.Framework, cs clientset.Interface) (*v1.Pod, error) { 273 pod, err := cs.CoreV1().Pods(f.Namespace.Name).Create(ctx, StackdriverExporterPod(stackdriverExporterPod1, f.Namespace.Name, stackdriverExporterLabel, CustomMetricName, CustomMetricValue), metav1.CreateOptions{}) 274 if err != nil { 275 return nil, err 276 } 277 _, err = cs.CoreV1().Pods(f.Namespace.Name).Create(ctx, StackdriverExporterPod(stackdriverExporterPod2, f.Namespace.Name, stackdriverExporterLabel, UnusedMetricName, UnusedMetricValue), metav1.CreateOptions{}) 278 return pod, err 279 }