k8s.io/kubernetes@v1.29.3/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package autoscaling 18 19 import ( 20 "context" 21 "fmt" 22 "math" 23 "time" 24 25 gcm "google.golang.org/api/monitoring/v3" 26 "google.golang.org/api/option" 27 appsv1 "k8s.io/api/apps/v1" 28 autoscalingv2 "k8s.io/api/autoscaling/v2" 29 v1 "k8s.io/api/core/v1" 30 "k8s.io/apimachinery/pkg/api/resource" 31 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 32 "k8s.io/apimachinery/pkg/util/wait" 33 clientset "k8s.io/client-go/kubernetes" 34 "k8s.io/kubernetes/test/e2e/feature" 35 "k8s.io/kubernetes/test/e2e/framework" 36 e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" 37 e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" 38 "k8s.io/kubernetes/test/e2e/instrumentation/monitoring" 39 admissionapi "k8s.io/pod-security-admission/api" 40 41 "github.com/onsi/ginkgo/v2" 42 "golang.org/x/oauth2/google" 43 ) 44 45 const ( 46 stackdriverExporterDeployment = "stackdriver-exporter-deployment" 47 dummyDeploymentName = "dummy-deployment" 48 stackdriverExporterPod = "stackdriver-exporter-pod" 49 externalMetricValue = int64(85) 50 ) 51 52 type externalMetricTarget struct { 53 value int64 54 isAverage bool 55 } 56 57 var _ = SIGDescribe("[HPA]", feature.CustomMetricsAutoscaling, "Horizontal pod autoscaling (scale resource: Custom Metrics from Stackdriver)", func() { 58 ginkgo.BeforeEach(func() { 59 e2eskipper.SkipUnlessProviderIs("gce", "gke") 60 }) 61 62 f := framework.NewDefaultFramework("horizontal-pod-autoscaling") 63 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged 64 65 ginkgo.Describe("with Custom Metric of type Pod from Stackdriver", func() { 66 ginkgo.It("should scale down", func(ctx context.Context) { 67 initialReplicas := 2 68 // metric should cause scale down 69 metricValue := int64(100) 70 metricTarget := 2 * metricValue 71 metricSpecs := []autoscalingv2.MetricSpec{ 72 podMetricSpecWithAverageValueTarget(monitoring.CustomMetricName, metricTarget), 73 } 74 tc := CustomMetricTestCase{ 75 framework: f, 76 kubeClient: f.ClientSet, 77 initialReplicas: initialReplicas, 78 scaledReplicas: 1, 79 deployment: monitoring.SimpleStackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue), 80 hpa: hpa("custom-metrics-pods-hpa", f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, 1, 3, metricSpecs), 81 } 82 tc.Run(ctx) 83 }) 84 85 ginkgo.It("should scale up with two metrics", func(ctx context.Context) { 86 initialReplicas := 1 87 // metric 1 would cause a scale down, if not for metric 2 88 metric1Value := int64(100) 89 metric1Target := 2 * metric1Value 90 // metric2 should cause a scale up 91 metric2Value := int64(200) 92 metric2Target := int64(0.5 * float64(metric2Value)) 93 metricSpecs := []autoscalingv2.MetricSpec{ 94 podMetricSpecWithAverageValueTarget("metric1", metric1Target), 95 podMetricSpecWithAverageValueTarget("metric2", metric2Target), 96 } 97 containers := []monitoring.CustomMetricContainerSpec{ 98 { 99 Name: "stackdriver-exporter-metric1", 100 MetricName: "metric1", 101 MetricValue: metric1Value, 102 }, 103 { 104 Name: "stackdriver-exporter-metric2", 105 MetricName: "metric2", 106 MetricValue: metric2Value, 107 }, 108 } 109 tc := CustomMetricTestCase{ 110 framework: f, 111 kubeClient: f.ClientSet, 112 initialReplicas: initialReplicas, 113 scaledReplicas: 3, 114 deployment: monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers), 115 hpa: hpa("custom-metrics-pods-hpa", f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, 1, 3, metricSpecs), 116 } 117 tc.Run(ctx) 118 }) 119 120 ginkgo.It("should scale down with Prometheus", func(ctx context.Context) { 121 initialReplicas := 2 122 // metric should cause scale down 123 metricValue := int64(100) 124 metricTarget := 2 * metricValue 125 metricSpecs := []autoscalingv2.MetricSpec{ 126 podMetricSpecWithAverageValueTarget(monitoring.CustomMetricName, metricTarget), 127 } 128 tc := CustomMetricTestCase{ 129 framework: f, 130 kubeClient: f.ClientSet, 131 initialReplicas: initialReplicas, 132 scaledReplicas: 1, 133 deployment: monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue), 134 hpa: hpa("custom-metrics-pods-hpa", f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, 1, 3, metricSpecs), 135 } 136 tc.Run(ctx) 137 }) 138 }) 139 140 ginkgo.Describe("with Custom Metric of type Object from Stackdriver", func() { 141 ginkgo.It("should scale down", func(ctx context.Context) { 142 initialReplicas := 2 143 // metric should cause scale down 144 metricValue := int64(100) 145 metricTarget := 2 * metricValue 146 metricSpecs := []autoscalingv2.MetricSpec{ 147 objectMetricSpecWithValueTarget(metricTarget), 148 } 149 tc := CustomMetricTestCase{ 150 framework: f, 151 kubeClient: f.ClientSet, 152 initialReplicas: initialReplicas, 153 scaledReplicas: 1, 154 deployment: noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)), 155 pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue), 156 hpa: hpa("custom-metrics-objects-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs), 157 } 158 tc.Run(ctx) 159 }) 160 161 ginkgo.It("should scale down to 0", func(ctx context.Context) { 162 initialReplicas := 2 163 // metric should cause scale down 164 metricValue := int64(0) 165 metricTarget := int64(200) 166 metricSpecs := []autoscalingv2.MetricSpec{ 167 objectMetricSpecWithValueTarget(metricTarget), 168 } 169 tc := CustomMetricTestCase{ 170 framework: f, 171 kubeClient: f.ClientSet, 172 initialReplicas: initialReplicas, 173 scaledReplicas: 0, 174 deployment: noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)), 175 pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue), 176 hpa: hpa("custom-metrics-objects-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 0, 3, metricSpecs), 177 } 178 tc.Run(ctx) 179 }) 180 }) 181 182 ginkgo.Describe("with External Metric from Stackdriver", func() { 183 ginkgo.It("should scale down with target value", func(ctx context.Context) { 184 initialReplicas := 2 185 // metric should cause scale down 186 metricValue := externalMetricValue 187 metricTarget := 3 * metricValue 188 metricSpecs := []autoscalingv2.MetricSpec{ 189 externalMetricSpecWithTarget("target", f.Namespace.ObjectMeta.Name, externalMetricTarget{ 190 value: metricTarget, 191 isAverage: false, 192 }), 193 } 194 tc := CustomMetricTestCase{ 195 framework: f, 196 kubeClient: f.ClientSet, 197 initialReplicas: initialReplicas, 198 scaledReplicas: 1, 199 deployment: noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)), 200 pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, "target", metricValue), 201 hpa: hpa("custom-metrics-external-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs), 202 } 203 tc.Run(ctx) 204 }) 205 206 ginkgo.It("should scale down with target average value", func(ctx context.Context) { 207 initialReplicas := 2 208 // metric should cause scale down 209 metricValue := externalMetricValue 210 metricAverageTarget := 3 * metricValue 211 metricSpecs := []autoscalingv2.MetricSpec{ 212 externalMetricSpecWithTarget("target_average", f.Namespace.ObjectMeta.Name, externalMetricTarget{ 213 value: metricAverageTarget, 214 isAverage: true, 215 }), 216 } 217 tc := CustomMetricTestCase{ 218 framework: f, 219 kubeClient: f.ClientSet, 220 initialReplicas: initialReplicas, 221 scaledReplicas: 1, 222 deployment: noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)), 223 pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, "target_average", externalMetricValue), 224 hpa: hpa("custom-metrics-external-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs), 225 } 226 tc.Run(ctx) 227 }) 228 229 ginkgo.It("should scale up with two metrics", func(ctx context.Context) { 230 initialReplicas := 1 231 // metric 1 would cause a scale down, if not for metric 2 232 metric1Value := externalMetricValue 233 metric1Target := 2 * metric1Value 234 // metric2 should cause a scale up 235 metric2Value := externalMetricValue 236 metric2Target := int64(math.Ceil(0.5 * float64(metric2Value))) 237 metricSpecs := []autoscalingv2.MetricSpec{ 238 externalMetricSpecWithTarget("external_metric_1", f.Namespace.ObjectMeta.Name, externalMetricTarget{ 239 value: metric1Target, 240 isAverage: true, 241 }), 242 externalMetricSpecWithTarget("external_metric_2", f.Namespace.ObjectMeta.Name, externalMetricTarget{ 243 value: metric2Target, 244 isAverage: true, 245 }), 246 } 247 containers := []monitoring.CustomMetricContainerSpec{ 248 { 249 Name: "stackdriver-exporter-metric1", 250 MetricName: "external_metric_1", 251 MetricValue: metric1Value, 252 }, 253 { 254 Name: "stackdriver-exporter-metric2", 255 MetricName: "external_metric_2", 256 MetricValue: metric2Value, 257 }, 258 } 259 tc := CustomMetricTestCase{ 260 framework: f, 261 kubeClient: f.ClientSet, 262 initialReplicas: initialReplicas, 263 scaledReplicas: 3, 264 deployment: monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers), 265 hpa: hpa("custom-metrics-external-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs), 266 } 267 tc.Run(ctx) 268 }) 269 }) 270 271 ginkgo.Describe("with multiple metrics of different types", func() { 272 ginkgo.It("should scale up when one metric is missing (Pod and External metrics)", func(ctx context.Context) { 273 initialReplicas := 1 274 // First metric a pod metric which is missing. 275 // Second metric is external metric which is present, it should cause scale up. 276 metricSpecs := []autoscalingv2.MetricSpec{ 277 podMetricSpecWithAverageValueTarget(monitoring.CustomMetricName, 2*externalMetricValue), 278 externalMetricSpecWithTarget("external_metric", f.Namespace.ObjectMeta.Name, externalMetricTarget{ 279 value: int64(math.Ceil(0.5 * float64(externalMetricValue))), 280 isAverage: true, 281 }), 282 } 283 containers := []monitoring.CustomMetricContainerSpec{ 284 { 285 Name: "stackdriver-exporter-metric", 286 MetricName: "external_metric", 287 MetricValue: externalMetricValue, 288 }, 289 // Pod Resource metric is missing from here. 290 } 291 tc := CustomMetricTestCase{ 292 framework: f, 293 kubeClient: f.ClientSet, 294 initialReplicas: initialReplicas, 295 scaledReplicas: 3, 296 deployment: monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers), 297 hpa: hpa("multiple-metrics", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs)} 298 tc.Run(ctx) 299 }) 300 301 ginkgo.It("should scale up when one metric is missing (Resource and Object metrics)", func(ctx context.Context) { 302 initialReplicas := 1 303 metricValue := int64(100) 304 // First metric a resource metric which is missing (no consumption). 305 // Second metric is object metric which is present, it should cause scale up. 306 metricSpecs := []autoscalingv2.MetricSpec{ 307 resourceMetricSpecWithAverageUtilizationTarget(50), 308 objectMetricSpecWithValueTarget(int64(math.Ceil(0.5 * float64(metricValue)))), 309 } 310 tc := CustomMetricTestCase{ 311 framework: f, 312 kubeClient: f.ClientSet, 313 initialReplicas: initialReplicas, 314 scaledReplicas: 3, 315 deployment: monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 0), 316 pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue), 317 hpa: hpa("multiple-metrics", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs)} 318 tc.Run(ctx) 319 }) 320 321 ginkgo.It("should not scale down when one metric is missing (Container Resource and External Metrics)", func(ctx context.Context) { 322 initialReplicas := 2 323 // First metric a container resource metric which is missing. 324 // Second metric is external metric which is present, it should cause scale down if the first metric wasn't missing. 325 metricSpecs := []autoscalingv2.MetricSpec{ 326 containerResourceMetricSpecWithAverageUtilizationTarget("container-resource-metric", 50), 327 externalMetricSpecWithTarget("external_metric", f.Namespace.ObjectMeta.Name, externalMetricTarget{ 328 value: 2 * externalMetricValue, 329 isAverage: true, 330 }), 331 } 332 containers := []monitoring.CustomMetricContainerSpec{ 333 { 334 Name: "stackdriver-exporter-metric", 335 MetricName: "external_metric", 336 MetricValue: externalMetricValue, 337 }, 338 // Container Resource metric is missing from here. 339 } 340 tc := CustomMetricTestCase{ 341 framework: f, 342 kubeClient: f.ClientSet, 343 initialReplicas: initialReplicas, 344 scaledReplicas: initialReplicas, 345 verifyStability: true, 346 deployment: monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers), 347 hpa: hpa("multiple-metrics", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs)} 348 tc.Run(ctx) 349 }) 350 351 ginkgo.It("should not scale down when one metric is missing (Pod and Object Metrics)", func(ctx context.Context) { 352 initialReplicas := 2 353 metricValue := int64(100) 354 // First metric an object metric which is missing. 355 // Second metric is pod metric which is present, it should cause scale down if the first metric wasn't missing. 356 metricSpecs := []autoscalingv2.MetricSpec{ 357 objectMetricSpecWithValueTarget(int64(math.Ceil(0.5 * float64(metricValue)))), 358 podMetricSpecWithAverageValueTarget("pod_metric", 2*metricValue), 359 } 360 containers := []monitoring.CustomMetricContainerSpec{ 361 { 362 Name: "stackdriver-exporter-metric", 363 MetricName: "pod_metric", 364 MetricValue: metricValue, 365 }, 366 } 367 tc := CustomMetricTestCase{ 368 framework: f, 369 kubeClient: f.ClientSet, 370 initialReplicas: initialReplicas, 371 scaledReplicas: initialReplicas, 372 verifyStability: true, 373 deployment: monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers), 374 hpa: hpa("multiple-metrics", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs)} 375 tc.Run(ctx) 376 }) 377 }) 378 379 }) 380 381 // CustomMetricTestCase is a struct for test cases. 382 type CustomMetricTestCase struct { 383 framework *framework.Framework 384 hpa *autoscalingv2.HorizontalPodAutoscaler 385 kubeClient clientset.Interface 386 deployment *appsv1.Deployment 387 pod *v1.Pod 388 initialReplicas int 389 scaledReplicas int 390 verifyStability bool 391 } 392 393 // Run starts test case. 394 func (tc *CustomMetricTestCase) Run(ctx context.Context) { 395 projectID := framework.TestContext.CloudConfig.ProjectID 396 397 client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) 398 if err != nil { 399 framework.Failf("Failed to initialize gcm default client, %v", err) 400 } 401 402 // Hack for running tests locally, needed to authenticate in Stackdriver 403 // If this is your use case, create application default credentials: 404 // $ gcloud auth application-default login 405 // and uncomment following lines: 406 407 // ts, err := google.DefaultTokenSource(oauth2.NoContext) 408 // framework.Logf("Couldn't get application default credentials, %v", err) 409 // if err != nil { 410 // framework.Failf("Error accessing application default credentials, %v", err) 411 // } 412 // client = oauth2.NewClient(oauth2.NoContext, ts) 413 414 gcmService, err := gcm.NewService(ctx, option.WithHTTPClient(client)) 415 if err != nil { 416 framework.Failf("Failed to create gcm service, %v", err) 417 } 418 419 // Set up a cluster: create a custom metric and set up k8s-sd adapter 420 err = monitoring.CreateDescriptors(gcmService, projectID) 421 if err != nil { 422 framework.Failf("Failed to create metric descriptor: %v", err) 423 } 424 defer monitoring.CleanupDescriptors(gcmService, projectID) 425 426 err = monitoring.CreateAdapter(monitoring.AdapterDefault) 427 defer monitoring.CleanupAdapter(monitoring.AdapterDefault) 428 if err != nil { 429 framework.Failf("Failed to set up: %v", err) 430 } 431 432 // Run application that exports the metric 433 err = createDeploymentToScale(ctx, tc.framework, tc.kubeClient, tc.deployment, tc.pod) 434 if err != nil { 435 framework.Failf("Failed to create stackdriver-exporter pod: %v", err) 436 } 437 ginkgo.DeferCleanup(cleanupDeploymentsToScale, tc.framework, tc.kubeClient, tc.deployment, tc.pod) 438 439 // Wait for the deployment to run 440 waitForReplicas(ctx, tc.deployment.ObjectMeta.Name, tc.framework.Namespace.ObjectMeta.Name, tc.kubeClient, 15*time.Minute, tc.initialReplicas) 441 442 // Autoscale the deployment 443 _, err = tc.kubeClient.AutoscalingV2().HorizontalPodAutoscalers(tc.framework.Namespace.ObjectMeta.Name).Create(ctx, tc.hpa, metav1.CreateOptions{}) 444 if err != nil { 445 framework.Failf("Failed to create HPA: %v", err) 446 } 447 ginkgo.DeferCleanup(framework.IgnoreNotFound(tc.kubeClient.AutoscalingV2().HorizontalPodAutoscalers(tc.framework.Namespace.ObjectMeta.Name).Delete), tc.hpa.ObjectMeta.Name, metav1.DeleteOptions{}) 448 449 waitForReplicas(ctx, tc.deployment.ObjectMeta.Name, tc.framework.Namespace.ObjectMeta.Name, tc.kubeClient, 15*time.Minute, tc.scaledReplicas) 450 451 if tc.verifyStability { 452 ensureDesiredReplicasInRange(ctx, tc.deployment.ObjectMeta.Name, tc.framework.Namespace.ObjectMeta.Name, tc.kubeClient, tc.scaledReplicas, tc.scaledReplicas, 10*time.Minute) 453 } 454 } 455 456 func createDeploymentToScale(ctx context.Context, f *framework.Framework, cs clientset.Interface, deployment *appsv1.Deployment, pod *v1.Pod) error { 457 if deployment != nil { 458 _, err := cs.AppsV1().Deployments(f.Namespace.ObjectMeta.Name).Create(ctx, deployment, metav1.CreateOptions{}) 459 if err != nil { 460 return err 461 } 462 } 463 if pod != nil { 464 _, err := cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Create(ctx, pod, metav1.CreateOptions{}) 465 if err != nil { 466 return err 467 } 468 } 469 return nil 470 } 471 472 func cleanupDeploymentsToScale(ctx context.Context, f *framework.Framework, cs clientset.Interface, deployment *appsv1.Deployment, pod *v1.Pod) { 473 if deployment != nil { 474 _ = cs.AppsV1().Deployments(f.Namespace.ObjectMeta.Name).Delete(ctx, deployment.ObjectMeta.Name, metav1.DeleteOptions{}) 475 } 476 if pod != nil { 477 _ = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Delete(ctx, pod.ObjectMeta.Name, metav1.DeleteOptions{}) 478 } 479 } 480 481 func podMetricSpecWithAverageValueTarget(metric string, targetValue int64) autoscalingv2.MetricSpec { 482 return autoscalingv2.MetricSpec{ 483 Type: autoscalingv2.PodsMetricSourceType, 484 Pods: &autoscalingv2.PodsMetricSource{ 485 Metric: autoscalingv2.MetricIdentifier{ 486 Name: metric, 487 }, 488 Target: autoscalingv2.MetricTarget{ 489 Type: autoscalingv2.AverageValueMetricType, 490 AverageValue: resource.NewQuantity(targetValue, resource.DecimalSI), 491 }, 492 }, 493 } 494 } 495 496 func objectMetricSpecWithValueTarget(targetValue int64) autoscalingv2.MetricSpec { 497 return autoscalingv2.MetricSpec{ 498 Type: autoscalingv2.ObjectMetricSourceType, 499 Object: &autoscalingv2.ObjectMetricSource{ 500 Metric: autoscalingv2.MetricIdentifier{ 501 Name: monitoring.CustomMetricName, 502 }, 503 DescribedObject: autoscalingv2.CrossVersionObjectReference{ 504 Kind: "Pod", 505 Name: stackdriverExporterPod, 506 }, 507 Target: autoscalingv2.MetricTarget{ 508 Type: autoscalingv2.ValueMetricType, 509 Value: resource.NewQuantity(targetValue, resource.DecimalSI), 510 }, 511 }, 512 } 513 } 514 515 func resourceMetricSpecWithAverageUtilizationTarget(targetValue int32) autoscalingv2.MetricSpec { 516 return autoscalingv2.MetricSpec{ 517 Type: autoscalingv2.ResourceMetricSourceType, 518 Resource: &autoscalingv2.ResourceMetricSource{ 519 Name: v1.ResourceCPU, 520 Target: autoscalingv2.MetricTarget{ 521 Type: autoscalingv2.UtilizationMetricType, 522 AverageUtilization: &targetValue, 523 }, 524 }, 525 } 526 } 527 528 func containerResourceMetricSpecWithAverageUtilizationTarget(containerName string, targetValue int32) autoscalingv2.MetricSpec { 529 return autoscalingv2.MetricSpec{ 530 Type: autoscalingv2.ContainerResourceMetricSourceType, 531 ContainerResource: &autoscalingv2.ContainerResourceMetricSource{ 532 Name: v1.ResourceCPU, 533 Container: containerName, 534 Target: autoscalingv2.MetricTarget{ 535 Type: autoscalingv2.UtilizationMetricType, 536 AverageUtilization: &targetValue, 537 }, 538 }, 539 } 540 } 541 542 func externalMetricSpecWithTarget(metric string, namespace string, target externalMetricTarget) autoscalingv2.MetricSpec { 543 selector := &metav1.LabelSelector{ 544 MatchLabels: map[string]string{"resource.type": "k8s_pod"}, 545 MatchExpressions: []metav1.LabelSelectorRequirement{ 546 { 547 Key: "resource.labels.namespace_name", 548 Operator: metav1.LabelSelectorOpIn, 549 Values: []string{namespace}, 550 }, 551 { 552 Key: "resource.labels.pod_name", 553 Operator: metav1.LabelSelectorOpExists, 554 Values: []string{}, 555 }, 556 }, 557 } 558 metricSpec := autoscalingv2.MetricSpec{ 559 Type: autoscalingv2.ExternalMetricSourceType, 560 External: &autoscalingv2.ExternalMetricSource{ 561 Metric: autoscalingv2.MetricIdentifier{ 562 Name: "custom.googleapis.com|" + metric, 563 Selector: selector, 564 }, 565 }, 566 } 567 if target.isAverage { 568 metricSpec.External.Target.Type = autoscalingv2.AverageValueMetricType 569 metricSpec.External.Target.AverageValue = resource.NewQuantity(target.value, resource.DecimalSI) 570 } else { 571 metricSpec.External.Target.Type = autoscalingv2.ValueMetricType 572 metricSpec.External.Target.Value = resource.NewQuantity(target.value, resource.DecimalSI) 573 } 574 return metricSpec 575 } 576 577 func hpa(name, namespace, deploymentName string, minReplicas, maxReplicas int32, metricSpecs []autoscalingv2.MetricSpec) *autoscalingv2.HorizontalPodAutoscaler { 578 return &autoscalingv2.HorizontalPodAutoscaler{ 579 ObjectMeta: metav1.ObjectMeta{ 580 Name: name, 581 Namespace: namespace, 582 }, 583 Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ 584 Metrics: metricSpecs, 585 MinReplicas: &minReplicas, 586 MaxReplicas: maxReplicas, 587 ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ 588 APIVersion: "apps/v1", 589 Kind: "Deployment", 590 Name: deploymentName, 591 }, 592 }, 593 } 594 } 595 596 func waitForReplicas(ctx context.Context, deploymentName, namespace string, cs clientset.Interface, timeout time.Duration, desiredReplicas int) { 597 interval := 20 * time.Second 598 err := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) { 599 deployment, err := cs.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) 600 if err != nil { 601 framework.Failf("Failed to get replication controller %s: %v", deployment, err) 602 } 603 replicas := int(deployment.Status.ReadyReplicas) 604 framework.Logf("waiting for %d replicas (current: %d)", desiredReplicas, replicas) 605 return replicas == desiredReplicas, nil // Expected number of replicas found. Exit. 606 }) 607 if err != nil { 608 framework.Failf("Timeout waiting %v for %v replicas", timeout, desiredReplicas) 609 } 610 } 611 612 func ensureDesiredReplicasInRange(ctx context.Context, deploymentName, namespace string, cs clientset.Interface, minDesiredReplicas, maxDesiredReplicas int, timeout time.Duration) { 613 interval := 60 * time.Second 614 err := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) { 615 deployment, err := cs.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) 616 if err != nil { 617 framework.Failf("Failed to get replication controller %s: %v", deployment, err) 618 } 619 replicas := int(deployment.Status.ReadyReplicas) 620 framework.Logf("expecting there to be in [%d, %d] replicas (are: %d)", minDesiredReplicas, maxDesiredReplicas, replicas) 621 if replicas < minDesiredReplicas { 622 return false, fmt.Errorf("number of replicas below target") 623 } else if replicas > maxDesiredReplicas { 624 return false, fmt.Errorf("number of replicas above target") 625 } else { 626 return false, nil // Expected number of replicas found. Continue polling until timeout. 627 } 628 }) 629 // The call above always returns an error, but if it is timeout, it's OK (condition satisfied all the time). 630 if wait.Interrupted(err) { 631 framework.Logf("Number of replicas was stable over %v", timeout) 632 return 633 } 634 framework.ExpectNoErrorWithOffset(1, err) 635 } 636 637 func noExporterDeployment(name, namespace string, replicas int32) *appsv1.Deployment { 638 d := e2edeployment.NewDeployment(name, replicas, map[string]string{"name": name}, "", "", appsv1.RollingUpdateDeploymentStrategyType) 639 d.ObjectMeta.Namespace = namespace 640 d.Spec.Template.Spec = v1.PodSpec{Containers: []v1.Container{ 641 { 642 Name: "sleeper", 643 Image: "registry.k8s.io/e2e-test-images/agnhost:2.40", 644 ImagePullPolicy: v1.PullAlways, 645 Command: []string{"/agnhost"}, 646 Args: []string{"pause"}, // do nothing forever 647 }, 648 }} 649 return d 650 }