k8s.io/kubernetes@v1.29.3/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go (about) 1 /* 2 Copyright 2022 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package autoscaling 18 19 import ( 20 "context" 21 "time" 22 23 autoscalingv2 "k8s.io/api/autoscaling/v2" 24 "k8s.io/kubernetes/test/e2e/feature" 25 "k8s.io/kubernetes/test/e2e/framework" 26 e2eautoscaling "k8s.io/kubernetes/test/e2e/framework/autoscaling" 27 admissionapi "k8s.io/pod-security-admission/api" 28 29 "github.com/onsi/ginkgo/v2" 30 "github.com/onsi/gomega" 31 ) 32 33 var _ = SIGDescribe(feature.HPA, framework.WithSerial(), framework.WithSlow(), "Horizontal pod autoscaling (non-default behavior)", func() { 34 f := framework.NewDefaultFramework("horizontal-pod-autoscaling") 35 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged 36 37 hpaName := "consumer" 38 39 podCPURequest := 500 40 targetCPUUtilizationPercent := 25 41 42 // usageForReplicas returns usage for (n - 0.5) replicas as if they would consume all CPU 43 // under the target. The 0.5 replica reduction is to accommodate for the deviation between 44 // the actual consumed cpu and requested usage by the ResourceConsumer. 45 // HPA rounds up the recommendations. So, if the usage is e.g. for 3.5 replicas, 46 // the recommended replica number will be 4. 47 usageForReplicas := func(replicas int) int { 48 usagePerReplica := podCPURequest * targetCPUUtilizationPercent / 100 49 return replicas*usagePerReplica - usagePerReplica/2 50 } 51 52 fullWindowOfNewUsage := 30 * time.Second 53 windowWithOldUsagePasses := 30 * time.Second 54 newPodMetricsDelay := 15 * time.Second 55 metricsAvailableDelay := fullWindowOfNewUsage + windowWithOldUsagePasses + newPodMetricsDelay 56 57 hpaReconciliationInterval := 15 * time.Second 58 actuationDelay := 10 * time.Second 59 maxHPAReactionTime := metricsAvailableDelay + hpaReconciliationInterval + actuationDelay 60 61 maxConsumeCPUDelay := 30 * time.Second 62 waitForReplicasPollInterval := 20 * time.Second 63 maxResourceConsumerDelay := maxConsumeCPUDelay + waitForReplicasPollInterval 64 65 waitBuffer := 1 * time.Minute 66 67 ginkgo.Describe("with short downscale stabilization window", func() { 68 ginkgo.It("should scale down soon after the stabilization period", func(ctx context.Context) { 69 ginkgo.By("setting up resource consumer and HPA") 70 initPods := 1 71 initCPUUsageTotal := usageForReplicas(initPods) 72 upScaleStabilization := 0 * time.Minute 73 downScaleStabilization := 1 * time.Minute 74 75 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 76 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 77 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 78 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 79 ) 80 ginkgo.DeferCleanup(rc.CleanUp) 81 82 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 83 rc, int32(targetCPUUtilizationPercent), 1, 5, 84 e2eautoscaling.HPABehaviorWithStabilizationWindows(upScaleStabilization, downScaleStabilization), 85 ) 86 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 87 88 // making sure HPA is ready, doing its job and already has a recommendation recorded 89 // for stabilization logic before lowering the consumption 90 ginkgo.By("triggering scale up to record a recommendation") 91 rc.ConsumeCPU(usageForReplicas(3)) 92 rc.WaitForReplicas(ctx, 3, maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) 93 94 ginkgo.By("triggering scale down by lowering consumption") 95 rc.ConsumeCPU(usageForReplicas(2)) 96 waitStart := time.Now() 97 rc.WaitForReplicas(ctx, 2, downScaleStabilization+maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) 98 timeWaited := time.Now().Sub(waitStart) 99 100 ginkgo.By("verifying time waited for a scale down") 101 framework.Logf("time waited for scale down: %s", timeWaited) 102 gomega.Expect(timeWaited).To(gomega.BeNumerically(">", downScaleStabilization), "waited %s, wanted more than %s", timeWaited, downScaleStabilization) 103 deadline := downScaleStabilization + maxHPAReactionTime + maxResourceConsumerDelay 104 gomega.Expect(timeWaited).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaited, deadline) 105 }) 106 }) 107 108 ginkgo.Describe("with long upscale stabilization window", func() { 109 ginkgo.It("should scale up only after the stabilization period", func(ctx context.Context) { 110 ginkgo.By("setting up resource consumer and HPA") 111 initPods := 2 112 initCPUUsageTotal := usageForReplicas(initPods) 113 upScaleStabilization := 3 * time.Minute 114 downScaleStabilization := 0 * time.Minute 115 116 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 117 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 118 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 119 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 120 ) 121 ginkgo.DeferCleanup(rc.CleanUp) 122 123 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 124 rc, int32(targetCPUUtilizationPercent), 1, 10, 125 e2eautoscaling.HPABehaviorWithStabilizationWindows(upScaleStabilization, downScaleStabilization), 126 ) 127 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 128 129 // making sure HPA is ready, doing its job and already has a recommendation recorded 130 // for stabilization logic before increasing the consumption 131 ginkgo.By("triggering scale down to record a recommendation") 132 rc.ConsumeCPU(usageForReplicas(1)) 133 rc.WaitForReplicas(ctx, 1, maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) 134 135 ginkgo.By("triggering scale up by increasing consumption") 136 rc.ConsumeCPU(usageForReplicas(3)) 137 waitStart := time.Now() 138 rc.WaitForReplicas(ctx, 3, upScaleStabilization+maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) 139 timeWaited := time.Now().Sub(waitStart) 140 141 ginkgo.By("verifying time waited for a scale up") 142 framework.Logf("time waited for scale up: %s", timeWaited) 143 gomega.Expect(timeWaited).To(gomega.BeNumerically(">", upScaleStabilization), "waited %s, wanted more than %s", timeWaited, upScaleStabilization) 144 deadline := upScaleStabilization + maxHPAReactionTime + maxResourceConsumerDelay 145 gomega.Expect(timeWaited).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaited, deadline) 146 }) 147 }) 148 149 ginkgo.Describe("with autoscaling disabled", func() { 150 ginkgo.It("shouldn't scale up", func(ctx context.Context) { 151 ginkgo.By("setting up resource consumer and HPA") 152 initPods := 1 153 initCPUUsageTotal := usageForReplicas(initPods) 154 155 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 156 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 157 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 158 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 159 ) 160 ginkgo.DeferCleanup(rc.CleanUp) 161 162 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 163 rc, int32(targetCPUUtilizationPercent), 1, 10, e2eautoscaling.HPABehaviorWithScaleDisabled(e2eautoscaling.ScaleUpDirection), 164 ) 165 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 166 167 waitDeadline := maxHPAReactionTime + maxResourceConsumerDelay + waitBuffer 168 169 ginkgo.By("trying to trigger scale up") 170 rc.ConsumeCPU(usageForReplicas(8)) 171 waitStart := time.Now() 172 173 rc.EnsureDesiredReplicasInRange(ctx, initPods, initPods, waitDeadline, hpa.Name) 174 timeWaited := time.Now().Sub(waitStart) 175 176 ginkgo.By("verifying time waited for a scale up") 177 framework.Logf("time waited for scale up: %s", timeWaited) 178 gomega.Expect(timeWaited).To(gomega.BeNumerically(">", waitDeadline), "waited %s, wanted to wait more than %s", timeWaited, waitDeadline) 179 180 ginkgo.By("verifying number of replicas") 181 replicas := rc.GetReplicas(ctx) 182 gomega.Expect(replicas).To(gomega.BeNumerically("==", initPods), "had %s replicas, still have %s replicas after time deadline", initPods, replicas) 183 }) 184 185 ginkgo.It("shouldn't scale down", func(ctx context.Context) { 186 ginkgo.By("setting up resource consumer and HPA") 187 initPods := 3 188 initCPUUsageTotal := usageForReplicas(initPods) 189 190 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 191 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 192 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 193 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 194 ) 195 ginkgo.DeferCleanup(rc.CleanUp) 196 197 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 198 rc, int32(targetCPUUtilizationPercent), 1, 10, e2eautoscaling.HPABehaviorWithScaleDisabled(e2eautoscaling.ScaleDownDirection), 199 ) 200 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 201 202 defaultDownscaleStabilisation := 5 * time.Minute 203 waitDeadline := maxHPAReactionTime + maxResourceConsumerDelay + defaultDownscaleStabilisation 204 205 ginkgo.By("trying to trigger scale down") 206 rc.ConsumeCPU(usageForReplicas(1)) 207 waitStart := time.Now() 208 209 rc.EnsureDesiredReplicasInRange(ctx, initPods, initPods, waitDeadline, hpa.Name) 210 timeWaited := time.Now().Sub(waitStart) 211 212 ginkgo.By("verifying time waited for a scale down") 213 framework.Logf("time waited for scale down: %s", timeWaited) 214 gomega.Expect(timeWaited).To(gomega.BeNumerically(">", waitDeadline), "waited %s, wanted to wait more than %s", timeWaited, waitDeadline) 215 216 ginkgo.By("verifying number of replicas") 217 replicas := rc.GetReplicas(ctx) 218 gomega.Expect(replicas).To(gomega.BeNumerically("==", initPods), "had %s replicas, still have %s replicas after time deadline", initPods, replicas) 219 }) 220 221 }) 222 223 ginkgo.Describe("with scale limited by number of Pods rate", func() { 224 ginkgo.It("should scale up no more than given number of Pods per minute", func(ctx context.Context) { 225 ginkgo.By("setting up resource consumer and HPA") 226 initPods := 1 227 initCPUUsageTotal := usageForReplicas(initPods) 228 limitWindowLength := 1 * time.Minute 229 podsLimitPerMinute := 1 230 231 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 232 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 233 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 234 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 235 ) 236 ginkgo.DeferCleanup(rc.CleanUp) 237 238 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 239 rc, int32(targetCPUUtilizationPercent), 1, 10, 240 e2eautoscaling.HPABehaviorWithScaleLimitedByNumberOfPods(e2eautoscaling.ScaleUpDirection, int32(podsLimitPerMinute), int32(limitWindowLength.Seconds())), 241 ) 242 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 243 244 ginkgo.By("triggering scale up by increasing consumption") 245 rc.ConsumeCPU(usageForReplicas(3)) 246 247 waitStart := time.Now() 248 rc.WaitForReplicas(ctx, 2, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) 249 timeWaitedFor2 := time.Now().Sub(waitStart) 250 251 waitStart = time.Now() 252 rc.WaitForReplicas(ctx, 3, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) 253 timeWaitedFor3 := time.Now().Sub(waitStart) 254 255 ginkgo.By("verifying time waited for a scale up to 2 replicas") 256 deadline := limitWindowLength + maxHPAReactionTime + maxResourceConsumerDelay 257 // First scale event can happen right away, as there were no scale events in the past. 258 gomega.Expect(timeWaitedFor2).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaitedFor2, deadline) 259 260 ginkgo.By("verifying time waited for a scale up to 3 replicas") 261 // Second scale event needs to respect limit window. 262 gomega.Expect(timeWaitedFor3).To(gomega.BeNumerically(">", limitWindowLength), "waited %s, wanted to wait more than %s", timeWaitedFor3, limitWindowLength) 263 gomega.Expect(timeWaitedFor3).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaitedFor3, deadline) 264 }) 265 266 ginkgo.It("should scale down no more than given number of Pods per minute", func(ctx context.Context) { 267 ginkgo.By("setting up resource consumer and HPA") 268 initPods := 3 269 initCPUUsageTotal := usageForReplicas(initPods) 270 limitWindowLength := 1 * time.Minute 271 podsLimitPerMinute := 1 272 273 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 274 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 275 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 276 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 277 ) 278 ginkgo.DeferCleanup(rc.CleanUp) 279 280 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 281 rc, int32(targetCPUUtilizationPercent), 1, 10, 282 e2eautoscaling.HPABehaviorWithScaleLimitedByNumberOfPods(e2eautoscaling.ScaleDownDirection, int32(podsLimitPerMinute), int32(limitWindowLength.Seconds())), 283 ) 284 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 285 286 ginkgo.By("triggering scale down by lowering consumption") 287 rc.ConsumeCPU(usageForReplicas(1)) 288 289 waitStart := time.Now() 290 rc.WaitForReplicas(ctx, 2, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) 291 timeWaitedFor2 := time.Now().Sub(waitStart) 292 293 waitStart = time.Now() 294 rc.WaitForReplicas(ctx, 1, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) 295 timeWaitedFor1 := time.Now().Sub(waitStart) 296 297 ginkgo.By("verifying time waited for a scale down to 2 replicas") 298 deadline := limitWindowLength + maxHPAReactionTime + maxResourceConsumerDelay 299 // First scale event can happen right away, as there were no scale events in the past. 300 gomega.Expect(timeWaitedFor2).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaitedFor2, deadline) 301 302 ginkgo.By("verifying time waited for a scale down to 1 replicas") 303 // Second scale event needs to respect limit window. 304 gomega.Expect(timeWaitedFor1).To(gomega.BeNumerically(">", limitWindowLength), "waited %s, wanted more than %s", timeWaitedFor1, limitWindowLength) 305 gomega.Expect(timeWaitedFor1).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaitedFor1, deadline) 306 }) 307 }) 308 309 ginkgo.Describe("with scale limited by percentage", func() { 310 ginkgo.It("should scale up no more than given percentage of current Pods per minute", func(ctx context.Context) { 311 ginkgo.By("setting up resource consumer and HPA") 312 initPods := 2 313 initCPUUsageTotal := usageForReplicas(initPods) 314 limitWindowLength := 1 * time.Minute 315 percentageLimitPerMinute := 50 316 317 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 318 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 319 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 320 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 321 ) 322 ginkgo.DeferCleanup(rc.CleanUp) 323 324 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 325 rc, int32(targetCPUUtilizationPercent), 1, 10, 326 e2eautoscaling.HPABehaviorWithScaleLimitedByPercentage(e2eautoscaling.ScaleUpDirection, int32(percentageLimitPerMinute), int32(limitWindowLength.Seconds())), 327 ) 328 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 329 330 ginkgo.By("triggering scale up by increasing consumption") 331 rc.ConsumeCPU(usageForReplicas(8)) 332 333 waitStart := time.Now() 334 rc.WaitForReplicas(ctx, 3, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) 335 timeWaitedFor3 := time.Now().Sub(waitStart) 336 337 waitStart = time.Now() 338 // Scale up limited by percentage takes ceiling, so new replicas number is ceil(3 * 1.5) = ceil(4.5) = 5 339 rc.WaitForReplicas(ctx, 5, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) 340 timeWaitedFor5 := time.Now().Sub(waitStart) 341 342 ginkgo.By("verifying time waited for a scale up to 3 replicas") 343 deadline := limitWindowLength + maxHPAReactionTime + maxResourceConsumerDelay 344 // First scale event can happen right away, as there were no scale events in the past. 345 gomega.Expect(timeWaitedFor3).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaitedFor3, deadline) 346 347 ginkgo.By("verifying time waited for a scale up to 5 replicas") 348 // Second scale event needs to respect limit window. 349 gomega.Expect(timeWaitedFor5).To(gomega.BeNumerically(">", limitWindowLength), "waited %s, wanted to wait more than %s", timeWaitedFor5, limitWindowLength) 350 gomega.Expect(timeWaitedFor5).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaitedFor5, deadline) 351 }) 352 353 ginkgo.It("should scale down no more than given percentage of current Pods per minute", func(ctx context.Context) { 354 ginkgo.By("setting up resource consumer and HPA") 355 initPods := 7 356 initCPUUsageTotal := usageForReplicas(initPods) 357 limitWindowLength := 1 * time.Minute 358 percentageLimitPerMinute := 25 359 360 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 361 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 362 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 363 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 364 ) 365 ginkgo.DeferCleanup(rc.CleanUp) 366 367 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 368 rc, int32(targetCPUUtilizationPercent), 1, 10, 369 e2eautoscaling.HPABehaviorWithScaleLimitedByPercentage(e2eautoscaling.ScaleDownDirection, int32(percentageLimitPerMinute), int32(limitWindowLength.Seconds())), 370 ) 371 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 372 373 ginkgo.By("triggering scale down by lowering consumption") 374 rc.ConsumeCPU(usageForReplicas(1)) 375 376 waitStart := time.Now() 377 rc.WaitForReplicas(ctx, 5, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) 378 timeWaitedFor5 := time.Now().Sub(waitStart) 379 380 waitStart = time.Now() 381 // Scale down limited by percentage takes floor, so new replicas number is floor(5 * 0.75) = floor(3.75) = 3 382 rc.WaitForReplicas(ctx, 3, maxHPAReactionTime+maxResourceConsumerDelay+limitWindowLength) 383 timeWaitedFor3 := time.Now().Sub(waitStart) 384 385 ginkgo.By("verifying time waited for a scale down to 5 replicas") 386 deadline := limitWindowLength + maxHPAReactionTime + maxResourceConsumerDelay 387 // First scale event can happen right away, as there were no scale events in the past. 388 gomega.Expect(timeWaitedFor5).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaitedFor5, deadline) 389 390 ginkgo.By("verifying time waited for a scale down to 3 replicas") 391 // Second scale event needs to respect limit window. 392 gomega.Expect(timeWaitedFor3).To(gomega.BeNumerically(">", limitWindowLength), "waited %s, wanted more than %s", timeWaitedFor3, limitWindowLength) 393 gomega.Expect(timeWaitedFor3).To(gomega.BeNumerically("<", deadline), "waited %s, wanted less than %s", timeWaitedFor3, deadline) 394 }) 395 }) 396 397 ginkgo.Describe("with both scale up and down controls configured", func() { 398 waitBuffer := 2 * time.Minute 399 400 ginkgo.It("should keep recommendation within the range over two stabilization windows", func(ctx context.Context) { 401 ginkgo.By("setting up resource consumer and HPA") 402 initPods := 1 403 initCPUUsageTotal := usageForReplicas(initPods) 404 upScaleStabilization := 3 * time.Minute 405 downScaleStabilization := 3 * time.Minute 406 407 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 408 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 409 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 410 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 411 ) 412 ginkgo.DeferCleanup(rc.CleanUp) 413 414 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 415 rc, int32(targetCPUUtilizationPercent), 1, 5, 416 e2eautoscaling.HPABehaviorWithStabilizationWindows(upScaleStabilization, downScaleStabilization), 417 ) 418 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 419 420 ginkgo.By("triggering scale up by increasing consumption") 421 rc.ConsumeCPU(usageForReplicas(3)) 422 waitDeadline := upScaleStabilization 423 424 ginkgo.By("verifying number of replicas stay in desired range within stabilisation window") 425 rc.EnsureDesiredReplicasInRange(ctx, 1, 1, waitDeadline, hpa.Name) 426 427 ginkgo.By("waiting for replicas to scale up after stabilisation window passed") 428 waitStart := time.Now() 429 waitDeadline = maxHPAReactionTime + maxResourceConsumerDelay + waitBuffer 430 rc.WaitForReplicas(ctx, 3, waitDeadline) 431 timeWaited := time.Now().Sub(waitStart) 432 framework.Logf("time waited for scale up: %s", timeWaited) 433 gomega.Expect(timeWaited).To(gomega.BeNumerically("<", waitDeadline), "waited %s, wanted less than %s", timeWaited, waitDeadline) 434 435 ginkgo.By("triggering scale down by lowering consumption") 436 rc.ConsumeCPU(usageForReplicas(2)) 437 waitDeadline = downScaleStabilization 438 439 ginkgo.By("verifying number of replicas stay in desired range within stabilisation window") 440 rc.EnsureDesiredReplicasInRange(ctx, 3, 3, waitDeadline, hpa.Name) 441 442 ginkgo.By("waiting for replicas to scale down after stabilisation window passed") 443 waitStart = time.Now() 444 waitDeadline = maxHPAReactionTime + maxResourceConsumerDelay + waitBuffer 445 rc.WaitForReplicas(ctx, 2, waitDeadline) 446 timeWaited = time.Now().Sub(waitStart) 447 framework.Logf("time waited for scale down: %s", timeWaited) 448 gomega.Expect(timeWaited).To(gomega.BeNumerically("<", waitDeadline), "waited %s, wanted less than %s", timeWaited, waitDeadline) 449 }) 450 451 ginkgo.It("should keep recommendation within the range with stabilization window and pod limit rate", func(ctx context.Context) { 452 ginkgo.By("setting up resource consumer and HPA") 453 initPods := 2 454 initCPUUsageTotal := usageForReplicas(initPods) 455 downScaleStabilization := 3 * time.Minute 456 limitWindowLength := 2 * time.Minute 457 podsLimitPerMinute := 1 458 459 rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, 460 hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, 461 initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, 462 f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, 463 ) 464 ginkgo.DeferCleanup(rc.CleanUp) 465 466 scaleUpRule := e2eautoscaling.HPAScalingRuleWithScalingPolicy(autoscalingv2.PodsScalingPolicy, int32(podsLimitPerMinute), int32(limitWindowLength.Seconds())) 467 scaleDownRule := e2eautoscaling.HPAScalingRuleWithStabilizationWindow(int32(downScaleStabilization.Seconds())) 468 hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, 469 rc, int32(targetCPUUtilizationPercent), 2, 5, 470 e2eautoscaling.HPABehaviorWithScaleUpAndDownRules(scaleUpRule, scaleDownRule), 471 ) 472 ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) 473 474 ginkgo.By("triggering scale up by increasing consumption") 475 rc.ConsumeCPU(usageForReplicas(4)) 476 waitDeadline := limitWindowLength 477 478 ginkgo.By("verifying number of replicas stay in desired range with pod limit rate") 479 rc.EnsureDesiredReplicasInRange(ctx, 2, 3, waitDeadline, hpa.Name) 480 481 ginkgo.By("waiting for replicas to scale up") 482 waitStart := time.Now() 483 waitDeadline = limitWindowLength + maxHPAReactionTime + maxResourceConsumerDelay + waitBuffer 484 rc.WaitForReplicas(ctx, 4, waitDeadline) 485 timeWaited := time.Now().Sub(waitStart) 486 framework.Logf("time waited for scale up: %s", timeWaited) 487 gomega.Default.Expect(timeWaited).To(gomega.BeNumerically("<", waitDeadline), "waited %s, wanted less than %s", timeWaited, waitDeadline) 488 489 ginkgo.By("triggering scale down by lowering consumption") 490 rc.ConsumeCPU(usageForReplicas(2)) 491 492 ginkgo.By("verifying number of replicas stay in desired range within stabilisation window") 493 waitDeadline = downScaleStabilization 494 rc.EnsureDesiredReplicasInRange(ctx, 4, 4, waitDeadline, hpa.Name) 495 496 ginkgo.By("waiting for replicas to scale down after stabilisation window passed") 497 waitStart = time.Now() 498 waitDeadline = maxHPAReactionTime + maxResourceConsumerDelay + waitBuffer 499 rc.WaitForReplicas(ctx, 2, waitDeadline) 500 timeWaited = time.Now().Sub(waitStart) 501 framework.Logf("time waited for scale down: %s", timeWaited) 502 gomega.Expect(timeWaited).To(gomega.BeNumerically("<", waitDeadline), "waited %s, wanted less than %s", timeWaited, waitDeadline) 503 }) 504 }) 505 })