k8s.io/apiserver@v0.31.1/pkg/util/flowcontrol/borrowing_test.go (about) 1 /* 2 Copyright 2022 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package flowcontrol 18 19 import ( 20 "context" 21 "fmt" 22 "sync" 23 "testing" 24 "time" 25 26 flowcontrol "k8s.io/api/flowcontrol/v1" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 "k8s.io/apimachinery/pkg/runtime" 29 "k8s.io/apimachinery/pkg/util/wait" 30 "k8s.io/apiserver/pkg/authentication/user" 31 "k8s.io/apiserver/pkg/endpoints/request" 32 fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing" 33 "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/eventclock" 34 fqs "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset" 35 "k8s.io/apiserver/pkg/util/flowcontrol/metrics" 36 fcrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" 37 "k8s.io/client-go/informers" 38 clientsetfake "k8s.io/client-go/kubernetes/fake" 39 "k8s.io/utils/ptr" 40 ) 41 42 type borrowingTestConstraints struct { 43 lendable, borrowing int32 44 } 45 46 // TestBorrowing tests borrowing of concurrency between priority levels. 47 // It runs two scenarios, one where the borrowing hits the limit on 48 // lendable concurrency and one where the borrowing hits the limit on 49 // borrowing of concurrency. 50 // Both scenarios are the same except for the limits. 51 // The test defines two priority levels, identified as "flows" 0 and 1. 52 // Both priority levels have a nominal concurrency limit of 12. 53 // The test maintains 24 concurrent clients for priority level 0 54 // and 6 for level 1, 55 // using an exec func that simply sleeps for 250 ms, for 56 // 25 seconds. The first 10 seconds of behavior are ignored, allowing 57 // the borrowing to start at any point during that time. The test 58 // continues for another 15 seconds, and checks that the delivered 59 // concurrency is about 16 for flow 0 and 6 for flow 1. 60 func TestBorrowing(t *testing.T) { 61 clientsPerFlow := [2]int{24, 6} 62 metrics.Register() 63 for _, testCase := range []struct { 64 name string 65 constraints []borrowingTestConstraints 66 }{ 67 { 68 name: "lendable-limited", 69 constraints: []borrowingTestConstraints{ 70 {lendable: 50, borrowing: 67}, 71 {lendable: 33, borrowing: 50}, 72 }}, 73 { 74 name: "borrowing-limited", 75 constraints: []borrowingTestConstraints{ 76 {lendable: 50, borrowing: 33}, 77 {lendable: 67, borrowing: 50}, 78 }}, 79 } { 80 t.Run(testCase.name, func(t *testing.T) { 81 fsObjs := make([]*flowcontrol.FlowSchema, 2) 82 plcObjs := make([]*flowcontrol.PriorityLevelConfiguration, 2) 83 usernames := make([]string, 2) 84 cfgObjs := []runtime.Object{} 85 for flow := 0; flow < 2; flow++ { 86 usernames[flow] = fmt.Sprintf("test-user%d", flow) 87 plName := fmt.Sprintf("test-pl%d", flow) 88 fsObjs[flow] = &flowcontrol.FlowSchema{ 89 ObjectMeta: metav1.ObjectMeta{ 90 Name: fmt.Sprintf("test-fs%d", flow), 91 }, 92 Spec: flowcontrol.FlowSchemaSpec{ 93 MatchingPrecedence: 100, 94 PriorityLevelConfiguration: flowcontrol.PriorityLevelConfigurationReference{ 95 Name: plName, 96 }, 97 DistinguisherMethod: &flowcontrol.FlowDistinguisherMethod{ 98 Type: flowcontrol.FlowDistinguisherMethodByUserType, 99 }, 100 Rules: []flowcontrol.PolicyRulesWithSubjects{{ 101 Subjects: []flowcontrol.Subject{{ 102 Kind: flowcontrol.SubjectKindUser, 103 User: &flowcontrol.UserSubject{Name: usernames[flow]}, 104 }}, 105 NonResourceRules: []flowcontrol.NonResourcePolicyRule{{ 106 Verbs: []string{"*"}, 107 NonResourceURLs: []string{"*"}, 108 }}, 109 }}, 110 }, 111 } 112 plcObjs[flow] = &flowcontrol.PriorityLevelConfiguration{ 113 ObjectMeta: metav1.ObjectMeta{ 114 Name: plName, 115 }, 116 Spec: flowcontrol.PriorityLevelConfigurationSpec{ 117 Type: flowcontrol.PriorityLevelEnablementLimited, 118 Limited: &flowcontrol.LimitedPriorityLevelConfiguration{ 119 NominalConcurrencyShares: ptr.To(int32(100)), 120 LendablePercent: &testCase.constraints[flow].lendable, 121 BorrowingLimitPercent: &testCase.constraints[flow].borrowing, 122 LimitResponse: flowcontrol.LimitResponse{ 123 Type: flowcontrol.LimitResponseTypeQueue, 124 Queuing: &flowcontrol.QueuingConfiguration{ 125 Queues: 10, 126 HandSize: 2, 127 QueueLengthLimit: 10, 128 }, 129 }, 130 }, 131 }, 132 } 133 cfgObjs = append(cfgObjs, fsObjs[flow], plcObjs[flow]) 134 } 135 clientset := clientsetfake.NewSimpleClientset(cfgObjs...) 136 informerFactory := informers.NewSharedInformerFactory(clientset, time.Second) 137 flowcontrolClient := clientset.FlowcontrolV1() 138 clk := eventclock.Real{} 139 controller := newTestableController(TestableConfig{ 140 Name: "Controller", 141 Clock: clk, 142 AsFieldManager: ConfigConsumerAsFieldManager, 143 FoundToDangling: func(found bool) bool { return !found }, 144 InformerFactory: informerFactory, 145 FlowcontrolClient: flowcontrolClient, 146 ServerConcurrencyLimit: 24, 147 ReqsGaugeVec: metrics.PriorityLevelConcurrencyGaugeVec, 148 ExecSeatsGaugeVec: metrics.PriorityLevelExecutionSeatsGaugeVec, 149 QueueSetFactory: fqs.NewQueueSetFactory(clk), 150 }) 151 152 ctx, cancel := context.WithTimeout(context.Background(), 50*time.Second) 153 stopCh := ctx.Done() 154 controllerCompletionCh := make(chan error) 155 156 informerFactory.Start(stopCh) 157 158 status := informerFactory.WaitForCacheSync(ctx.Done()) 159 if names := unsynced(status); len(names) > 0 { 160 t.Fatalf("WaitForCacheSync did not successfully complete, resources=%#v", names) 161 } 162 163 go func() { 164 controllerCompletionCh <- controller.Run(stopCh) 165 }() 166 167 // ensure that the controller has run its first loop. 168 err := wait.PollImmediate(100*time.Millisecond, 5*time.Second, func() (done bool, err error) { 169 return controller.hasPriorityLevelState(plcObjs[0].Name), nil 170 }) 171 if err != nil { 172 t.Errorf("expected the controller to reconcile the priority level configuration object: %s, error: %s", plcObjs[0].Name, err) 173 } 174 175 concIntegrators := make([]fq.Integrator, 2) 176 reqInfo := &request.RequestInfo{ 177 IsResourceRequest: false, 178 Path: "/foobar", 179 Verb: "GET", 180 } 181 noteFn := func(fs *flowcontrol.FlowSchema, plc *flowcontrol.PriorityLevelConfiguration, fd string) {} 182 workEstr := func() fcrequest.WorkEstimate { return fcrequest.WorkEstimate{InitialSeats: 1} } 183 qnf := fq.QueueNoteFn(func(bool) {}) 184 var startWG sync.WaitGroup 185 startWG.Add(clientsPerFlow[0] + clientsPerFlow[1]) 186 // Launch 20 client threads for each flow 187 for flow := 0; flow < 2; flow++ { 188 username := usernames[flow] 189 flowUser := testUser{name: username} 190 rd := RequestDigest{ 191 RequestInfo: reqInfo, 192 User: flowUser, 193 } 194 concIntegrator := fq.NewNamedIntegrator(clk, username) 195 concIntegrators[flow] = concIntegrator 196 exec := func() { 197 concIntegrator.Inc() 198 clk.Sleep(250 * time.Millisecond) 199 concIntegrator.Dec() 200 } 201 nThreads := clientsPerFlow[flow] 202 for thread := 0; thread < nThreads; thread++ { 203 go func() { 204 startWG.Done() 205 wait.Until(func() { controller.Handle(ctx, rd, noteFn, workEstr, qnf, exec) }, 0, ctx.Done()) 206 }() 207 } 208 } 209 startWG.Wait() 210 // Make sure the controller has had time to sense the load and adjust 211 clk.Sleep(10 * time.Second) 212 // Start the stats that matter from now 213 for _, ci := range concIntegrators { 214 ci.Reset() 215 } 216 // Run for 15 seconds 217 clk.Sleep(15 * time.Second) 218 // Collect the delivered concurrency stats 219 results0 := concIntegrators[0].Reset() 220 results1 := concIntegrators[1].Reset() 221 // shut down all the async stuff 222 cancel() 223 224 // Do the checking 225 226 t.Log("waiting for the controller Run function to shutdown gracefully") 227 controllerErr := <-controllerCompletionCh 228 close(controllerCompletionCh) 229 if controllerErr != nil { 230 t.Errorf("expected nil error from controller Run function, but got: %#v", controllerErr) 231 } 232 if results0.Average < 15.5 || results0.Average > 16.1 { 233 t.Errorf("Flow 0 got average concurrency of %v but expected about 16", results0.Average) 234 } else { 235 t.Logf("Flow 0 got average concurrency of %v and expected about 16", results0.Average) 236 } 237 if results1.Average < 5.5 || results1.Average > 6.1 { 238 t.Errorf("Flow 1 got average concurrency of %v but expected about 6", results1.Average) 239 } else { 240 t.Logf("Flow 1 got average concurrency of %v and expected about 6", results1.Average) 241 } 242 }) 243 } 244 } 245 246 type testUser struct{ name string } 247 248 func (tu testUser) GetName() string { return tu.name } 249 func (tu testUser) GetUID() string { return tu.name } 250 func (tu testUser) GetGroups() []string { return []string{user.AllAuthenticated} } 251 func (tu testUser) GetExtra() map[string][]string { return map[string][]string{} }