k8s.io/apiserver@v0.31.1/pkg/util/flowcontrol/borrowing_test.go (about)

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package flowcontrol
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"sync"
    23  	"testing"
    24  	"time"
    25  
    26  	flowcontrol "k8s.io/api/flowcontrol/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/apimachinery/pkg/runtime"
    29  	"k8s.io/apimachinery/pkg/util/wait"
    30  	"k8s.io/apiserver/pkg/authentication/user"
    31  	"k8s.io/apiserver/pkg/endpoints/request"
    32  	fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
    33  	"k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/eventclock"
    34  	fqs "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset"
    35  	"k8s.io/apiserver/pkg/util/flowcontrol/metrics"
    36  	fcrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
    37  	"k8s.io/client-go/informers"
    38  	clientsetfake "k8s.io/client-go/kubernetes/fake"
    39  	"k8s.io/utils/ptr"
    40  )
    41  
    42  type borrowingTestConstraints struct {
    43  	lendable, borrowing int32
    44  }
    45  
    46  // TestBorrowing tests borrowing of concurrency between priority levels.
    47  // It runs two scenarios, one where the borrowing hits the limit on
    48  // lendable concurrency and one where the borrowing hits the limit on
    49  // borrowing of concurrency.
    50  // Both scenarios are the same except for the limits.
    51  // The test defines two priority levels, identified as "flows" 0 and 1.
    52  // Both priority levels have a nominal concurrency limit of 12.
    53  // The test maintains 24 concurrent clients for priority level 0
    54  // and 6 for level 1,
    55  // using an exec func that simply sleeps for 250 ms, for
    56  // 25 seconds.  The first 10 seconds of behavior are ignored, allowing
    57  // the borrowing to start at any point during that time.  The test
    58  // continues for another 15 seconds, and checks that the delivered
    59  // concurrency is about 16 for flow 0 and 6 for flow 1.
    60  func TestBorrowing(t *testing.T) {
    61  	clientsPerFlow := [2]int{24, 6}
    62  	metrics.Register()
    63  	for _, testCase := range []struct {
    64  		name        string
    65  		constraints []borrowingTestConstraints
    66  	}{
    67  		{
    68  			name: "lendable-limited",
    69  			constraints: []borrowingTestConstraints{
    70  				{lendable: 50, borrowing: 67},
    71  				{lendable: 33, borrowing: 50},
    72  			}},
    73  		{
    74  			name: "borrowing-limited",
    75  			constraints: []borrowingTestConstraints{
    76  				{lendable: 50, borrowing: 33},
    77  				{lendable: 67, borrowing: 50},
    78  			}},
    79  	} {
    80  		t.Run(testCase.name, func(t *testing.T) {
    81  			fsObjs := make([]*flowcontrol.FlowSchema, 2)
    82  			plcObjs := make([]*flowcontrol.PriorityLevelConfiguration, 2)
    83  			usernames := make([]string, 2)
    84  			cfgObjs := []runtime.Object{}
    85  			for flow := 0; flow < 2; flow++ {
    86  				usernames[flow] = fmt.Sprintf("test-user%d", flow)
    87  				plName := fmt.Sprintf("test-pl%d", flow)
    88  				fsObjs[flow] = &flowcontrol.FlowSchema{
    89  					ObjectMeta: metav1.ObjectMeta{
    90  						Name: fmt.Sprintf("test-fs%d", flow),
    91  					},
    92  					Spec: flowcontrol.FlowSchemaSpec{
    93  						MatchingPrecedence: 100,
    94  						PriorityLevelConfiguration: flowcontrol.PriorityLevelConfigurationReference{
    95  							Name: plName,
    96  						},
    97  						DistinguisherMethod: &flowcontrol.FlowDistinguisherMethod{
    98  							Type: flowcontrol.FlowDistinguisherMethodByUserType,
    99  						},
   100  						Rules: []flowcontrol.PolicyRulesWithSubjects{{
   101  							Subjects: []flowcontrol.Subject{{
   102  								Kind: flowcontrol.SubjectKindUser,
   103  								User: &flowcontrol.UserSubject{Name: usernames[flow]},
   104  							}},
   105  							NonResourceRules: []flowcontrol.NonResourcePolicyRule{{
   106  								Verbs:           []string{"*"},
   107  								NonResourceURLs: []string{"*"},
   108  							}},
   109  						}},
   110  					},
   111  				}
   112  				plcObjs[flow] = &flowcontrol.PriorityLevelConfiguration{
   113  					ObjectMeta: metav1.ObjectMeta{
   114  						Name: plName,
   115  					},
   116  					Spec: flowcontrol.PriorityLevelConfigurationSpec{
   117  						Type: flowcontrol.PriorityLevelEnablementLimited,
   118  						Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
   119  							NominalConcurrencyShares: ptr.To(int32(100)),
   120  							LendablePercent:          &testCase.constraints[flow].lendable,
   121  							BorrowingLimitPercent:    &testCase.constraints[flow].borrowing,
   122  							LimitResponse: flowcontrol.LimitResponse{
   123  								Type: flowcontrol.LimitResponseTypeQueue,
   124  								Queuing: &flowcontrol.QueuingConfiguration{
   125  									Queues:           10,
   126  									HandSize:         2,
   127  									QueueLengthLimit: 10,
   128  								},
   129  							},
   130  						},
   131  					},
   132  				}
   133  				cfgObjs = append(cfgObjs, fsObjs[flow], plcObjs[flow])
   134  			}
   135  			clientset := clientsetfake.NewSimpleClientset(cfgObjs...)
   136  			informerFactory := informers.NewSharedInformerFactory(clientset, time.Second)
   137  			flowcontrolClient := clientset.FlowcontrolV1()
   138  			clk := eventclock.Real{}
   139  			controller := newTestableController(TestableConfig{
   140  				Name:                   "Controller",
   141  				Clock:                  clk,
   142  				AsFieldManager:         ConfigConsumerAsFieldManager,
   143  				FoundToDangling:        func(found bool) bool { return !found },
   144  				InformerFactory:        informerFactory,
   145  				FlowcontrolClient:      flowcontrolClient,
   146  				ServerConcurrencyLimit: 24,
   147  				ReqsGaugeVec:           metrics.PriorityLevelConcurrencyGaugeVec,
   148  				ExecSeatsGaugeVec:      metrics.PriorityLevelExecutionSeatsGaugeVec,
   149  				QueueSetFactory:        fqs.NewQueueSetFactory(clk),
   150  			})
   151  
   152  			ctx, cancel := context.WithTimeout(context.Background(), 50*time.Second)
   153  			stopCh := ctx.Done()
   154  			controllerCompletionCh := make(chan error)
   155  
   156  			informerFactory.Start(stopCh)
   157  
   158  			status := informerFactory.WaitForCacheSync(ctx.Done())
   159  			if names := unsynced(status); len(names) > 0 {
   160  				t.Fatalf("WaitForCacheSync did not successfully complete, resources=%#v", names)
   161  			}
   162  
   163  			go func() {
   164  				controllerCompletionCh <- controller.Run(stopCh)
   165  			}()
   166  
   167  			// ensure that the controller has run its first loop.
   168  			err := wait.PollImmediate(100*time.Millisecond, 5*time.Second, func() (done bool, err error) {
   169  				return controller.hasPriorityLevelState(plcObjs[0].Name), nil
   170  			})
   171  			if err != nil {
   172  				t.Errorf("expected the controller to reconcile the priority level configuration object: %s, error: %s", plcObjs[0].Name, err)
   173  			}
   174  
   175  			concIntegrators := make([]fq.Integrator, 2)
   176  			reqInfo := &request.RequestInfo{
   177  				IsResourceRequest: false,
   178  				Path:              "/foobar",
   179  				Verb:              "GET",
   180  			}
   181  			noteFn := func(fs *flowcontrol.FlowSchema, plc *flowcontrol.PriorityLevelConfiguration, fd string) {}
   182  			workEstr := func() fcrequest.WorkEstimate { return fcrequest.WorkEstimate{InitialSeats: 1} }
   183  			qnf := fq.QueueNoteFn(func(bool) {})
   184  			var startWG sync.WaitGroup
   185  			startWG.Add(clientsPerFlow[0] + clientsPerFlow[1])
   186  			// Launch 20 client threads for each flow
   187  			for flow := 0; flow < 2; flow++ {
   188  				username := usernames[flow]
   189  				flowUser := testUser{name: username}
   190  				rd := RequestDigest{
   191  					RequestInfo: reqInfo,
   192  					User:        flowUser,
   193  				}
   194  				concIntegrator := fq.NewNamedIntegrator(clk, username)
   195  				concIntegrators[flow] = concIntegrator
   196  				exec := func() {
   197  					concIntegrator.Inc()
   198  					clk.Sleep(250 * time.Millisecond)
   199  					concIntegrator.Dec()
   200  				}
   201  				nThreads := clientsPerFlow[flow]
   202  				for thread := 0; thread < nThreads; thread++ {
   203  					go func() {
   204  						startWG.Done()
   205  						wait.Until(func() { controller.Handle(ctx, rd, noteFn, workEstr, qnf, exec) }, 0, ctx.Done())
   206  					}()
   207  				}
   208  			}
   209  			startWG.Wait()
   210  			// Make sure the controller has had time to sense the load and adjust
   211  			clk.Sleep(10 * time.Second)
   212  			// Start the stats that matter from now
   213  			for _, ci := range concIntegrators {
   214  				ci.Reset()
   215  			}
   216  			// Run for 15 seconds
   217  			clk.Sleep(15 * time.Second)
   218  			// Collect the delivered concurrency stats
   219  			results0 := concIntegrators[0].Reset()
   220  			results1 := concIntegrators[1].Reset()
   221  			// shut down all the async stuff
   222  			cancel()
   223  
   224  			// Do the checking
   225  
   226  			t.Log("waiting for the controller Run function to shutdown gracefully")
   227  			controllerErr := <-controllerCompletionCh
   228  			close(controllerCompletionCh)
   229  			if controllerErr != nil {
   230  				t.Errorf("expected nil error from controller Run function, but got: %#v", controllerErr)
   231  			}
   232  			if results0.Average < 15.5 || results0.Average > 16.1 {
   233  				t.Errorf("Flow 0 got average concurrency of %v but expected about 16", results0.Average)
   234  			} else {
   235  				t.Logf("Flow 0 got average concurrency of %v and expected about 16", results0.Average)
   236  			}
   237  			if results1.Average < 5.5 || results1.Average > 6.1 {
   238  				t.Errorf("Flow 1 got average concurrency of %v but expected about 6", results1.Average)
   239  			} else {
   240  				t.Logf("Flow 1 got average concurrency of %v and expected about 6", results1.Average)
   241  			}
   242  		})
   243  	}
   244  }
   245  
   246  type testUser struct{ name string }
   247  
   248  func (tu testUser) GetName() string               { return tu.name }
   249  func (tu testUser) GetUID() string                { return tu.name }
   250  func (tu testUser) GetGroups() []string           { return []string{user.AllAuthenticated} }
   251  func (tu testUser) GetExtra() map[string][]string { return map[string][]string{} }