k8s.io/kubernetes@v1.29.3/test/integration/scheduler_perf/dra_test.go (about) 1 /* 2 Copyright 2019 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package benchmark 18 19 import ( 20 "context" 21 "fmt" 22 "path/filepath" 23 "sync" 24 "testing" 25 26 resourcev1alpha2 "k8s.io/api/resource/v1alpha2" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 clientset "k8s.io/client-go/kubernetes" 29 "k8s.io/client-go/util/workqueue" 30 "k8s.io/klog/v2" 31 draapp "k8s.io/kubernetes/test/e2e/dra/test-driver/app" 32 ) 33 34 // createResourceClaimsOp defines an op where resource claims are created. 35 type createResourceClaimsOp struct { 36 // Must be createResourceClaimsOpcode. 37 Opcode operationCode 38 // Number of claims to create. Parameterizable through CountParam. 39 Count int 40 // Template parameter for Count. 41 CountParam string 42 // Namespace the claims should be created in. 43 Namespace string 44 // Path to spec file describing the claims to create. 45 TemplatePath string 46 } 47 48 var _ realOp = &createResourceClaimsOp{} 49 var _ runnableOp = &createResourceClaimsOp{} 50 51 func (op *createResourceClaimsOp) isValid(allowParameterization bool) error { 52 if op.Opcode != createResourceClaimsOpcode { 53 return fmt.Errorf("invalid opcode %q; expected %q", op.Opcode, createResourceClaimsOpcode) 54 } 55 if !isValidCount(allowParameterization, op.Count, op.CountParam) { 56 return fmt.Errorf("invalid Count=%d / CountParam=%q", op.Count, op.CountParam) 57 } 58 if op.Namespace == "" { 59 return fmt.Errorf("Namespace must be set") 60 } 61 if op.TemplatePath == "" { 62 return fmt.Errorf("TemplatePath must be set") 63 } 64 return nil 65 } 66 67 func (op *createResourceClaimsOp) collectsMetrics() bool { 68 return false 69 } 70 func (op *createResourceClaimsOp) patchParams(w *workload) (realOp, error) { 71 if op.CountParam != "" { 72 var err error 73 op.Count, err = w.Params.get(op.CountParam[1:]) 74 if err != nil { 75 return nil, err 76 } 77 } 78 return op, op.isValid(false) 79 } 80 81 func (op *createResourceClaimsOp) requiredNamespaces() []string { 82 return []string{op.Namespace} 83 } 84 85 func (op *createResourceClaimsOp) run(ctx context.Context, tb testing.TB, clientset clientset.Interface) { 86 tb.Logf("creating %d claims in namespace %q", op.Count, op.Namespace) 87 88 var claimTemplate *resourcev1alpha2.ResourceClaim 89 if err := getSpecFromFile(&op.TemplatePath, &claimTemplate); err != nil { 90 tb.Fatalf("parsing ResourceClaim %q: %v", op.TemplatePath, err) 91 } 92 var createErr error 93 var mutex sync.Mutex 94 create := func(i int) { 95 err := func() error { 96 if _, err := clientset.ResourceV1alpha2().ResourceClaims(op.Namespace).Create(ctx, claimTemplate.DeepCopy(), metav1.CreateOptions{}); err != nil { 97 return fmt.Errorf("create claim: %v", err) 98 } 99 return nil 100 }() 101 if err != nil { 102 mutex.Lock() 103 defer mutex.Unlock() 104 createErr = err 105 } 106 } 107 108 workers := op.Count 109 if workers > 30 { 110 workers = 30 111 } 112 workqueue.ParallelizeUntil(ctx, workers, op.Count, create) 113 if createErr != nil { 114 tb.Fatal(createErr.Error()) 115 } 116 } 117 118 // createResourceClassOpType customizes createOp for creating a ResourceClass. 119 type createResourceClassOpType struct{} 120 121 func (c createResourceClassOpType) Opcode() operationCode { return createResourceClassOpcode } 122 func (c createResourceClassOpType) Name() string { return "ResourceClass" } 123 func (c createResourceClassOpType) Namespaced() bool { return false } 124 func (c createResourceClassOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClass, metav1.CreateOptions) (*resourcev1alpha2.ResourceClass, error) { 125 return client.ResourceV1alpha2().ResourceClasses().Create 126 } 127 128 // createResourceClassOpType customizes createOp for creating a ResourceClaim. 129 type createResourceClaimTemplateOpType struct{} 130 131 func (c createResourceClaimTemplateOpType) Opcode() operationCode { 132 return createResourceClaimTemplateOpcode 133 } 134 func (c createResourceClaimTemplateOpType) Name() string { return "ResourceClaimTemplate" } 135 func (c createResourceClaimTemplateOpType) Namespaced() bool { return true } 136 func (c createResourceClaimTemplateOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClaimTemplate, metav1.CreateOptions) (*resourcev1alpha2.ResourceClaimTemplate, error) { 137 return client.ResourceV1alpha2().ResourceClaimTemplates(namespace).Create 138 } 139 140 // createResourceDriverOp defines an op where resource claims are created. 141 type createResourceDriverOp struct { 142 // Must be createResourceDriverOpcode. 143 Opcode operationCode 144 // Name of the driver, used to reference it in a resource class. 145 DriverName string 146 // Number of claims to allow per node. Parameterizable through MaxClaimsPerNodeParam. 147 MaxClaimsPerNode int 148 // Template parameter for MaxClaimsPerNode. 149 MaxClaimsPerNodeParam string 150 // Nodes matching this glob pattern have resources managed by the driver. 151 Nodes string 152 } 153 154 var _ realOp = &createResourceDriverOp{} 155 var _ runnableOp = &createResourceDriverOp{} 156 157 func (op *createResourceDriverOp) isValid(allowParameterization bool) error { 158 if op.Opcode != createResourceDriverOpcode { 159 return fmt.Errorf("invalid opcode %q; expected %q", op.Opcode, createResourceDriverOpcode) 160 } 161 if !isValidCount(allowParameterization, op.MaxClaimsPerNode, op.MaxClaimsPerNodeParam) { 162 return fmt.Errorf("invalid MaxClaimsPerNode=%d / MaxClaimsPerNodeParam=%q", op.MaxClaimsPerNode, op.MaxClaimsPerNodeParam) 163 } 164 if op.DriverName == "" { 165 return fmt.Errorf("DriverName must be set") 166 } 167 if op.Nodes == "" { 168 return fmt.Errorf("Nodes must be set") 169 } 170 return nil 171 } 172 173 func (op *createResourceDriverOp) collectsMetrics() bool { 174 return false 175 } 176 func (op *createResourceDriverOp) patchParams(w *workload) (realOp, error) { 177 if op.MaxClaimsPerNodeParam != "" { 178 var err error 179 op.MaxClaimsPerNode, err = w.Params.get(op.MaxClaimsPerNodeParam[1:]) 180 if err != nil { 181 return nil, err 182 } 183 } 184 return op, op.isValid(false) 185 } 186 187 func (op *createResourceDriverOp) requiredNamespaces() []string { return nil } 188 189 func (op *createResourceDriverOp) run(ctx context.Context, tb testing.TB, clientset clientset.Interface) { 190 tb.Logf("creating resource driver %q for nodes matching %q", op.DriverName, op.Nodes) 191 192 // Start the controller side of the DRA test driver such that it simulates 193 // per-node resources. 194 resources := draapp.Resources{ 195 DriverName: op.DriverName, 196 NodeLocal: true, 197 MaxAllocations: op.MaxClaimsPerNode, 198 } 199 200 nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) 201 if err != nil { 202 tb.Fatalf("list nodes: %v", err) 203 } 204 for _, node := range nodes.Items { 205 match, err := filepath.Match(op.Nodes, node.Name) 206 if err != nil { 207 tb.Fatalf("matching glob pattern %q against node name %q: %v", op.Nodes, node.Name, err) 208 } 209 if match { 210 resources.Nodes = append(resources.Nodes, node.Name) 211 } 212 } 213 214 controller := draapp.NewController(clientset, resources) 215 ctx, cancel := context.WithCancel(ctx) 216 var wg sync.WaitGroup 217 wg.Add(1) 218 go func() { 219 defer wg.Done() 220 ctx := klog.NewContext(ctx, klog.LoggerWithName(klog.FromContext(ctx), op.DriverName)) 221 controller.Run(ctx, 5 /* workers */) 222 }() 223 tb.Cleanup(func() { 224 tb.Logf("stopping resource driver %q", op.DriverName) 225 // We must cancel before waiting. 226 cancel() 227 wg.Wait() 228 tb.Logf("stopped resource driver %q", op.DriverName) 229 }) 230 }