k8s.io/test-infra@v0.0.0-20240520184403-27c6b4c223d8/releng/generate_tests.py (about) 1 #!/usr/bin/env python3 2 3 # Copyright 2017 The Kubernetes Authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Create e2e test definitions. 18 19 Usage example: 20 21 In $GOPATH/src/k8s.io/test-infra, 22 23 $ make -C releng generate-tests \ 24 --yaml-config-path=releng/test_config.yaml \ 25 """ 26 27 import argparse 28 import hashlib 29 import os 30 import ruamel.yaml 31 32 yaml = ruamel.yaml.YAML(typ='rt') 33 yaml.width = float("inf") 34 35 PROW_CONFIG_TEMPLATE = """ 36 tags: 37 - generated # AUTO-GENERATED by releng/generate_tests.py - DO NOT EDIT! 38 interval: 39 cron: 40 labels: 41 preset-service-account: "true" 42 preset-k8s-ssh: "true" 43 decorate: true 44 decoration_config: 45 timeout: 180m 46 name: 47 spec: 48 containers: 49 - command: 50 args: 51 env: 52 image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240515-17c6d50e24-master 53 resources: 54 requests: 55 cpu: 1000m 56 memory: 3Gi 57 limits: 58 cpu: 1000m 59 memory: 3Gi 60 """ 61 62 63 E2E_TESTGRID_CONFIG_TEMPLATE = """ 64 name: 65 gcs_prefix: 66 column_header: 67 - configuration_value: node_os_image 68 - configuration_value: master_os_image 69 - configuration_value: Commit 70 - configuration_value: infra-commit 71 """ 72 73 GCS_LOG_PREFIX = "kubernetes-jenkins/logs/" 74 75 COMMENT = 'AUTO-GENERATED by releng/generate_tests.py - DO NOT EDIT.' 76 77 def get_sha1_hash(data): 78 """Returns the SHA1 hash of the specified data.""" 79 sha1_hash = hashlib.sha1() 80 sha1_hash.update(data.encode('utf-8')) 81 return sha1_hash.hexdigest() 82 83 84 def substitute(job_name, lines): 85 """Replace '${job_name_hash}' in lines with the SHA1 hash of job_name.""" 86 return [line.replace('${job_name_hash}', get_sha1_hash(job_name)[:10]) \ 87 for line in lines] 88 89 def get_args(job_name, field): 90 """Returns a list of args for the given field.""" 91 if not field: 92 return [] 93 return substitute(job_name, field.get('args', [])) 94 95 96 def write_prow_configs_file(output_file, job_defs): 97 """Writes the Prow configurations into output_file.""" 98 print(f'writing prow configuration to: {output_file}') 99 with open(output_file, 'w') as fp: 100 yaml.dump(job_defs, fp) 101 102 def write_testgrid_config_file(output_file, testgrid_config): 103 """Writes the TestGrid test group configurations into output_file.""" 104 print(f'writing testgrid configuration to: {output_file}') 105 with open(output_file, 'w') as fp: 106 fp.write('# ' + COMMENT + '\n\n') 107 yaml.dump(testgrid_config, fp) 108 109 def apply_job_overrides(envs_or_args, job_envs_or_args): 110 '''Applies the envs or args overrides defined in the job level''' 111 original_envs_or_args = envs_or_args[:] 112 for job_env_or_arg in job_envs_or_args: 113 name = job_env_or_arg.split('=', 1)[0] 114 env_or_arg = next( 115 (x for x in original_envs_or_args if (x.strip().startswith('%s=' % name) or 116 x.strip() == name)), None) 117 if env_or_arg: 118 envs_or_args.remove(env_or_arg) 119 envs_or_args.append(job_env_or_arg) 120 121 122 class E2ENodeTest: 123 124 def __init__(self, job_name, job, config): 125 self.job_name = job_name 126 self.job = job 127 self.common = config['nodeCommon'] 128 self.images = config['nodeImages'] 129 self.k8s_versions = config['nodeK8sVersions'] 130 self.test_suites = config['nodeTestSuites'] 131 132 def __get_job_def(self, args): 133 """Returns the job definition from the given args.""" 134 return { 135 'scenario': 'kubernetes_e2e', 136 'args': args, 137 'sigOwners': self.job.get('sigOwners') or ['UNNOWN'], 138 # Indicates that this job definition is auto-generated. 139 'tags': ['generated'], 140 '_comment': COMMENT, 141 } 142 143 def __get_prow_config(self, test_suite, k8s_version): 144 """Returns the Prow config for the job from the given fields.""" 145 prow_config = yaml.load(PROW_CONFIG_TEMPLATE) 146 prow_config['name'] = self.job_name 147 # use cluster from test_suite, or job, or not at all 148 if 'cluster' in test_suite: 149 prow_config['cluster'] = test_suite['cluster'] 150 elif 'cluster' in self.job: 151 prow_config['cluster'] = self.job['cluster'] 152 # use resources from test_suite, or job, or default 153 if 'resources' in test_suite: 154 prow_config['spec']['containers'][0]['resources'] = test_suite['resources'] 155 elif 'resources' in self.job: 156 prow_config['spec']['containers'][0]['resources'] = self.job['resources'] 157 # pull interval or cron from job 158 if 'interval' in self.job: 159 del prow_config['cron'] 160 prow_config['interval'] = self.job['interval'] 161 elif 'cron' in self.job: 162 del prow_config['cron'] 163 prow_config['cron'] = self.job['cron'] 164 else: 165 raise Exception("no interval or cron definition found") 166 # Assumes that the value in --timeout is of minutes. 167 timeout = int(next( 168 x[10:-1] for x in test_suite['args'] if ( 169 x.startswith('--timeout=')))) 170 container = prow_config['spec']['containers'][0] 171 if not container['args']: 172 container['args'] = [] 173 if not container['env']: 174 container['env'] = [] 175 # Prow timeout = job timeout + 20min 176 prow_config['decoration_config']['timeout'] = '{}m'.format(timeout + 20) 177 container['args'].extend(k8s_version.get('args', [])) 178 container['args'].append('--root=/go/src') 179 container['env'].extend([{'name':'GOPATH', 'value': '/go'}]) 180 # Specify the appropriate kubekins-e2e image. This allows us to use a 181 # specific image (containing a particular Go version) to build and 182 # trigger the node e2e test to avoid issues like 183 # https://github.com/kubernetes/kubernetes/issues/43534. 184 if k8s_version.get('prowImage', None): 185 container['image'] = k8s_version['prowImage'] 186 return prow_config 187 188 def generate(self): 189 '''Returns the job and the Prow configurations for this test.''' 190 print(f'generating e2enode job: {self.job_name}') 191 fields = self.job_name.split('-') 192 if len(fields) != 6: 193 raise ValueError('Expected 6 fields in job name', self.job_name) 194 195 image = self.images[fields[3]] 196 k8s_version = self.k8s_versions[fields[4][3:]] 197 test_suite = self.test_suites[fields[5]] 198 199 # envs are disallowed in node e2e tests. 200 if 'envs' in self.common or 'envs' in image or 'envs' in test_suite: 201 raise ValueError( 202 'envs are disallowed in node e2e test', self.job_name) 203 # Generates args. 204 args = [] 205 args.extend(get_args(self.job_name, self.common)) 206 args.extend(get_args(self.job_name, image)) 207 args.extend(get_args(self.job_name, test_suite)) 208 # Generates job config. 209 job_config = self.__get_job_def(args) 210 # Generates prow config. 211 prow_config = self.__get_prow_config(test_suite, k8s_version) 212 213 # Combine --node-args 214 node_args = [] 215 job_args = [] 216 for arg in job_config['args']: 217 if '--node-args=' in arg: 218 node_args.append(arg.split('=', 1)[1]) 219 else: 220 job_args.append(arg) 221 222 if node_args: 223 flag = '--node-args=' 224 for node_arg in node_args: 225 flag += '%s ' % node_arg 226 job_args.append(flag.strip()) 227 228 job_config['args'] = job_args 229 230 if image.get('testgrid_prefix') is not None: 231 dashboard = '%s-%s-%s' % (image['testgrid_prefix'], fields[3], 232 fields[4]) 233 annotations = prow_config.setdefault('annotations', {}) 234 annotations['testgrid-dashboards'] = dashboard 235 tab_name = '%s-%s-%s' % (fields[3], fields[4], fields[5]) 236 annotations['testgrid-tab-name'] = tab_name 237 238 return job_config, prow_config, None 239 240 241 class E2ETest: 242 243 def __init__(self, output_dir, job_name, job, config): 244 self.env_filename = os.path.join(output_dir, '%s.env' % job_name) 245 self.job_name = job_name 246 self.job = job 247 self.common = config['common'] 248 self.cloud_providers = config['cloudProviders'] 249 self.images = config['images'] 250 self.k8s_versions = config['k8sVersions'] 251 self.test_suites = config['testSuites'] 252 253 def __get_job_def(self, args): 254 """Returns the job definition from the given args.""" 255 return { 256 'scenario': 'kubernetes_e2e', 257 'args': args, 258 'sigOwners': self.job.get('sigOwners') or ['UNNOWN'], 259 # Indicates that this job definition is auto-generated. 260 'tags': ['generated'], 261 '_comment': COMMENT, 262 } 263 264 def __get_prow_config(self, test_suite): 265 """Returns the Prow config for the e2e job from the given fields.""" 266 prow_config = yaml.load(PROW_CONFIG_TEMPLATE) 267 prow_config['name'] = self.job_name 268 # use cluster from test_suite, or job, or not at all 269 if 'cluster' in test_suite: 270 prow_config['cluster'] = test_suite['cluster'] 271 elif 'cluster' in self.job: 272 prow_config['cluster'] = self.job['cluster'] 273 # use resources from test_suite, or job, or default 274 if 'resources' in test_suite: 275 prow_config['spec']['containers'][0]['resources'] = test_suite['resources'] 276 elif 'resources' in self.job: 277 prow_config['spec']['containers'][0]['resources'] = self.job['resources'] 278 if 'interval' in self.job: 279 del prow_config['cron'] 280 prow_config['interval'] = self.job['interval'] 281 elif 'cron' in self.job: 282 del prow_config['interval'] 283 prow_config['cron'] = self.job['cron'] 284 else: 285 raise Exception("no interval or cron definition found") 286 # Assumes that the value in --timeout is of minutes. 287 timeout = int(next( 288 x[10:-1] for x in test_suite['args'] if ( 289 x.startswith('--timeout=')))) 290 container = prow_config['spec']['containers'][0] 291 if not container['args']: 292 container['args'] = [] 293 # Prow timeout = job timeout + 20min 294 prow_config['decoration_config']['timeout'] = '{}m'.format(timeout + 20) 295 return prow_config 296 297 def __get_testgrid_config(self): 298 tg_config = yaml.load(E2E_TESTGRID_CONFIG_TEMPLATE) 299 tg_config['name'] = self.job_name 300 tg_config['gcs_prefix'] = GCS_LOG_PREFIX + self.job_name 301 return tg_config 302 303 def initialize_dashboards_with_release_blocking_info(self, version): 304 dashboards = [] 305 if self.job.get('releaseBlocking'): 306 dashboards.append('sig-release-%s-blocking' % version) 307 elif self.job.get('releaseInforming'): 308 dashboards.append('sig-release-%s-informing' % version) 309 else: 310 dashboards.append('sig-release-generated') 311 return dashboards 312 313 def generate(self): 314 '''Returns the job and the Prow configurations for this test.''' 315 print(f'generating e2e job: {self.job_name}') 316 fields = self.job_name.split('-') 317 if len(fields) != 7: 318 raise ValueError('Expected 7 fields in job name', self.job_name) 319 320 cloud_provider = self.cloud_providers[fields[3]] 321 image = self.images[fields[4]] 322 k8s_version = self.k8s_versions[fields[5][3:]] 323 test_suite = self.test_suites[fields[6]] 324 325 # Generates args. 326 args = [] 327 args.extend(get_args(self.job_name, self.common)) 328 args.extend(get_args(self.job_name, cloud_provider)) 329 args.extend(get_args(self.job_name, image)) 330 args.extend(get_args(self.job_name, k8s_version)) 331 args.extend(get_args(self.job_name, test_suite)) 332 # Generates job config. 333 job_config = self.__get_job_def(args) 334 # Generates Prow config. 335 prow_config = self.__get_prow_config(test_suite) 336 337 tg_config = self.__get_testgrid_config() 338 339 annotations = prow_config.setdefault('annotations', {}) 340 tab_name = '%s-%s-%s-%s' % (fields[3], fields[4], fields[5], fields[6]) 341 annotations['testgrid-tab-name'] = tab_name 342 dashboards = self.initialize_dashboards_with_release_blocking_info(k8s_version['version']) 343 if image.get('testgrid_prefix') is not None: 344 dashboard = '%s-%s-%s' % (image['testgrid_prefix'], fields[4], 345 fields[5]) 346 dashboards.append(dashboard) 347 annotations['testgrid-dashboards'] = ', '.join(dashboards) 348 if 'testgridNumFailuresToAlert' in self.job: 349 annotations['testgrid-num-failures-to-alert'] = ('%s' % 350 self.job['testgridNumFailuresToAlert']) 351 352 return job_config, prow_config, tg_config 353 354 355 def for_each_job(output_dir, job_name, job, yaml_config): 356 """Returns the job config and the Prow config for one test job.""" 357 fields = job_name.split('-') 358 if len(fields) < 3: 359 raise ValueError('Expected at least 3 fields in job name', job_name) 360 job_type = fields[2] 361 362 # Generates configurations. 363 if job_type == 'e2e': 364 generator = E2ETest(output_dir, job_name, job, yaml_config) 365 elif job_type == 'e2enode': 366 generator = E2ENodeTest(job_name, job, yaml_config) 367 else: 368 raise ValueError(f'Job {job_name} has unexpected job type ', job_type) 369 job_config, prow_config, testgrid_config = generator.generate() 370 371 # Applies job-level overrides. 372 apply_job_overrides(job_config['args'], get_args(job_name, job)) 373 374 # merge job_config into prow_config 375 args = prow_config['spec']['containers'][0]['args'] 376 args.extend(job_config['args']) 377 prow_config['spec']['containers'][0]['command'] = \ 378 ['runner.sh', '/workspace/scenarios/{}.py'.format(job_config['scenario'])] 379 380 return prow_config, testgrid_config 381 382 383 def main(yaml_config_path, output_dir, testgrid_output_path): 384 """Creates test job definitions. 385 386 Converts the test configurations in yaml_config_path to the job definitions 387 in output_dir/generated.yaml. 388 """ 389 # TODO(yguo0905): Validate the configurations from yaml_config_path. 390 391 with open(yaml_config_path) as fp: 392 yaml_config = yaml.load(fp) 393 394 output_config = {} 395 output_config['periodics'] = [] 396 testgrid_config = {'test_groups': []} 397 job_names = sorted(yaml_config['jobs'].keys()) 398 for job_name in job_names: 399 # Get the envs and args for each job defined under "jobs". 400 prow, testgrid = for_each_job( 401 output_dir, job_name, yaml_config['jobs'][job_name], yaml_config) 402 output_config['periodics'].append(prow) 403 if testgrid is not None: 404 testgrid_config['test_groups'].append(testgrid) 405 406 # Write the job definitions to --output-dir/generated.yaml 407 write_prow_configs_file(output_dir + 'generated.yaml', output_config) 408 write_testgrid_config_file(testgrid_output_path, testgrid_config) 409 410 411 if __name__ == '__main__': 412 PARSER = argparse.ArgumentParser( 413 description='Create test definitions from the given yaml config') 414 PARSER.add_argument('--yaml-config-path', help='Path to config.yaml') 415 PARSER.add_argument( 416 '--output-dir', 417 help='Prowjob config output dir', 418 default='config/jobs/kubernetes/generated/') 419 PARSER.add_argument( 420 '--testgrid-output-path', 421 help='Path to testgrid output file', 422 default='config/testgrids/generated-test-config.yaml') 423 ARGS = PARSER.parse_args() 424 425 main( 426 ARGS.yaml_config_path, 427 ARGS.output_dir, 428 ARGS.testgrid_output_path)