github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/jobs/config_test.py (about) 1 #!/usr/bin/env python 2 3 # Copyright 2017 The Kubernetes Authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Tests for config.json and Prow configuration.""" 18 19 20 import unittest 21 22 import collections 23 import json 24 import os 25 import re 26 import sys 27 28 import config_sort 29 import yaml 30 31 # pylint: disable=too-many-public-methods, too-many-branches, too-many-locals, too-many-statements 32 33 def get_required_jobs(): 34 required_jobs = set() 35 configs_dir = config_sort.test_infra('mungegithub', 'submit-queue', 'deployment') 36 for root, _, files in os.walk(configs_dir): 37 for file_name in files: 38 if file_name == 'configmap.yaml': 39 path = os.path.join(root, file_name) 40 with open(path) as fp: 41 conf = yaml.safe_load(fp) 42 for job in conf.get('required-retest-contexts', '').split(','): 43 if job: 44 required_jobs.add(job) 45 return required_jobs 46 47 class JobTest(unittest.TestCase): 48 49 excludes = [ 50 'BUILD.bazel', # For bazel 51 'config.json', # For --json mode 52 'validOwners.json', # Contains a list of current sigs; sigs are allowed to own jobs 53 'config_sort.py', # Tool script to sort config.json 54 'config_test.py', # Script for testing config.json and Prow config. 55 'env_gc.py', # Tool script to garbage collect unused .env files. 56 'move_extract.py', 57 ] 58 # also exclude .pyc 59 excludes.extend(e + 'c' for e in excludes if e.endswith('.py')) 60 61 prow_config = '../prow/config.yaml' 62 63 realjobs = {} 64 prowjobs = [] 65 presubmits = [] 66 67 @property 68 def jobs(self): 69 """[(job, job_path)] sequence""" 70 for path, _, filenames in os.walk(config_sort.test_infra('jobs')): 71 print >>sys.stderr, path 72 if 'e2e_node' in path: 73 # Node e2e image configs, ignore them 74 continue 75 for job in [f for f in filenames if f not in self.excludes]: 76 job_path = os.path.join(path, job) 77 yield job, job_path 78 79 def test_config_is_sorted(self): 80 """Test jobs/config.json, prow/config.yaml and boskos/resources.yaml are sorted.""" 81 with open(config_sort.test_infra('jobs/config.json')) as fp: 82 original = fp.read() 83 expect = config_sort.sorted_job_config().getvalue() 84 if original != expect: 85 self.fail('jobs/config.json is not sorted, please run ' 86 '`bazel run //jobs:config_sort`') 87 with open(config_sort.test_infra('prow/config.yaml')) as fp: 88 original = fp.read() 89 expect = config_sort.sorted_prow_config( 90 config_sort.test_infra('prow/config.yaml')).getvalue() 91 if original != expect: 92 self.fail('prow/config.yaml is not sorted, please run ' 93 '`bazel run //jobs:config_sort`') 94 with open(config_sort.test_infra('boskos/resources.yaml')) as fp: 95 original = fp.read() 96 expect = config_sort.sorted_boskos_config().getvalue() 97 if original != expect: 98 self.fail('boskos/resources.yaml is not sorted, please run ' 99 '`bazel run //jobs:config_sort`') 100 101 # TODO(krzyzacy): disabled as we currently have multiple source of truth. 102 # We also should migrate shared env files into presets. 103 #def test_orphaned_env(self): 104 # orphans = env_gc.find_orphans() 105 # if orphans: 106 # self.fail('the following .env files are not referenced ' + 107 # 'in config.json, please run `bazel run //jobs:env_gc`: ' + 108 # ' '.join(orphans)) 109 110 def check_job_template(self, tmpl): 111 builders = tmpl.get('builders') 112 if not isinstance(builders, list): 113 self.fail(tmpl) 114 self.assertEquals(1, len(builders), builders) 115 shell = builders[0] 116 if not isinstance(shell, dict): 117 self.fail(tmpl) 118 self.assertEquals(1, len(shell), tmpl) 119 if 'raw' in shell: 120 self.assertEquals('maintenance-all-{suffix}', tmpl['name']) 121 return 122 cmd = shell.get('shell') 123 if not isinstance(cmd, basestring): 124 self.fail(tmpl) 125 self.assertIn('--service-account=', cmd) 126 self.assertIn('--upload=', cmd) 127 if 'kubernetes-security' in cmd: 128 self.assertIn('--upload=\'gs://kubernetes-security-jenkins/pr-logs\'', cmd) 129 elif '${{PULL_REFS}}' in cmd: 130 self.assertIn('--upload=\'gs://kubernetes-jenkins/pr-logs\'', cmd) 131 else: 132 self.assertIn('--upload=\'gs://kubernetes-jenkins/logs\'', cmd) 133 134 def add_prow_job(self, job): 135 name = job.get('name') 136 real_job = {} 137 real_job['name'] = name 138 if 'spec' in job: 139 spec = job.get('spec') 140 for container in spec.get('containers'): 141 if 'args' in container: 142 for arg in container.get('args'): 143 match = re.match(r'[\'\"]?--timeout=(\d+)', arg) 144 if match: 145 real_job['timeout'] = match.group(1) 146 if 'pull-' not in name and name in self.realjobs and name not in self.prowjobs: 147 self.fail('CI job %s exist in both Jenkins and Prow config!' % name) 148 if name not in self.realjobs: 149 self.realjobs[name] = real_job 150 self.prowjobs.append(name) 151 if 'run_after_success' in job: 152 for sub in job.get('run_after_success'): 153 self.add_prow_job(sub) 154 155 def load_prow_yaml(self, path): 156 with open(os.path.join( 157 os.path.dirname(__file__), path)) as fp: 158 doc = yaml.safe_load(fp) 159 160 if 'periodics' not in doc: 161 self.fail('No periodics in prow config!') 162 163 if 'presubmits' not in doc: 164 self.fail('No presubmits in prow config!') 165 166 for item in doc.get('periodics'): 167 self.add_prow_job(item) 168 169 if 'postsubmits' not in doc: 170 self.fail('No postsubmits in prow config!') 171 172 self.presubmits = doc.get('presubmits') 173 postsubmits = doc.get('postsubmits') 174 175 for _repo, joblist in self.presubmits.items() + postsubmits.items(): 176 for job in joblist: 177 self.add_prow_job(job) 178 179 def get_real_bootstrap_job(self, job): 180 key = os.path.splitext(job.strip())[0] 181 if not key in self.realjobs: 182 self.load_prow_yaml(self.prow_config) 183 self.assertIn(key, sorted(self.realjobs)) # sorted for clearer error message 184 return self.realjobs.get(key) 185 186 def test_valid_timeout(self): 187 """All e2e jobs has 20min or more container timeout than kubetest timeout.""" 188 bad_jobs = set() 189 with open(config_sort.test_infra('jobs/config.json')) as fp: 190 config = json.loads(fp.read()) 191 192 for job in config: 193 if config.get(job, {}).get('scenario') != 'kubernetes_e2e': 194 continue 195 realjob = self.get_real_bootstrap_job(job) 196 self.assertTrue(realjob) 197 self.assertIn('timeout', realjob, job) 198 container_timeout = int(realjob['timeout']) 199 200 kubetest_timeout = None 201 for arg in config[job]['args']: 202 mat = re.match(r'--timeout=(\d+)m', arg) 203 if not mat: 204 continue 205 kubetest_timeout = int(mat.group(1)) 206 if kubetest_timeout is None: 207 self.fail('Missing timeout: %s' % job) 208 if kubetest_timeout > container_timeout: 209 bad_jobs.add((job, kubetest_timeout, container_timeout)) 210 elif kubetest_timeout + 20 > container_timeout: 211 bad_jobs.add(( 212 'insufficient kubetest leeway', 213 job, kubetest_timeout, container_timeout 214 )) 215 if bad_jobs: 216 self.fail( 217 'jobs: %s, ' 218 'prow timeout need to be at least 20min longer than timeout in config.json' 219 % ('\n'.join(str(s) for s in bad_jobs)) 220 ) 221 222 def test_valid_job_config_json(self): 223 """Validate jobs/config.json.""" 224 # bootstrap integration test scripts 225 ignore = [ 226 'fake-failure', 227 'fake-branch', 228 'fake-pr', 229 'random_job', 230 ] 231 232 self.load_prow_yaml(self.prow_config) 233 config = config_sort.test_infra('jobs/config.json') 234 owners = config_sort.test_infra('jobs/validOwners.json') 235 with open(config) as fp, open(owners) as ownfp: 236 config = json.loads(fp.read()) 237 valid_owners = json.loads(ownfp.read()) 238 for job in config: 239 if job not in ignore: 240 self.assertTrue(job in self.prowjobs or job in self.realjobs, 241 '%s must have a matching jenkins/prow entry' % job) 242 243 # ownership assertions 244 self.assertIn('sigOwners', config[job], job) 245 self.assertIsInstance(config[job]['sigOwners'], list, job) 246 self.assertTrue(config[job]['sigOwners'], job) # non-empty 247 owners = config[job]['sigOwners'] 248 for owner in owners: 249 self.assertIsInstance(owner, basestring, job) 250 self.assertIn(owner, valid_owners, job) 251 252 # env assertions 253 self.assertTrue('scenario' in config[job], job) 254 scenario = config_sort.test_infra('scenarios/%s.py' % config[job]['scenario']) 255 self.assertTrue(os.path.isfile(scenario), job) 256 self.assertTrue(os.access(scenario, os.X_OK|os.R_OK), job) 257 args = config[job].get('args', []) 258 use_shared_build_in_args = False 259 extract_in_args = False 260 build_in_args = False 261 for arg in args: 262 if arg.startswith('--use-shared-build'): 263 use_shared_build_in_args = True 264 elif arg.startswith('--build'): 265 build_in_args = True 266 elif arg.startswith('--extract'): 267 extract_in_args = True 268 match = re.match(r'--env-file=([^\"]+)\.env', arg) 269 if match: 270 env_path = match.group(1) 271 self.assertTrue(env_path.startswith('jobs/'), env_path) 272 path = config_sort.test_infra('%s.env' % env_path) 273 self.assertTrue( 274 os.path.isfile(path), 275 '%s does not exist for %s' % (path, job)) 276 elif 'kops' not in job: 277 match = re.match(r'--cluster=([^\"]+)', arg) 278 if match: 279 cluster = match.group(1) 280 self.assertLessEqual( 281 len(cluster), 23, 282 'Job %r, --cluster should be 23 chars or fewer' % job 283 ) 284 # these args should not be combined: 285 # --use-shared-build and (--build or --extract) 286 self.assertFalse(use_shared_build_in_args and build_in_args) 287 self.assertFalse(use_shared_build_in_args and extract_in_args) 288 if config[job]['scenario'] == 'kubernetes_e2e': 289 if job in self.prowjobs: 290 for arg in args: 291 # --mode=local is default now 292 self.assertNotIn('--mode', arg, job) 293 else: 294 self.assertIn('--mode=docker', args, job) 295 for arg in args: 296 if "--env=" in arg: 297 self._check_env(job, arg.split("=", 1)[1]) 298 if '--provider=gke' in args: 299 self.assertTrue('--deployment=gke' in args, 300 '%s must use --deployment=gke' % job) 301 self.assertFalse(any('--gcp-master-image' in a for a in args), 302 '%s cannot use --gcp-master-image on GKE' % job) 303 self.assertFalse(any('--gcp-nodes' in a for a in args), 304 '%s cannot use --gcp-nodes on GKE' % job) 305 if '--deployment=gke' in args: 306 self.assertTrue(any('--gcp-node-image' in a for a in args), job) 307 self.assertNotIn('--charts-tests', args) # Use --charts 308 if any('--check_version_skew' in a for a in args): 309 self.fail('Use --check-version-skew, not --check_version_skew in %s' % job) 310 if '--check-leaked-resources=true' in args: 311 self.fail('Use --check-leaked-resources (no value) in %s' % job) 312 if '--check-leaked-resources==false' in args: 313 self.fail( 314 'Remove --check-leaked-resources=false (default value) from %s' % job) 315 if ( 316 '--env-file=jobs/pull-kubernetes-e2e.env' in args 317 and '--check-leaked-resources' in args): 318 self.fail('PR job %s should not check for resource leaks' % job) 319 # Consider deleting any job with --check-leaked-resources=false 320 if ( 321 '--provider=gce' not in args 322 and '--provider=gke' not in args 323 and '--check-leaked-resources' in args 324 and 'generated' not in config[job].get('tags', [])): 325 self.fail('Only GCP jobs can --check-leaked-resources, not %s' % job) 326 if '--mode=local' in args: 327 self.fail('--mode=local is default now, drop that for %s' % job) 328 329 extracts = [a for a in args if '--extract=' in a] 330 shared_builds = [a for a in args if '--use-shared-build' in a] 331 node_e2e = [a for a in args if '--deployment=node' in a] 332 local_e2e = [a for a in args if '--deployment=local' in a] 333 builds = [a for a in args if '--build' in a] 334 if shared_builds and extracts: 335 self.fail(('e2e jobs cannot have --use-shared-build' 336 ' and --extract: %s %s') % (job, args)) 337 elif not extracts and not shared_builds and not node_e2e: 338 # we should at least have --build and --stage 339 if not builds: 340 self.fail(('e2e job needs --extract or' 341 ' --use-shared-build or' 342 ' --build: %s %s') % (job, args)) 343 344 if shared_builds or node_e2e: 345 expected = 0 346 elif builds and not extracts: 347 expected = 0 348 elif 'ingress' in job: 349 expected = 1 350 elif any(s in job for s in [ 351 'upgrade', 'skew', 'downgrade', 'rollback', 352 'ci-kubernetes-e2e-gce-canary', 353 ]): 354 expected = 2 355 else: 356 expected = 1 357 if len(extracts) != expected: 358 self.fail('Wrong number of --extract args (%d != %d) in %s' % ( 359 len(extracts), expected, job)) 360 361 has_image_family = any( 362 [x for x in args if x.startswith('--image-family')]) 363 has_image_project = any( 364 [x for x in args if x.startswith('--image-project')]) 365 docker_mode = any( 366 [x for x in args if x.startswith('--mode=docker')]) 367 if ( 368 (has_image_family or has_image_project) 369 and docker_mode): 370 self.fail('--image-family / --image-project is not ' 371 'supported in docker mode: %s' % job) 372 if has_image_family != has_image_project: 373 self.fail('--image-family and --image-project must be' 374 'both set or unset: %s' % job) 375 376 if job.startswith('pull-kubernetes-') and not node_e2e and not local_e2e: 377 if 'gke' in job: 378 stage = 'gs://kubernetes-release-dev/ci' 379 suffix = True 380 elif 'kubeadm' in job: 381 # kubeadm-based jobs use out-of-band .deb artifacts, 382 # not the --stage flag. 383 continue 384 else: 385 stage = 'gs://kubernetes-release-pull/ci/%s' % job 386 suffix = False 387 if not shared_builds: 388 self.assertIn('--stage=%s' % stage, args) 389 self.assertEquals( 390 suffix, 391 any('--stage-suffix=' in a for a in args), 392 ('--stage-suffix=', suffix, job, args)) 393 394 395 def test_valid_env(self): 396 for job, job_path in self.jobs: 397 with open(job_path) as fp: 398 data = fp.read() 399 if 'kops' in job: # TODO(fejta): update this one too 400 continue 401 self.assertNotIn( 402 'JENKINS_USE_LOCAL_BINARIES=', 403 data, 404 'Send --extract=local to config.json, not JENKINS_USE_LOCAL_BINARIES in %s' % job) 405 self.assertNotIn( 406 'JENKINS_USE_EXISTING_BINARIES=', 407 data, 408 'Send --extract=local to config.json, not JENKINS_USE_EXISTING_BINARIES in %s' % job) # pylint: disable=line-too-long 409 410 def test_only_jobs(self): 411 """Ensure that everything in jobs/ is a valid job name and script.""" 412 for job, job_path in self.jobs: 413 # Jobs should have simple names: letters, numbers, -, . 414 self.assertTrue(re.match(r'[.0-9a-z-_]+.env', job), job) 415 # Jobs should point to a real, executable file 416 # Note: it is easy to forget to chmod +x 417 self.assertTrue(os.path.isfile(job_path), job_path) 418 self.assertFalse(os.path.islink(job_path), job_path) 419 self.assertTrue(os.access(job_path, os.R_OK), job_path) 420 421 def test_all_project_are_unique(self): 422 # pylint: disable=line-too-long 423 allowed_list = { 424 # The cos image validation jobs intentionally share projects. 425 'ci-kubernetes-e2e-gce-cosdev-k8sdev-default': 'ci-kubernetes-e2e-gce-cos*', 426 'ci-kubernetes-e2e-gce-cosdev-k8sdev-serial': 'ci-kubernetes-e2e-gce-cos*', 427 'ci-kubernetes-e2e-gce-cosdev-k8sdev-slow': 'ci-kubernetes-e2e-gce-cos*', 428 'ci-kubernetes-e2e-gce-cosdev-k8sstable1-default': 'ci-kubernetes-e2e-gce-cos*', 429 'ci-kubernetes-e2e-gce-cosdev-k8sstable1-serial': 'ci-kubernetes-e2e-gce-cos*', 430 'ci-kubernetes-e2e-gce-cosdev-k8sstable1-slow': 'ci-kubernetes-e2e-gce-cos*', 431 'ci-kubernetes-e2e-gce-cosdev-k8sbeta-default': 'ci-kubernetes-e2e-gce-cos*', 432 'ci-kubernetes-e2e-gce-cosdev-k8sbeta-serial': 'ci-kubernetes-e2e-gce-cos*', 433 'ci-kubernetes-e2e-gce-cosdev-k8sbeta-slow': 'ci-kubernetes-e2e-gce-cos*', 434 'ci-kubernetes-e2e-gce-cosbeta-k8sdev-default': 'ci-kubernetes-e2e-gce-cos*', 435 'ci-kubernetes-e2e-gce-cosbeta-k8sdev-serial': 'ci-kubernetes-e2e-gce-cos*', 436 'ci-kubernetes-e2e-gce-cosbeta-k8sdev-slow': 'ci-kubernetes-e2e-gce-cos*', 437 'ci-kubernetes-e2e-gce-cosbeta-k8sbeta-default': 'ci-kubernetes-e2e-gce-cos*', 438 'ci-kubernetes-e2e-gce-cosbeta-k8sbeta-serial': 'ci-kubernetes-e2e-gce-cos*', 439 'ci-kubernetes-e2e-gce-cosbeta-k8sbeta-slow': 'ci-kubernetes-e2e-gce-cos*', 440 'ci-kubernetes-e2e-gce-cosbeta-k8sstable1-default': 'ci-kubernetes-e2e-gce-cos*', 441 'ci-kubernetes-e2e-gce-cosbeta-k8sstable1-serial': 'ci-kubernetes-e2e-gce-cos*', 442 'ci-kubernetes-e2e-gce-cosbeta-k8sstable1-slow': 'ci-kubernetes-e2e-gce-cos*', 443 'ci-kubernetes-e2e-gce-cosbeta-k8sstable2-default': 'ci-kubernetes-e2e-gce-cos*', 444 'ci-kubernetes-e2e-gce-cosbeta-k8sstable2-serial': 'ci-kubernetes-e2e-gce-cos*', 445 'ci-kubernetes-e2e-gce-cosbeta-k8sstable2-slow': 'ci-kubernetes-e2e-gce-cos*', 446 'ci-kubernetes-e2e-gce-cosbeta-k8sstable3-default': 'ci-kubernetes-e2e-gce-cos*', 447 'ci-kubernetes-e2e-gce-cosbeta-k8sstable3-serial': 'ci-kubernetes-e2e-gce-cos*', 448 'ci-kubernetes-e2e-gce-cosbeta-k8sstable3-slow': 'ci-kubernetes-e2e-gce-cos*', 449 'ci-kubernetes-e2e-gce-cosstable1-k8sdev-default': 'ci-kubernetes-e2e-gce-cos*', 450 'ci-kubernetes-e2e-gce-cosstable1-k8sdev-serial': 'ci-kubernetes-e2e-gce-cos*', 451 'ci-kubernetes-e2e-gce-cosstable1-k8sdev-slow': 'ci-kubernetes-e2e-gce-cos*', 452 'ci-kubernetes-e2e-gce-cosstable1-k8sbeta-default': 'ci-kubernetes-e2e-gce-cos*', 453 'ci-kubernetes-e2e-gce-cosstable1-k8sbeta-serial': 'ci-kubernetes-e2e-gce-cos*', 454 'ci-kubernetes-e2e-gce-cosstable1-k8sbeta-slow': 'ci-kubernetes-e2e-gce-cos*', 455 'ci-kubernetes-e2e-gce-cosstable1-k8sstable1-default': 'ci-kubernetes-e2e-gce-cos*', 456 'ci-kubernetes-e2e-gce-cosstable1-k8sstable1-serial': 'ci-kubernetes-e2e-gce-cos*', 457 'ci-kubernetes-e2e-gce-cosstable1-k8sstable1-slow': 'ci-kubernetes-e2e-gce-cos*', 458 'ci-kubernetes-e2e-gce-cosstable1-k8sstable2-default': 'ci-kubernetes-e2e-gce-cos*', 459 'ci-kubernetes-e2e-gce-cosstable1-k8sstable2-serial': 'ci-kubernetes-e2e-gce-cos*', 460 'ci-kubernetes-e2e-gce-cosstable1-k8sstable2-slow': 'ci-kubernetes-e2e-gce-cos*', 461 'ci-kubernetes-e2e-gce-cosstable1-k8sstable3-default': 'ci-kubernetes-e2e-gce-cos*', 462 'ci-kubernetes-e2e-gce-cosstable1-k8sstable3-serial': 'ci-kubernetes-e2e-gce-cos*', 463 'ci-kubernetes-e2e-gce-cosstable1-k8sstable3-slow': 'ci-kubernetes-e2e-gce-cos*', 464 'ci-kubernetes-e2enode-cosbeta-k8sdev-default': 'ci-kubernetes-e2e-gce-cos*', 465 'ci-kubernetes-e2enode-cosbeta-k8sdev-serial': 'ci-kubernetes-e2e-gce-cos*', 466 'ci-kubernetes-e2enode-cosbeta-k8sbeta-default': 'ci-kubernetes-e2e-gce-cos*', 467 'ci-kubernetes-e2enode-cosbeta-k8sbeta-serial': 'ci-kubernetes-e2e-gce-cos*', 468 'ci-kubernetes-e2enode-cosbeta-k8sstable1-default': 'ci-kubernetes-e2e-gce-cos*', 469 'ci-kubernetes-e2enode-cosbeta-k8sstable1-serial': 'ci-kubernetes-e2e-gce-cos*', 470 'ci-kubernetes-e2enode-cosbeta-k8sstable2-default': 'ci-kubernetes-e2e-gce-cos*', 471 'ci-kubernetes-e2enode-cosbeta-k8sstable2-serial': 'ci-kubernetes-e2e-gce-cos*', 472 'ci-kubernetes-e2enode-cosbeta-k8sstable3-default': 'ci-kubernetes-e2e-gce-cos*', 473 'ci-kubernetes-e2enode-cosbeta-k8sstable3-serial': 'ci-kubernetes-e2e-gce-cos*', 474 475 # The ubuntu image validation jobs intentionally share projects. 476 'ci-kubernetes-e2enode-ubuntu1-k8sbeta-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', 477 'ci-kubernetes-e2enode-ubuntu1-k8sbeta-serial': 'ci-kubernetes-e2e-ubuntu-node*', 478 'ci-kubernetes-e2enode-ubuntu1-k8sstable1-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', 479 'ci-kubernetes-e2enode-ubuntu1-k8sstable1-serial': 'ci-kubernetes-e2e-ubuntu-node*', 480 'ci-kubernetes-e2enode-ubuntu1-k8sstable2-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', 481 'ci-kubernetes-e2enode-ubuntu1-k8sstable2-serial': 'ci-kubernetes-e2e-ubuntu-node*', 482 'ci-kubernetes-e2enode-ubuntu1-k8sstable3-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', 483 'ci-kubernetes-e2enode-ubuntu1-k8sstable3-serial': 'ci-kubernetes-e2e-ubuntu-node*', 484 485 'ci-kubernetes-e2e-gce-ubuntu1-k8sbeta-default': 'ci-kubernetes-e2e-gce-ubuntu*', 486 'ci-kubernetes-e2e-gce-ubuntu1-k8sbeta-serial': 'ci-kubernetes-e2e-gce-ubuntu*', 487 'ci-kubernetes-e2e-gce-ubuntu1-k8sbeta-slow': 'ci-kubernetes-e2e-gce-ubuntu*', 488 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable1-default': 'ci-kubernetes-e2e-gce-ubuntu*', 489 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable1-serial': 'ci-kubernetes-e2e-gce-ubuntu*', 490 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable1-slow': 'ci-kubernetes-e2e-gce-ubuntu*', 491 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable2-default': 'ci-kubernetes-e2e-gce-ubuntu*', 492 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable2-serial': 'ci-kubernetes-e2e-gce-ubuntu*', 493 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable2-slow': 'ci-kubernetes-e2e-gce-ubuntu*', 494 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable3-default': 'ci-kubernetes-e2e-gce-ubuntu*', 495 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable3-serial': 'ci-kubernetes-e2e-gce-ubuntu*', 496 'ci-kubernetes-e2e-gce-ubuntu1-k8sstable3-slow': 'ci-kubernetes-e2e-gce-ubuntu*', 497 498 'ci-kubernetes-e2enode-ubuntu2-k8sbeta-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', 499 'ci-kubernetes-e2enode-ubuntu2-k8sbeta-serial': 'ci-kubernetes-e2e-ubuntu-node*', 500 'ci-kubernetes-e2enode-ubuntu2-k8sstable1-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', 501 'ci-kubernetes-e2enode-ubuntu2-k8sstable1-serial': 'ci-kubernetes-e2e-ubuntu-node*', 502 'ci-kubernetes-e2enode-ubuntu2-k8sstable2-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', 503 'ci-kubernetes-e2enode-ubuntu2-k8sstable2-serial': 'ci-kubernetes-e2e-ubuntu-node*', 504 'ci-kubernetes-e2enode-ubuntu2-k8sstable3-gkespec': 'ci-kubernetes-e2e-ubuntu-node*', 505 'ci-kubernetes-e2enode-ubuntu2-k8sstable3-serial': 'ci-kubernetes-e2e-ubuntu-node*', 506 507 'ci-kubernetes-e2e-gce-ubuntu2-k8sbeta-default': 'ci-kubernetes-e2e-gce-ubuntu*', 508 'ci-kubernetes-e2e-gce-ubuntu2-k8sbeta-serial': 'ci-kubernetes-e2e-gce-ubuntu*', 509 'ci-kubernetes-e2e-gce-ubuntu2-k8sbeta-slow': 'ci-kubernetes-e2e-gce-ubuntu*', 510 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable1-default': 'ci-kubernetes-e2e-gce-ubuntu*', 511 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable1-serial': 'ci-kubernetes-e2e-gce-ubuntu*', 512 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable1-slow': 'ci-kubernetes-e2e-gce-ubuntu*', 513 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable2-default': 'ci-kubernetes-e2e-gce-ubuntu*', 514 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable2-serial': 'ci-kubernetes-e2e-gce-ubuntu*', 515 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable2-slow': 'ci-kubernetes-e2e-gce-ubuntu*', 516 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable3-default': 'ci-kubernetes-e2e-gce-ubuntu*', 517 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable3-serial': 'ci-kubernetes-e2e-gce-ubuntu*', 518 'ci-kubernetes-e2e-gce-ubuntu2-k8sstable3-slow': 'ci-kubernetes-e2e-gce-ubuntu*', 519 520 # The release branch scalability jobs intentionally share projects. 521 'ci-kubernetes-e2e-gci-gce-scalability-stable2': 'ci-kubernetes-e2e-gci-gce-scalability-release-*', 522 'ci-kubernetes-e2e-gci-gce-scalability-stable1': 'ci-kubernetes-e2e-gci-gce-scalability-release-*', 523 'ci-kubernetes-e2e-gce-scalability': 'ci-kubernetes-e2e-gce-scalability-*', 524 'ci-kubernetes-e2e-gce-scalability-canary': 'ci-kubernetes-e2e-gce-scalability-*', 525 # TODO(fejta): remove these (found while migrating jobs) 526 'ci-kubernetes-kubemark-100-gce': 'ci-kubernetes-kubemark-*', 527 'ci-kubernetes-kubemark-100-canary': 'ci-kubernetes-kubemark-*', 528 'ci-kubernetes-kubemark-5-gce-last-release': 'ci-kubernetes-kubemark-*', 529 'ci-kubernetes-kubemark-high-density-100-gce': 'ci-kubernetes-kubemark-*', 530 'ci-kubernetes-kubemark-gce-scale': 'ci-kubernetes-scale-*', 531 'pull-kubernetes-kubemark-e2e-gce-big': 'ci-kubernetes-scale-*', 532 'pull-kubernetes-kubemark-e2e-gce-scale': 'ci-kubernetes-scale-*', 533 'pull-kubernetes-e2e-gce-100-performance': 'ci-kubernetes-scale-*', 534 'pull-kubernetes-e2e-gce-big-performance': 'ci-kubernetes-scale-*', 535 'pull-kubernetes-e2e-gce-large-performance': 'ci-kubernetes-scale-*', 536 'ci-kubernetes-e2e-gce-large-manual-up': 'ci-kubernetes-scale-*', 537 'ci-kubernetes-e2e-gce-large-manual-down': 'ci-kubernetes-scale-*', 538 'ci-kubernetes-e2e-gce-large-correctness': 'ci-kubernetes-scale-*', 539 'ci-kubernetes-e2e-gce-large-performance': 'ci-kubernetes-scale-*', 540 'ci-kubernetes-e2e-gce-scale-correctness': 'ci-kubernetes-scale-*', 541 'ci-kubernetes-e2e-gce-scale-performance': 'ci-kubernetes-scale-*', 542 'ci-kubernetes-e2e-gke-large-correctness': 'ci-kubernetes-scale-*', 543 'ci-kubernetes-e2e-gke-large-performance': 'ci-kubernetes-scale-*', 544 'ci-kubernetes-e2e-gke-large-performance-regional': 'ci-kubernetes-scale-*', 545 'ci-kubernetes-e2e-gke-large-deploy': 'ci-kubernetes-scale-*', 546 'ci-kubernetes-e2e-gke-large-teardown': 'ci-kubernetes-scale-*', 547 'ci-kubernetes-e2e-gke-scale-correctness': 'ci-kubernetes-scale-*', 548 'pull-kubernetes-e2e-gce': 'pull-kubernetes-e2e-gce-*', 549 'pull-kubernetes-e2e-gce-canary': 'pull-kubernetes-e2e-gce-*', 550 'ci-kubernetes-e2e-gce': 'ci-kubernetes-e2e-gce-*', 551 'ci-kubernetes-e2e-gce-canary': 'ci-kubernetes-e2e-gce-*', 552 'ci-kubernetes-node-kubelet-serial': 'ci-kubernetes-node-kubelet-*', 553 'ci-kubernetes-node-kubelet-orphans': 'ci-kubernetes-node-kubelet-*', 554 'ci-kubernetes-node-kubelet-serial-cpu-manager': 'ci-kubernetes-node-kubelet-*', 555 'ci-kubernetes-node-kubelet-features': 'ci-kubernetes-node-kubelet-*', 556 'ci-kubernetes-node-kubelet-flaky': 'ci-kubernetes-node-kubelet-*', 557 'ci-kubernetes-node-kubelet-conformance': 'ci-kubernetes-node-kubelet-*', 558 'ci-kubernetes-node-kubelet-benchmark': 'ci-kubernetes-node-kubelet-*', 559 'ci-kubernetes-node-kubelet': 'ci-kubernetes-node-kubelet-*', 560 'ci-kubernetes-node-kubelet-stable1': 'ci-kubernetes-node-kubelet-*', 561 'ci-kubernetes-node-kubelet-stable2': 'ci-kubernetes-node-kubelet-*', 562 'ci-kubernetes-node-kubelet-stable3': 'ci-kubernetes-node-kubelet-*', 563 'ci-kubernetes-node-kubelet-alpha': 'ci-kubernetes-node-kubelet-*', 564 'ci-kubernetes-node-kubelet-beta': 'ci-kubernetes-node-kubelet-*', 565 'ci-kubernetes-node-kubelet-beta-features': 'ci-kubernetes-node-kubelet-*', 566 'ci-kubernetes-node-kubelet-non-cri-1-6': 'ci-kubernetes-node-kubelet-*', 567 # The cri-containerd validation node e2e jobs intentionally share projects. 568 'ci-cri-containerd-node-e2e': 'cri-containerd-node-e2e-*', 569 'ci-cri-containerd-node-e2e-serial': 'cri-containerd-node-e2e-*', 570 'ci-cri-containerd-node-e2e-features': 'cri-containerd-node-e2e-*', 571 'ci-cri-containerd-node-e2e-flaky': 'cri-containerd-node-e2e-*', 572 'ci-cri-containerd-node-e2e-benchmark': 'cri-containerd-node-e2e-*', 573 'ci-containerd-node-e2e': 'cri-containerd-node-e2e-*', 574 'ci-containerd-node-e2e-1-1': 'cri-containerd-node-e2e-*', 575 'ci-containerd-node-e2e-features': 'cri-containerd-node-e2e-*', 576 # ci-cri-containerd-e2e-gce-stackdriver intentionally share projects with 577 # ci-kubernetes-e2e-gce-stackdriver. 578 'ci-kubernetes-e2e-gce-stackdriver': 'k8s-jkns-e2e-gce-stackdriver', 579 'ci-cri-containerd-e2e-gce-stackdriver': 'k8s-jkns-e2e-gce-stackdriver', 580 # ingress-GCE e2e jobs 581 'pull-ingress-gce-e2e': 'e2e-ingress-gce', 582 'ci-ingress-gce-e2e': 'e2e-ingress-gce', 583 # sig-autoscaling jobs intentionally share projetcs 584 'ci-kubernetes-e2e-gci-gce-autoscaling-hpa':'ci-kubernetes-e2e-gci-gce-autoscaling', 585 'ci-kubernetes-e2e-gci-gce-autoscaling-migs-hpa':'ci-kubernetes-e2e-gci-gce-autoscaling-migs', 586 'ci-kubernetes-e2e-gci-gke-autoscaling-hpa':'ci-kubernetes-e2e-gci-gke-autoscaling', 587 # gpu+autoscaling jobs intentionally share projects with gpu tests 588 'ci-kubernetes-e2e-gci-gke-autoscaling-gpu-v100': 'ci-kubernetes-e2e-gke-staging-latest-device-plugin-gpu-v100', 589 } 590 # pylint: enable=line-too-long 591 projects = collections.defaultdict(set) 592 boskos = [] 593 with open(config_sort.test_infra('boskos/resources.yaml')) as fp: 594 boskos_config = yaml.safe_load(fp) 595 for rtype in boskos_config['resources']: 596 if 'project' in rtype['type']: 597 for name in rtype['names']: 598 boskos.append(name) 599 600 with open(config_sort.test_infra('jobs/config.json')) as fp: 601 job_config = json.load(fp) 602 for job in job_config: 603 project = '' 604 cfg = job_config.get(job.rsplit('.', 1)[0], {}) 605 if cfg.get('scenario') == 'kubernetes_e2e': 606 for arg in cfg.get('args', []): 607 if not arg.startswith('--gcp-project='): 608 continue 609 project = arg.split('=', 1)[1] 610 if project: 611 if project in boskos: 612 self.fail('Project %s cannot be in boskos/resources.yaml!' % project) 613 projects[project].add(allowed_list.get(job, job)) 614 615 duplicates = [(p, j) for p, j in projects.items() if len(j) > 1] 616 if duplicates: 617 self.fail('Jobs duplicate projects:\n %s' % ( 618 '\n '.join('%s: %s' % t for t in duplicates))) 619 620 def test_jobs_do_not_source_shell(self): 621 for job, job_path in self.jobs: 622 with open(job_path) as fp: 623 script = fp.read() 624 self.assertFalse(re.search(r'\Wsource ', script), job) 625 self.assertNotIn('\n. ', script, job) 626 627 def _check_env(self, job, setting): 628 if not re.match(r'[0-9A-Z_]+=[^\n]*', setting): 629 self.fail('[%r]: Env %r: need to follow FOO=BAR pattern' % (job, setting)) 630 if '#' in setting: 631 self.fail('[%r]: Env %r: No inline comments' % (job, setting)) 632 if '"' in setting or '\'' in setting: 633 self.fail('[%r]: Env %r: No quote in env' % (job, setting)) 634 if '$' in setting: 635 self.fail('[%r]: Env %r: Please resolve variables in env' % (job, setting)) 636 if '{' in setting or '}' in setting: 637 self.fail('[%r]: Env %r: { and } are not allowed in env' % (job, setting)) 638 # also test for https://github.com/kubernetes/test-infra/issues/2829 639 # TODO(fejta): sort this list 640 black = [ 641 ('CHARTS_TEST=', '--charts-tests'), 642 ('CLUSTER_IP_RANGE=', '--test_args=--cluster-ip-range=FOO'), 643 ('CLOUDSDK_BUCKET=', '--gcp-cloud-sdk=gs://foo'), 644 ('CLUSTER_NAME=', '--cluster=FOO'), 645 ('E2E_CLEAN_START=', '--test_args=--clean-start=true'), 646 ('E2E_DOWN=', '--down=true|false'), 647 ('E2E_MIN_STARTUP_PODS=', '--test_args=--minStartupPods=FOO'), 648 ('E2E_NAME=', '--cluster=whatever'), 649 ('E2E_PUBLISH_PATH=', '--publish=gs://FOO'), 650 ('E2E_REPORT_DIR=', '--test_args=--report-dir=FOO'), 651 ('E2E_REPORT_PREFIX=', '--test_args=--report-prefix=FOO'), 652 ('E2E_TEST=', '--test=true|false'), 653 ('E2E_UPGRADE_TEST=', '--upgrade_args=FOO'), 654 ('E2E_UP=', '--up=true|false'), 655 ('E2E_OPT=', 'Send kubetest the flags directly'), 656 ('FAIL_ON_GCP_RESOURCE_LEAK=', '--check-leaked-resources=true|false'), 657 ('FEDERATION_DOWN=', '--down=true|false'), 658 ('FEDERATION_UP=', '--up=true|false'), 659 ('GINKGO_PARALLEL=', '--ginkgo-parallel=# (1 for serial)'), 660 ('GINKGO_PARALLEL_NODES=', '--ginkgo-parallel=# (1 for serial)'), 661 ('GINKGO_TEST_ARGS=', '--test_args=FOO'), 662 ('GINKGO_UPGRADE_TEST_ARGS=', '--upgrade_args=FOO'), 663 ('JENKINS_FEDERATION_PREFIX=', '--stage=gs://FOO'), 664 ('JENKINS_GCI_PATCH_K8S=', 'Unused, see --extract docs'), 665 ('JENKINS_PUBLISHED_VERSION=', '--extract=V'), 666 ('JENKINS_PUBLISHED_SKEW_VERSION=', '--extract=V'), 667 ('JENKINS_USE_SKEW_KUBECTL=', 'SKEW_KUBECTL=y'), 668 ('JENKINS_USE_SKEW_TESTS=', '--skew'), 669 ('JENKINS_SOAK_MODE', '--soak'), 670 ('JENKINS_SOAK_PREFIX', '--stage=gs://FOO'), 671 ('JENKINS_USE_EXISTING_BINARIES=', '--extract=local'), 672 ('JENKINS_USE_LOCAL_BINARIES=', '--extract=none'), 673 ('JENKINS_USE_SERVER_VERSION=', '--extract=gke'), 674 ('JENKINS_USE_GCI_VERSION=', '--extract=gci/FAMILY'), 675 ('JENKINS_USE_GCI_HEAD_IMAGE_FAMILY=', '--extract=gci/FAMILY'), 676 ('KUBE_GKE_NETWORK=', '--gcp-network=FOO'), 677 ('KUBE_GCE_NETWORK=', '--gcp-network=FOO'), 678 ('KUBE_GCE_ZONE=', '--gcp-zone=FOO'), 679 ('KUBEKINS_TIMEOUT=', '--timeout=XXm'), 680 ('KUBEMARK_TEST_ARGS=', '--test_args=FOO'), 681 ('KUBEMARK_TESTS=', '--test_args=--ginkgo.focus=FOO'), 682 ('KUBEMARK_MASTER_SIZE=', '--kubemark-master-size=FOO'), 683 ('KUBEMARK_NUM_NODES=', '--kubemark-nodes=FOO'), 684 ('KUBE_OS_DISTRIBUTION=', '--gcp-node-image=FOO and --gcp-master-image=FOO'), 685 ('KUBE_NODE_OS_DISTRIBUTION=', '--gcp-node-image=FOO'), 686 ('KUBE_MASTER_OS_DISTRIBUTION=', '--gcp-master-image=FOO'), 687 ('KUBERNETES_PROVIDER=', '--provider=FOO'), 688 ('PERF_TESTS=', '--perf'), 689 ('PROJECT=', '--gcp-project=FOO'), 690 ('SKEW_KUBECTL=', '--test_args=--kubectl-path=FOO'), 691 ('USE_KUBEMARK=', '--kubemark'), 692 ('ZONE=', '--gcp-zone=FOO'), 693 ] 694 for env, fix in black: 695 if 'kops' in job and env in [ 696 'JENKINS_PUBLISHED_VERSION=', 697 'JENKINS_USE_LOCAL_BINARIES=', 698 'GINKGO_TEST_ARGS=', 699 'KUBERNETES_PROVIDER=', 700 ]: 701 continue # TODO(fejta): migrate kops jobs 702 if setting.startswith(env): 703 self.fail('[%s]: Env %s: Convert %s to use %s in jobs/config.json' % ( 704 job, setting, env, fix)) 705 706 def test_envs_no_export(self): 707 for job, job_path in self.jobs: 708 if not job.endswith('.env'): 709 continue 710 with open(job_path) as fp: 711 lines = list(fp) 712 for line in lines: 713 line = line.strip() 714 self.assertFalse(line.endswith('\\')) 715 if not line: 716 continue 717 if line.startswith('#'): 718 continue 719 self._check_env(job, line) 720 721 def test_envs_non_empty(self): 722 bad = [] 723 for job, job_path in self.jobs: 724 if not job.endswith('.env'): 725 continue 726 with open(job_path) as fp: 727 lines = list(fp) 728 for line in lines: 729 line = line.strip() 730 if line and not line.startswith('#'): 731 break 732 else: 733 bad.append(job) 734 if bad: 735 self.fail('%s is empty, please remove the file(s)' % bad) 736 737 def test_no_bad_vars_in_jobs(self): 738 """Searches for jobs that contain ${{VAR}}""" 739 for job, job_path in self.jobs: 740 with open(job_path) as fp: 741 script = fp.read() 742 bad_vars = re.findall(r'(\${{.+}})', script) 743 if bad_vars: 744 self.fail('Job %s contains bad bash variables: %s' % (job, ' '.join(bad_vars))) 745 746 if __name__ == '__main__': 747 unittest.main()