github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/jenkins/bootstrap_test.py (about) 1 #!/usr/bin/env python 2 3 # Copyright 2016 The Kubernetes Authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Tests for bootstrap.""" 18 19 # pylint: disable=protected-access, attribute-defined-outside-init 20 21 import argparse 22 import json 23 import os 24 import select 25 import signal 26 import subprocess 27 import tempfile 28 import time 29 import unittest 30 31 import bootstrap 32 33 34 BRANCH = 'random_branch' 35 BUILD = 'random_build' 36 FAIL = ['/bin/bash', '-c', 'exit 1'] 37 JOB = 'random_job' 38 PASS = ['/bin/bash', '-c', 'exit 0'] 39 PULL = 12345 40 REPO = 'github.com/random_org/random_repo' 41 ROBOT = 'fake-service-account.json' 42 ROOT = '/random/root' 43 UPLOAD = 'fake-gs://fake-bucket' 44 45 46 class Stub(object): 47 """Replace thing.param with replacement until exiting with.""" 48 def __init__(self, thing, param, replacement): 49 self.thing = thing 50 self.param = param 51 self.replacement = replacement 52 self.old = getattr(thing, param) 53 setattr(thing, param, self.replacement) 54 55 def __enter__(self, *a, **kw): 56 return self.replacement 57 58 def __exit__(self, *a, **kw): 59 setattr(self.thing, self.param, self.old) 60 61 62 class FakeCall(object): 63 def __init__(self): 64 self.calls = [] 65 66 def __call__(self, *a, **kw): 67 self.calls.append((a, kw)) 68 69 70 class FakeSubprocess(object): 71 """Keep track of calls.""" 72 def __init__(self): 73 self.calls = [] 74 75 def __call__(self, cmd, *a, **kw): 76 self.calls.append((cmd, a, kw)) 77 78 79 # pylint: disable=invalid-name 80 def Pass(*_a, **_kw): 81 """Do nothing.""" 82 pass 83 84 85 def Truth(*_a, **_kw): 86 """Always true.""" 87 return True 88 89 90 def Bomb(*a, **kw): 91 """Always raise.""" 92 raise AssertionError('Should not happen', a, kw) 93 # pylint: enable=invalid-name 94 95 96 class ReadAllTest(unittest.TestCase): 97 endless = 0 98 ended = time.time() - 50 99 number = 0 100 end = -1 101 102 def fileno(self): 103 return self.end 104 105 def readline(self): 106 line = 'line %d\n' % self.number 107 self.number += 1 108 return line 109 110 def test_read_more(self): 111 """Read lines until we clear the buffer, noting there may be more.""" 112 lines = [] 113 total = 10 114 def more_lines(*_a, **_kw): 115 if len(lines) < total: 116 return [self], [], [] 117 return [], [], [] 118 with Stub(select, 'select', more_lines): 119 done = bootstrap.read_all(self.endless, self, lines.append) 120 121 self.assertFalse(done) 122 self.assertEquals(total, len(lines)) 123 expected = ['line %d' % d for d in range(total)] 124 self.assertEquals(expected, lines) 125 126 def test_read_expired(self): 127 """Read nothing as we are expired, noting there may be more.""" 128 lines = [] 129 with Stub(select, 'select', lambda *a, **kw: ([], [], [])): 130 done = bootstrap.read_all(self.ended, self, lines.append) 131 132 self.assertFalse(done) 133 self.assertFalse(lines) 134 135 def test_read_end(self): 136 """Note we reached the end of the stream.""" 137 lines = [] 138 with Stub(select, 'select', lambda *a, **kw: ([self], [], [])): 139 with Stub(self, 'readline', lambda: ''): 140 done = bootstrap.read_all(self.endless, self, lines.append) 141 142 self.assertTrue(done) 143 144 145 class TerminateTest(unittest.TestCase): 146 """Tests for termiante().""" 147 pid = 1234 148 pgid = 5555 149 terminated = False 150 killed = False 151 152 def terminate(self): 153 self.terminated = True 154 155 def kill(self): 156 self.killed = True 157 158 def getpgid(self, pid): 159 self.got = pid 160 return self.pgid 161 162 def killpg(self, pgig, sig): 163 self.killed_pg = (pgig, sig) 164 165 def test_terminate_later(self): 166 """Do nothing if end is in the future.""" 167 timeout = bootstrap.terminate(time.time() + 50, self, False) 168 self.assertFalse(timeout) 169 170 def test_terminate_never(self): 171 """Do nothing if end is zero.""" 172 timeout = bootstrap.terminate(0, self, False) 173 self.assertFalse(timeout) 174 175 def test_terminate_terminate(self): 176 """Terminate pid if after end and kill is false.""" 177 timeout = bootstrap.terminate(time.time() - 50, self, False) 178 self.assertTrue(timeout) 179 self.assertFalse(self.killed) 180 self.assertTrue(self.terminated) 181 182 def test_terminate_kill(self): 183 """Kill process group if after end and kill is true.""" 184 with Stub(os, 'getpgid', self.getpgid), Stub(os, 'killpg', self.killpg): 185 timeout = bootstrap.terminate(time.time() - 50, self, True) 186 self.assertTrue(timeout) 187 self.assertFalse(self.terminated) 188 self.assertTrue(self.killed) 189 self.assertEquals(self.pid, self.got) 190 self.assertEquals(self.killed_pg, (self.pgid, signal.SIGKILL)) 191 192 193 class SubprocessTest(unittest.TestCase): 194 """Tests for call().""" 195 196 def test_stdin(self): 197 """Will write to subprocess.stdin.""" 198 with self.assertRaises(subprocess.CalledProcessError) as cpe: 199 bootstrap._call(0, ['/bin/bash'], stdin='exit 92') 200 self.assertEquals(92, cpe.exception.returncode) 201 202 def test_check_true(self): 203 """Raise on non-zero exit codes if check is set.""" 204 with self.assertRaises(subprocess.CalledProcessError): 205 bootstrap._call(0, FAIL, check=True) 206 207 bootstrap._call(0, PASS, check=True) 208 209 def test_check_default(self): 210 """Default to check=True.""" 211 with self.assertRaises(subprocess.CalledProcessError): 212 bootstrap._call(0, FAIL) 213 214 bootstrap._call(0, PASS) 215 216 @staticmethod 217 def test_check_false(): 218 """Never raise when check is not set.""" 219 bootstrap._call(0, FAIL, check=False) 220 bootstrap._call(0, PASS, check=False) 221 222 def test_output(self): 223 """Output is returned when requested.""" 224 cmd = ['/bin/bash', '-c', 'echo hello world'] 225 self.assertEquals( 226 'hello world\n', bootstrap._call(0, cmd, output=True)) 227 228 def test_zombie(self): 229 with self.assertRaises(subprocess.CalledProcessError): 230 # make a zombie 231 bootstrap._call(0, ['/bin/bash', '-c', 'A=$BASHPID && ( kill -STOP $A ) & exit 1']) 232 233 234 class PullRefsTest(unittest.TestCase): 235 """Tests for pull_ref, branch_ref, ref_has_shas, and pull_numbers.""" 236 237 def test_multiple_no_shas(self): 238 """Test master,1111,2222.""" 239 self.assertEqual( 240 bootstrap.pull_ref('master,123,456'), 241 ([ 242 'master', 243 '+refs/pull/123/head:refs/pr/123', 244 '+refs/pull/456/head:refs/pr/456', 245 ], [ 246 'FETCH_HEAD', 247 'refs/pr/123', 248 'refs/pr/456', 249 ]), 250 ) 251 252 def test_pull_has_shas(self): 253 self.assertTrue(bootstrap.ref_has_shas('master:abcd')) 254 self.assertFalse(bootstrap.ref_has_shas('123')) 255 self.assertFalse(bootstrap.ref_has_shas(123)) 256 self.assertFalse(bootstrap.ref_has_shas(None)) 257 258 def test_pull_numbers(self): 259 self.assertListEqual(bootstrap.pull_numbers(123), ['123']) 260 self.assertListEqual(bootstrap.pull_numbers('master:abcd'), []) 261 self.assertListEqual( 262 bootstrap.pull_numbers('master:abcd,123:qwer,124:zxcv'), 263 ['123', '124']) 264 265 def test_pull_ref(self): 266 self.assertEqual( 267 bootstrap.pull_ref('master:abcd,123:effe'), 268 (['master', '+refs/pull/123/head:refs/pr/123'], ['abcd', 'effe']) 269 ) 270 self.assertEqual( 271 bootstrap.pull_ref('123'), 272 (['+refs/pull/123/merge'], ['FETCH_HEAD']) 273 ) 274 275 def test_branch_ref(self): 276 self.assertEqual( 277 bootstrap.branch_ref('branch:abcd'), 278 (['branch'], ['abcd']) 279 ) 280 self.assertEqual( 281 bootstrap.branch_ref('master'), 282 (['master'], ['FETCH_HEAD']) 283 ) 284 285 286 class ChooseSshKeyTest(unittest.TestCase): 287 """Tests for choose_ssh_key().""" 288 def test_empty(self): 289 """Do not change environ if no ssh key.""" 290 fake_env = {} 291 with Stub(os, 'environ', fake_env): 292 with bootstrap.choose_ssh_key(''): 293 self.assertFalse(fake_env) 294 295 def test_full(self): 296 fake_env = {} 297 with Stub(os, 'environ', fake_env): 298 with bootstrap.choose_ssh_key('hello there'): 299 self.assertIn('GIT_SSH', fake_env) 300 with open(fake_env['GIT_SSH']) as fp: 301 buf = fp.read() 302 self.assertIn('hello there', buf) 303 self.assertIn('ssh ', buf) 304 self.assertIn(' -i ', buf) 305 self.assertFalse(fake_env) # Resets env 306 307 def test_full_old_value(self): 308 fake_env = {'GIT_SSH': 'random-value'} 309 old_env = dict(fake_env) 310 with Stub(os, 'environ', fake_env): 311 with bootstrap.choose_ssh_key('hello there'): 312 self.assertNotEqual(old_env, fake_env) 313 self.assertEquals(old_env, fake_env) 314 315 316 class CheckoutTest(unittest.TestCase): 317 """Tests for checkout().""" 318 319 def test_clean(self): 320 """checkout cleans and resets if asked to.""" 321 fake = FakeSubprocess() 322 with Stub(os, 'chdir', Pass): 323 bootstrap.checkout(fake, REPO, None, PULL, clean=True) 324 325 self.assertTrue(any( 326 'clean' in cmd for cmd, _, _ in fake.calls if 'git' in cmd)) 327 self.assertTrue(any( 328 'reset' in cmd for cmd, _, _ in fake.calls if 'git' in cmd)) 329 330 def test_fetch_retries(self): 331 self.tries = 0 332 expected_attempts = 3 333 def third_time_charm(cmd, *_a, **_kw): 334 if 'fetch' not in cmd: # init/checkout are unlikely to fail 335 return 336 self.tries += 1 337 if self.tries != expected_attempts: 338 raise subprocess.CalledProcessError(128, cmd, None) 339 with Stub(os, 'chdir', Pass): 340 with Stub(time, 'sleep', Pass): 341 bootstrap.checkout(third_time_charm, REPO, None, PULL) 342 self.assertEquals(expected_attempts, self.tries) 343 344 def test_pull_ref(self): 345 """checkout fetches the right ref for a pull.""" 346 fake = FakeSubprocess() 347 with Stub(os, 'chdir', Pass): 348 bootstrap.checkout(fake, REPO, None, PULL) 349 350 expected_ref = bootstrap.pull_ref(PULL)[0][0] 351 self.assertTrue(any( 352 expected_ref in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd)) 353 354 def test_branch(self): 355 """checkout fetches the right ref for a branch.""" 356 fake = FakeSubprocess() 357 with Stub(os, 'chdir', Pass): 358 bootstrap.checkout(fake, REPO, BRANCH, None) 359 360 expected_ref = BRANCH 361 self.assertTrue(any( 362 expected_ref in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd)) 363 364 def test_repo(self): 365 """checkout initializes and fetches the right repo.""" 366 fake = FakeSubprocess() 367 with Stub(os, 'chdir', Pass): 368 bootstrap.checkout(fake, REPO, BRANCH, None) 369 370 expected_uri = 'https://%s' % REPO 371 self.assertTrue(any( 372 expected_uri in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd)) 373 374 def test_branch_xor_pull(self): 375 """Either branch or pull specified, not both.""" 376 with Stub(os, 'chdir', Bomb): 377 with self.assertRaises(ValueError): 378 bootstrap.checkout(Bomb, REPO, None, None) 379 with self.assertRaises(ValueError): 380 bootstrap.checkout(Bomb, REPO, BRANCH, PULL) 381 382 def test_happy(self): 383 """checkout sanity check.""" 384 fake = FakeSubprocess() 385 with Stub(os, 'chdir', Pass): 386 bootstrap.checkout(fake, REPO, BRANCH, None) 387 388 self.assertTrue(any( 389 '--tags' in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd)) 390 self.assertTrue(any( 391 'FETCH_HEAD' in cmd for cmd, _, _ in fake.calls 392 if 'checkout' in cmd)) 393 394 class ParseReposTest(unittest.TestCase): 395 def test_bare(self): 396 """--bare works.""" 397 args = bootstrap.parse_args(['--job=foo', '--bare']) 398 self.assertFalse(bootstrap.parse_repos(args)) 399 400 def test_pull_branch_none(self): 401 """args.pull and args.branch should be None""" 402 args = bootstrap.parse_args(['--job=foo', '--bare']) 403 self.assertIsNone(args.pull) 404 self.assertIsNone(args.branch) 405 406 def test_plain(self): 407 """"--repo=foo equals foo=master.""" 408 args = bootstrap.parse_args(['--job=foo', '--repo=foo']) 409 self.assertEquals( 410 {'foo': ('master', '')}, 411 bootstrap.parse_repos(args)) 412 413 def test_branch(self): 414 """--repo=foo=branch.""" 415 args = bootstrap.parse_args(['--job=foo', '--repo=foo=this']) 416 self.assertEquals( 417 {'foo': ('this', '')}, 418 bootstrap.parse_repos(args)) 419 420 def test_branch_commit(self): 421 """--repo=foo=branch:commit works.""" 422 args = bootstrap.parse_args(['--job=foo', '--repo=foo=this:abcd']) 423 self.assertEquals( 424 {'foo': ('this:abcd', '')}, 425 bootstrap.parse_repos(args)) 426 427 def test_parse_repos(self): 428 """--repo=foo=111,222 works""" 429 args = bootstrap.parse_args(['--job=foo', '--repo=foo=111,222']) 430 self.assertEquals( 431 {'foo': ('', '111,222')}, 432 bootstrap.parse_repos(args)) 433 434 def test_pull_branch(self): 435 """--repo=foo=master,111,222 works""" 436 args = bootstrap.parse_args(['--job=foo', '--repo=foo=master,111,222']) 437 self.assertEquals( 438 {'foo': ('', 'master,111,222')}, 439 bootstrap.parse_repos(args)) 440 441 def test_pull_release_branch(self): 442 """--repo=foo=release-3.14,&a-fancy%_branch+:abcd,222 works""" 443 args = bootstrap.parse_args(['--job=foo', 444 '--repo=foo=release-3.14,&a-fancy%_branch+:abcd,222']) 445 self.assertEquals( 446 {'foo': ('', 'release-3.14,&a-fancy%_branch+:abcd,222')}, 447 bootstrap.parse_repos(args)) 448 449 def test_pull_branch_commit(self): 450 """--repo=foo=master,111,222 works""" 451 args = bootstrap.parse_args(['--job=foo', 452 '--repo=foo=master:aaa,111:bbb,222:ccc']) 453 self.assertEquals( 454 {'foo': ('', 'master:aaa,111:bbb,222:ccc')}, 455 bootstrap.parse_repos(args)) 456 457 def test_multi_repo(self): 458 """--repo=foo=master,111,222 bar works""" 459 args = bootstrap.parse_args(['--job=foo', 460 '--repo=foo=master:aaa,111:bbb,222:ccc', 461 '--repo=bar']) 462 self.assertEquals( 463 { 464 'foo': ('', 'master:aaa,111:bbb,222:ccc'), 465 'bar': ('master', '')}, 466 bootstrap.parse_repos(args)) 467 468 469 class GSUtilTest(unittest.TestCase): 470 """Tests for GSUtil.""" 471 def test_upload_json(self): 472 fake = FakeSubprocess() 473 gsutil = bootstrap.GSUtil(fake) 474 gsutil.upload_json('fake_path', {'wee': 'fun'}) 475 self.assertTrue(any( 476 'application/json' in a for a in fake.calls[0][0])) 477 self.assertIn('stdin', fake.calls[0][2]) # kwargs 478 479 def test_upload_text_cached(self): 480 fake = FakeSubprocess() 481 gsutil = bootstrap.GSUtil(fake) 482 gsutil.upload_text('fake_path', 'hello world', cached=True) 483 self.assertFalse(any( 484 'Cache-Control' in a and 'max-age' in a 485 for a in fake.calls[0][0])) 486 self.assertIn('stdin', fake.calls[0][2]) # kwargs 487 488 def test_upload_text_default(self): 489 fake = FakeSubprocess() 490 gsutil = bootstrap.GSUtil(fake) 491 gsutil.upload_text('fake_path', 'hello world') 492 self.assertFalse(any( 493 'Cache-Control' in a and 'max-age' in a 494 for a in fake.calls[0][0])) 495 self.assertIn('stdin', fake.calls[0][2]) # kwargs 496 497 def test_upload_text_uncached(self): 498 fake = FakeSubprocess() 499 gsutil = bootstrap.GSUtil(fake) 500 gsutil.upload_text('fake_path', 'hello world', cached=False) 501 self.assertTrue(any( 502 'Cache-Control' in a and 'max-age' in a 503 for a in fake.calls[0][0])) 504 self.assertIn('stdin', fake.calls[0][2]) # kwargs 505 506 def test_upload_text_metalink(self): 507 fake = FakeSubprocess() 508 gsutil = bootstrap.GSUtil(fake) 509 gsutil.upload_text('txt', 'path', additional_headers=['foo: bar']) 510 self.assertTrue(any('foo: bar' in a for a in fake.calls[0][0])) 511 512 class FakeGSUtil(object): 513 generation = 123 514 515 def __init__(self): 516 self.cats = [] 517 self.jsons = [] 518 self.stats = [] 519 self.texts = [] 520 521 def cat(self, *a, **kw): 522 self.cats.append((a, kw)) 523 return 'this is not a list' 524 525 def stat(self, *a, **kw): 526 self.stats.append((a, kw)) 527 return 'Generation: %s' % self.generation 528 529 def upload_text(self, *args, **kwargs): 530 self.texts.append((args, kwargs)) 531 532 def upload_json(self, *args, **kwargs): 533 self.jsons.append((args, kwargs)) 534 535 class GubernatorUriTest(unittest.TestCase): 536 def create_path(self, uri): 537 self.fake_path = FakePath() 538 self.fake_path.build_log = uri 539 return self.fake_path 540 541 def test_non_gs(self): 542 uri = 'hello/world' 543 self.assertEquals('hello', bootstrap.gubernator_uri(self.create_path(uri))) 544 545 def test_multiple_gs(self): 546 uri = 'gs://hello/gs://there' 547 self.assertEquals( 548 bootstrap.GUBERNATOR + '/hello/gs:', 549 bootstrap.gubernator_uri(self.create_path(uri))) 550 551 def test_gs(self): 552 uri = 'gs://blah/blah/blah.txt' 553 self.assertEquals( 554 bootstrap.GUBERNATOR + '/blah/blah', 555 bootstrap.gubernator_uri(self.create_path(uri))) 556 557 558 559 class AppendResultTest(unittest.TestCase): 560 """Tests for append_result().""" 561 562 def test_new_job(self): 563 """Stat fails when the job doesn't exist.""" 564 gsutil = FakeGSUtil() 565 build = 123 566 version = 'v.interesting' 567 success = True 568 def fake_stat(*_a, **_kw): 569 raise subprocess.CalledProcessError(1, ['gsutil'], None) 570 gsutil.stat = fake_stat 571 bootstrap.append_result(gsutil, 'fake_path', build, version, success) 572 cache = gsutil.jsons[0][0][1] 573 self.assertEquals(1, len(cache)) 574 575 def test_collision_cat(self): 576 """cat fails if the cache has been updated.""" 577 gsutil = FakeGSUtil() 578 build = 42 579 version = 'v1' 580 success = False 581 generations = ['555', '444'] 582 orig_stat = gsutil.stat 583 def fake_stat(*a, **kw): 584 gsutil.generation = generations.pop() 585 return orig_stat(*a, **kw) 586 def fake_cat(_, gen): 587 if gen == '555': # Which version is requested? 588 return '[{"hello": 111}]' 589 raise subprocess.CalledProcessError(1, ['gsutil'], None) 590 with Stub(bootstrap, 'random_sleep', Pass): 591 with Stub(gsutil, 'stat', fake_stat): 592 with Stub(gsutil, 'cat', fake_cat): 593 bootstrap.append_result( 594 gsutil, 'fake_path', build, version, success) 595 self.assertIn('generation', gsutil.jsons[-1][1], gsutil.jsons) 596 self.assertEquals('555', gsutil.jsons[-1][1]['generation'], gsutil.jsons) 597 598 def test_collision_upload(self): 599 """Test when upload_json tries to update an old version.""" 600 gsutil = FakeGSUtil() 601 build = 42 602 version = 'v1' 603 success = False 604 generations = [555, 444] 605 orig = gsutil.upload_json 606 def fake_upload(path, cache, generation): 607 if generation == '555': 608 return orig(path, cache, generation=generation) 609 raise subprocess.CalledProcessError(128, ['gsutil'], None) 610 orig_stat = gsutil.stat 611 def fake_stat(*a, **kw): 612 gsutil.generation = generations.pop() 613 return orig_stat(*a, **kw) 614 def fake_cat(*_a, **_kw): 615 return '[{"hello": 111}]' 616 gsutil.stat = fake_stat 617 gsutil.upload_json = fake_upload 618 gsutil.cat = fake_cat 619 with Stub(bootstrap, 'random_sleep', Pass): 620 bootstrap.append_result( 621 gsutil, 'fake_path', build, version, success) 622 self.assertIn('generation', gsutil.jsons[-1][1], gsutil.jsons) 623 self.assertEquals('555', gsutil.jsons[-1][1]['generation'], gsutil.jsons) 624 625 def test_handle_junk(self): 626 gsutil = FakeGSUtil() 627 gsutil.cat = lambda *a, **kw: '!@!$!@$@!$' 628 build = 123 629 version = 'v.interesting' 630 success = True 631 bootstrap.append_result(gsutil, 'fake_path', build, version, success) 632 cache = gsutil.jsons[0][0][1] 633 self.assertEquals(1, len(cache)) 634 self.assertIn(build, cache[0].values()) 635 self.assertIn(version, cache[0].values()) 636 637 def test_passed_is_bool(self): 638 build = 123 639 version = 'v.interesting' 640 def try_run(success): 641 gsutil = FakeGSUtil() 642 bootstrap.append_result(gsutil, 'fake_path', build, version, success) 643 cache = gsutil.jsons[0][0][1] 644 self.assertTrue(isinstance(cache[0]['passed'], bool)) 645 646 try_run(1) 647 try_run(0) 648 try_run(None) 649 try_run('') 650 try_run('hello') 651 try_run('true') 652 653 def test_truncate(self): 654 old = json.dumps({n: True for n in range(100000)}) 655 gsutil = FakeGSUtil() 656 build = 123 657 version = 'v.interesting' 658 success = True 659 bootstrap.append_result(gsutil, 'fake_path', build, version, success) 660 cache = gsutil.jsons[0][0][1] 661 self.assertLess(len(cache), len(old)) 662 663 664 class FinishTest(unittest.TestCase): 665 """Tests for finish().""" 666 def setUp(self): 667 self.stubs = [ 668 Stub(bootstrap.GSUtil, 'upload_artifacts', Pass), 669 Stub(bootstrap, 'append_result', Pass), 670 Stub(os.path, 'isfile', Pass), 671 Stub(os.path, 'isdir', Pass), 672 ] 673 674 def tearDown(self): 675 for stub in self.stubs: 676 with stub: 677 pass 678 679 def test_no_version(self): 680 gsutil = FakeGSUtil() 681 paths = FakePath() 682 success = True 683 artifacts = 'not-a-dir' 684 no_version = '' 685 version = 'should not have found it' 686 repos = repo({REPO: ('master', '')}) 687 with Stub(bootstrap, 'metadata', lambda *a: {'random-meta': version}): 688 bootstrap.finish(gsutil, paths, success, artifacts, 689 BUILD, no_version, repos, FakeCall()) 690 bootstrap.finish(gsutil, paths, success, artifacts, BUILD, no_version, repos, FakeCall()) 691 calls = gsutil.jsons[-1] 692 # json data is second positional argument 693 self.assertNotIn('job-version', calls[0][1]) 694 self.assertNotIn('version', calls[0][1]) 695 self.assertTrue(calls[0][1].get('metadata')) 696 697 def test_metadata_version(self): 698 """Test that we will extract version info from metadata.""" 699 self.check_metadata_version('job-version') 700 self.check_metadata_version('version') 701 702 def check_metadata_version(self, key): 703 gsutil = FakeGSUtil() 704 paths = FakePath() 705 success = True 706 artifacts = 'not-a-dir' 707 no_version = '' 708 version = 'found it' 709 with Stub(bootstrap, 'metadata', lambda *a: {key: version}): 710 bootstrap.finish(gsutil, paths, success, artifacts, BUILD, no_version, REPO, FakeCall()) 711 calls = gsutil.jsons[-1] 712 # Meta is second positional argument 713 self.assertEquals(version, calls[0][1].get('job-version')) 714 self.assertEquals(version, calls[0][1].get('version')) 715 716 def test_ignore_err_up_artifacts(self): 717 paths = FakePath() 718 gsutil = FakeGSUtil() 719 local_artifacts = None 720 build = 123 721 version = 'v1.terrible' 722 success = True 723 calls = [] 724 with Stub(os.path, 'isdir', lambda _: True): 725 with Stub(os, 'walk', lambda d: [(True, True, True)]): 726 def fake_upload(*a, **kw): 727 calls.append((a, kw)) 728 raise subprocess.CalledProcessError(1, ['fakecmd'], None) 729 gsutil.upload_artifacts = fake_upload 730 repos = repo({REPO: ('master', '')}) 731 bootstrap.finish( 732 gsutil, paths, success, local_artifacts, 733 build, version, repos, FakeCall()) 734 self.assertTrue(calls) 735 736 737 def test_ignore_error_uploadtext(self): 738 paths = FakePath() 739 gsutil = FakeGSUtil() 740 local_artifacts = None 741 build = 123 742 version = 'v1.terrible' 743 success = True 744 calls = [] 745 with Stub(os.path, 'isdir', lambda _: True): 746 with Stub(os, 'walk', lambda d: [(True, True, True)]): 747 def fake_upload(*a, **kw): 748 calls.append((a, kw)) 749 raise subprocess.CalledProcessError(1, ['fakecmd'], None) 750 gsutil.upload_artifacts = Pass 751 gsutil.upload_text = fake_upload 752 repos = repo({REPO: ('master', '')}) 753 bootstrap.finish( 754 gsutil, paths, success, local_artifacts, 755 build, version, repos, FakeCall()) 756 self.assertTrue(calls) 757 self.assertGreater(calls, 1) 758 759 def test_skip_upload_artifacts(self): 760 """Do not upload artifacts dir if it doesn't exist.""" 761 paths = FakePath() 762 gsutil = FakeGSUtil() 763 local_artifacts = None 764 build = 123 765 version = 'v1.terrible' 766 success = True 767 calls = [] 768 with Stub(os.path, 'isdir', lambda _: False): 769 with Stub(bootstrap.GSUtil, 'upload_artifacts', Bomb): 770 repos = repo({REPO: ('master', '')}) 771 bootstrap.finish( 772 gsutil, paths, success, local_artifacts, 773 build, version, repos, FakeCall()) 774 self.assertFalse(calls) 775 776 777 class MetadataTest(unittest.TestCase): 778 779 def test_always_set_metadata(self): 780 repos = repo({REPO: ('master', '')}) 781 meta = bootstrap.metadata(repos, 'missing-artifacts-dir', FakeCall()) 782 self.assertIn('repo', meta) 783 self.assertEquals(REPO, meta['repo']) 784 785 def test_multi_repo(self): 786 repos = repo({REPO: ('foo', ''), 'other-repo': ('', '123,456')}) 787 meta = bootstrap.metadata(repos, 'missing-artifacts-dir', FakeCall()) 788 self.assertIn('repo', meta) 789 self.assertEquals(REPO, meta['repo']) 790 self.assertIn(REPO, meta.get('repos')) 791 self.assertEquals('foo', meta['repos'][REPO]) 792 self.assertIn('other-repo', meta.get('repos')) 793 self.assertEquals('123,456', meta['repos']['other-repo']) 794 795 796 SECONDS = 10 797 798 799 def fake_environment(set_home=True, set_node=True, set_job=True, 800 set_jenkins_home=True, set_workspace=True, **kwargs): 801 if set_home: 802 kwargs.setdefault(bootstrap.HOME_ENV, '/fake/home-dir') 803 if set_node: 804 kwargs.setdefault(bootstrap.NODE_ENV, 'fake-node') 805 if set_job: 806 kwargs.setdefault(bootstrap.JOB_ENV, JOB) 807 if set_jenkins_home: 808 kwargs.setdefault(bootstrap.JENKINS_HOME_ENV, '/fake/home-dir') 809 if set_workspace: 810 kwargs.setdefault(bootstrap.WORKSPACE_ENV, '/fake/workspace') 811 return kwargs 812 813 814 class BuildNameTest(unittest.TestCase): 815 """Tests for build_name().""" 816 817 def test_auto(self): 818 """Automatically select a build if not done by user.""" 819 with Stub(os, 'environ', fake_environment()) as fake: 820 bootstrap.build_name(SECONDS) 821 self.assertTrue(fake[bootstrap.BUILD_ENV]) 822 823 def test_manual(self): 824 """Respect user-selected build.""" 825 with Stub(os, 'environ', fake_environment()) as fake: 826 truth = 'erick is awesome' 827 fake[bootstrap.BUILD_ENV] = truth 828 self.assertEquals(truth, fake[bootstrap.BUILD_ENV]) 829 830 def test_unique(self): 831 """New build every minute.""" 832 with Stub(os, 'environ', fake_environment()) as fake: 833 bootstrap.build_name(SECONDS) 834 first = fake[bootstrap.BUILD_ENV] 835 del fake[bootstrap.BUILD_ENV] 836 bootstrap.build_name(SECONDS + 60) 837 self.assertNotEqual(first, fake[bootstrap.BUILD_ENV]) 838 839 840 class SetupCredentialsTest(unittest.TestCase): 841 """Tests for setup_credentials().""" 842 843 def setUp(self): 844 keys = { 845 bootstrap.GCE_KEY_ENV: 'fake-key', 846 bootstrap.SERVICE_ACCOUNT_ENV: 'fake-service-account.json', 847 } 848 self.env = fake_environment(**keys) 849 850 def test_norobot_noupload_noenv(self): 851 """Can avoid setting up credentials.""" 852 del self.env[bootstrap.SERVICE_ACCOUNT_ENV] 853 with Stub(os, 'environ', self.env): 854 bootstrap.setup_credentials(Bomb, None, None) 855 856 def test_upload_no_robot_raises(self): 857 del self.env[bootstrap.SERVICE_ACCOUNT_ENV] 858 with Stub(os, 'environ', self.env): 859 with self.assertRaises(ValueError): 860 bootstrap.setup_credentials(Pass, None, 'gs://fake') 861 862 863 def test_application_credentials(self): 864 """Raise if GOOGLE_APPLICATION_CREDENTIALS does not exist.""" 865 del self.env[bootstrap.SERVICE_ACCOUNT_ENV] 866 with Stub(os, 'environ', self.env) as fake: 867 gac = 'FAKE_CREDS.json' 868 fake['HOME'] = 'kansas' 869 with Stub(os.path, 'isfile', lambda p: p != gac): 870 with self.assertRaises(IOError): 871 bootstrap.setup_credentials(Pass, gac, UPLOAD) 872 873 with Stub(os.path, 'isfile', Truth): 874 call = lambda *a, **kw: 'robot' 875 bootstrap.setup_credentials(call, gac, UPLOAD) 876 # setup_creds should set SERVICE_ACCOUNT_ENV 877 self.assertEquals(gac, fake.get(bootstrap.SERVICE_ACCOUNT_ENV)) 878 # now that SERVICE_ACCOUNT_ENV is set, it should try to activate 879 # this 880 with Stub(os.path, 'isfile', lambda p: p != gac): 881 with self.assertRaises(IOError): 882 bootstrap.setup_credentials(Pass, None, UPLOAD) 883 884 885 class SetupMagicEnvironmentTest(unittest.TestCase): 886 def test_home_workspace_on_jenkins(self): 887 """WORKSPACE/HOME are set correctly for the Jenkins environment.""" 888 env = fake_environment(set_jenkins_home=True, set_workspace=True) 889 cwd = '/fake/random-location' 890 old_home = env[bootstrap.HOME_ENV] 891 old_workspace = env[bootstrap.WORKSPACE_ENV] 892 with Stub(os, 'environ', env): 893 with Stub(os, 'getcwd', lambda: cwd): 894 bootstrap.setup_magic_environment(JOB) 895 896 self.assertIn(bootstrap.WORKSPACE_ENV, env) 897 self.assertNotEquals(env[bootstrap.HOME_ENV], 898 env[bootstrap.WORKSPACE_ENV]) 899 self.assertNotEquals(old_home, env[bootstrap.HOME_ENV]) 900 self.assertEquals(cwd, env[bootstrap.HOME_ENV]) 901 self.assertEquals(old_workspace, env[bootstrap.WORKSPACE_ENV]) 902 self.assertNotEquals(cwd, env[bootstrap.WORKSPACE_ENV]) 903 904 def test_home_workspace_in_k8s(self): 905 """WORKSPACE/HOME are set correctly for the kubernetes environment.""" 906 env = fake_environment(set_jenkins_home=False, set_workspace=True) 907 cwd = '/fake/random-location' 908 old_home = env[bootstrap.HOME_ENV] 909 old_workspace = env[bootstrap.WORKSPACE_ENV] 910 with Stub(os, 'environ', env): 911 with Stub(os, 'getcwd', lambda: cwd): 912 bootstrap.setup_magic_environment(JOB) 913 914 self.assertIn(bootstrap.WORKSPACE_ENV, env) 915 self.assertNotEquals(env[bootstrap.HOME_ENV], 916 env[bootstrap.WORKSPACE_ENV]) 917 self.assertEquals(old_home, env[bootstrap.HOME_ENV]) 918 self.assertNotEquals(cwd, env[bootstrap.HOME_ENV]) 919 self.assertEquals(old_workspace, env[bootstrap.WORKSPACE_ENV]) 920 self.assertNotEquals(cwd, env[bootstrap.WORKSPACE_ENV]) 921 922 def test_workspace_always_set(self): 923 """WORKSPACE is set to cwd when unset in initial environment.""" 924 env = fake_environment(set_workspace=False) 925 cwd = '/fake/random-location' 926 with Stub(os, 'environ', env): 927 with Stub(os, 'getcwd', lambda: cwd): 928 bootstrap.setup_magic_environment(JOB) 929 930 self.assertIn(bootstrap.WORKSPACE_ENV, env) 931 self.assertEquals(cwd, env[bootstrap.HOME_ENV]) 932 self.assertEquals(cwd, env[bootstrap.WORKSPACE_ENV]) 933 934 def test_job_env_mismatch(self): 935 env = fake_environment() 936 with Stub(os, 'environ', env): 937 self.assertNotEquals('this-is-a-job', env[bootstrap.JOB_ENV]) 938 bootstrap.setup_magic_environment('this-is-a-job') 939 self.assertEquals('this-is-a-job', env[bootstrap.JOB_ENV]) 940 941 def test_expected(self): 942 env = fake_environment() 943 del env[bootstrap.JOB_ENV] 944 del env[bootstrap.NODE_ENV] 945 with Stub(os, 'environ', env): 946 bootstrap.setup_magic_environment(JOB) 947 948 def check(name): 949 self.assertIn(name, env) 950 951 # Some of these are probably silly to check... 952 # TODO(fejta): remove as many of these from our infra as possible. 953 check(bootstrap.JOB_ENV) 954 check(bootstrap.CLOUDSDK_ENV) 955 check(bootstrap.BOOTSTRAP_ENV) 956 check(bootstrap.WORKSPACE_ENV) 957 self.assertNotIn(bootstrap.SERVICE_ACCOUNT_ENV, env) 958 959 def test_node_present(self): 960 expected = 'whatever' 961 env = {bootstrap.NODE_ENV: expected} 962 with Stub(os, 'environ', env): 963 self.assertEquals(expected, bootstrap.node()) 964 self.assertEquals(expected, env[bootstrap.NODE_ENV]) 965 966 def test_node_missing(self): 967 env = {} 968 with Stub(os, 'environ', env): 969 expected = bootstrap.node() 970 self.assertTrue(expected) 971 self.assertEquals(expected, env[bootstrap.NODE_ENV]) 972 973 974 975 def test_cloud_sdk_config(self): 976 cwd = 'now-here' 977 env = fake_environment() 978 with Stub(os, 'environ', env): 979 with Stub(os, 'getcwd', lambda: cwd): 980 bootstrap.setup_magic_environment(JOB) 981 982 983 self.assertTrue(env[bootstrap.CLOUDSDK_ENV].startswith(cwd)) 984 985 986 class FakePath(object): 987 artifacts = 'fake_artifacts' 988 pr_latest = 'fake_pr_latest' 989 pr_build_link = 'fake_pr_link' 990 build_log = 'fake_log_path' 991 pr_path = 'fake_pr_path' 992 pr_result_cache = 'fake_pr_result_cache' 993 latest = 'fake_latest' 994 result_cache = 'fake_result_cache' 995 started = 'fake_started.json' 996 finished = 'fake_finished.json' 997 def __call__(self, *arg, **kw): 998 self.arg = arg 999 self.kw = kw 1000 return self 1001 1002 1003 class FakeLogging(object): 1004 close = Pass 1005 def __call__(self, *_a, **_kw): 1006 return self 1007 1008 1009 class FakeFinish(object): 1010 called = False 1011 result = None 1012 def __call__(self, unused_a, unused_b, success, *a, **kw): 1013 self.called = True 1014 self.result = success 1015 1016 def repo(config): 1017 repos = bootstrap.Repos() 1018 for cur_repo, tup in config.items(): 1019 repos[cur_repo] = tup 1020 return repos 1021 1022 class PRPathsTest(unittest.TestCase): 1023 def test_kubernetes_kubernetes(self): 1024 """Test the kubernetes/kubernetes prefix.""" 1025 path = bootstrap.pr_paths(UPLOAD, repo({'kubernetes/kubernetes': ('', PULL)}), JOB, BUILD) 1026 self.assertTrue(any( 1027 str(PULL) == p for p in path.build_log.split('/'))) 1028 1029 def test_kubernetes(self): 1030 """Test the kubernetes/something prefix.""" 1031 path = bootstrap.pr_paths(UPLOAD, repo({'kubernetes/prefix': ('', PULL)}), JOB, BUILD) 1032 self.assertTrue(any( 1033 'prefix' in p for p in path.build_log.split('/')), path.build_log) 1034 self.assertTrue(any( 1035 str(PULL) in p for p in path.build_log.split('/')), path.build_log) 1036 1037 def test_other(self): 1038 """Test the none kubernetes prefixes.""" 1039 path = bootstrap.pr_paths(UPLOAD, repo({'github.com/random/repo': ('', PULL)}), JOB, BUILD) 1040 self.assertTrue(any( 1041 'random_repo' in p for p in path.build_log.split('/')), path.build_log) 1042 self.assertTrue(any( 1043 str(PULL) in p for p in path.build_log.split('/')), path.build_log) 1044 1045 1046 class FakeArgs(object): 1047 bare = False 1048 clean = False 1049 git_cache = '' 1050 job = JOB 1051 root = ROOT 1052 service_account = ROBOT 1053 ssh = False 1054 timeout = 0 1055 upload = UPLOAD 1056 json = False 1057 1058 def __init__(self, **kw): 1059 self.branch = BRANCH 1060 self.pull = PULL 1061 self.repo = [REPO] 1062 for key, val in kw.items(): 1063 if not hasattr(self, key): 1064 raise AttributeError(self, key) 1065 setattr(self, key, val) 1066 1067 1068 def test_bootstrap(**kw): 1069 if isinstance(kw.get('repo'), basestring): 1070 kw['repo'] = [kw['repo']] 1071 return bootstrap.bootstrap(FakeArgs(**kw)) 1072 1073 class BootstrapTest(unittest.TestCase): 1074 1075 def setUp(self): 1076 self.boiler = [ 1077 Stub(bootstrap, 'checkout', Pass), 1078 Stub(bootstrap, 'finish', Pass), 1079 Stub(bootstrap.GSUtil, 'copy_file', Pass), 1080 Stub(bootstrap, 'node', lambda: 'fake-node'), 1081 Stub(bootstrap, 'setup_credentials', Pass), 1082 Stub(bootstrap, 'setup_logging', FakeLogging()), 1083 Stub(bootstrap, 'start', Pass), 1084 Stub(bootstrap, '_call', Pass), 1085 Stub(os, 'environ', fake_environment()), 1086 Stub(os, 'chdir', Pass), 1087 Stub(os, 'makedirs', Pass), 1088 ] 1089 1090 def tearDown(self): 1091 for stub in self.boiler: 1092 with stub: # Leaving with restores things 1093 pass 1094 1095 def test_setcreds_setroot_fails(self): 1096 """We should still call setup_credentials even if setup_root blows up.""" 1097 called = set() 1098 with Stub(bootstrap, 'setup_root', Bomb): 1099 with Stub(bootstrap, 'setup_credentials', 1100 lambda *a, **kw: called.add('setup_credentials')): 1101 with Stub(bootstrap, 'finish', lambda *a, **kw: called.add('finish')): 1102 with self.assertRaises(AssertionError): 1103 test_bootstrap() 1104 1105 for needed in ['setup_credentials', 'finish']: 1106 self.assertIn(needed, called) 1107 1108 def test_empty_repo(self): 1109 repo_name = None 1110 with Stub(bootstrap, 'checkout', Bomb): 1111 test_bootstrap(repo=repo_name, branch=None, pull=None, bare=True) 1112 with self.assertRaises(ValueError): 1113 test_bootstrap(repo=repo_name, branch=None, pull=PULL) 1114 with self.assertRaises(ValueError): 1115 test_bootstrap(repo=repo_name, branch=BRANCH, pull=None) 1116 1117 def test_root_not_exists(self): 1118 with Stub(os, 'chdir', FakeCall()) as fake_chdir: 1119 with Stub(os.path, 'exists', lambda p: False): 1120 with Stub(os, 'makedirs', FakeCall()) as fake_makedirs: 1121 test_bootstrap(branch=None) 1122 self.assertTrue(any(ROOT in c[0] for c in fake_chdir.calls), fake_chdir.calls) 1123 self.assertTrue(any(ROOT in c[0] for c in fake_makedirs.calls), fake_makedirs.calls) 1124 1125 def test_root_exists(self): 1126 with Stub(os, 'chdir', FakeCall()) as fake_chdir: 1127 test_bootstrap(branch=None) 1128 self.assertTrue(any(ROOT in c[0] for c in fake_chdir.calls)) 1129 1130 def test_pr_paths(self): 1131 """Use a pr_paths when pull is set.""" 1132 1133 with Stub(bootstrap, 'ci_paths', Bomb): 1134 with Stub(bootstrap, 'pr_paths', FakePath()) as path: 1135 test_bootstrap(branch=None, pull=PULL, root='.') 1136 self.assertEquals(PULL, path.arg[1][REPO][1], (PULL, path.arg)) 1137 1138 def test_ci_paths(self): 1139 """Use a ci_paths when branch is set.""" 1140 1141 with Stub(bootstrap, 'pr_paths', Bomb): 1142 with Stub(bootstrap, 'ci_paths', FakePath()) as path: 1143 test_bootstrap(pull=None, branch=BRANCH) 1144 self.assertFalse(any( 1145 PULL in o for o in (path.arg, path.kw))) 1146 1147 def test_finish_when_start_fails(self): 1148 """Finish is called even if start fails.""" 1149 with Stub(bootstrap, 'finish', FakeFinish()) as fake: 1150 with Stub(bootstrap, 'start', Bomb): 1151 with self.assertRaises(AssertionError): 1152 test_bootstrap(pull=None) 1153 self.assertTrue(fake.called) 1154 self.assertTrue(fake.result is False) # Distinguish from None 1155 1156 def test_finish_when_build_fails(self): 1157 """Finish is called even if the build fails.""" 1158 def call_error(*_a, **_kw): 1159 raise subprocess.CalledProcessError(1, [], '') 1160 with Stub(bootstrap, 'finish', FakeFinish()) as fake: 1161 with Stub(bootstrap, '_call', call_error): 1162 with self.assertRaises(SystemExit): 1163 test_bootstrap(pull=None) 1164 self.assertTrue(fake.called) 1165 self.assertTrue(fake.result is False) # Distinguish from None 1166 1167 def test_happy(self): 1168 with Stub(bootstrap, 'finish', FakeFinish()) as fake: 1169 test_bootstrap(pull=None) 1170 self.assertTrue(fake.called) 1171 self.assertTrue(fake.result) # Distinguish from None 1172 1173 def test_job_env(self): 1174 """bootstrap sets JOB_NAME.""" 1175 with Stub(os, 'environ', fake_environment()) as env: 1176 test_bootstrap(pull=None) 1177 self.assertIn(bootstrap.JOB_ENV, env) 1178 1179 def test_job_script_expands_vars(self): 1180 fake = { 1181 'HELLO': 'awesome', 1182 'WORLD': 'sauce', 1183 } 1184 with Stub(os, 'environ', fake): 1185 actual = bootstrap.job_args( 1186 ['$HELLO ${WORLD}', 'happy', '${MISSING}']) 1187 self.assertEquals(['awesome sauce', 'happy', '${MISSING}'], actual) 1188 1189 1190 class RepositoryTest(unittest.TestCase): 1191 def test_kubernetes_kubernetes(self): 1192 expected = 'https://github.com/kubernetes/kubernetes' 1193 actual = bootstrap.repository('k8s.io/kubernetes', '') 1194 self.assertEquals(expected, actual) 1195 1196 def test_kubernetes_testinfra(self): 1197 expected = 'https://github.com/kubernetes/test-infra' 1198 actual = bootstrap.repository('k8s.io/test-infra', '') 1199 self.assertEquals(expected, actual) 1200 1201 def test_whatever(self): 1202 expected = 'https://foo.com/bar' 1203 actual = bootstrap.repository('foo.com/bar', '') 1204 self.assertEquals(expected, actual) 1205 1206 def test_k8s_k8s_ssh(self): 1207 expected = 'git@github.com:kubernetes/kubernetes' 1208 actual = bootstrap.repository('k8s.io/kubernetes', 'path') 1209 self.assertEquals(expected, actual) 1210 1211 def test_k8s_k8s_ssh_with_colon(self): 1212 expected = 'git@github.com:kubernetes/kubernetes' 1213 actual = bootstrap.repository('github.com:kubernetes/kubernetes', 'path') 1214 self.assertEquals(expected, actual) 1215 1216 def test_whatever_ssh(self): 1217 expected = 'git@foo.com:bar' 1218 actual = bootstrap.repository('foo.com/bar', 'path') 1219 self.assertEquals(expected, actual) 1220 1221 1222 1223 class IntegrationTest(unittest.TestCase): 1224 REPO = 'hello/world' 1225 MASTER = 'fake-master-file' 1226 BRANCH_FILE = 'fake-branch-file' 1227 PR_FILE = 'fake-pr-file' 1228 BRANCH = 'another-branch' 1229 PR_NUM = 42 1230 PR_TAG = bootstrap.pull_ref(PR_NUM)[0][0].strip('+') 1231 1232 def fake_repo(self, fake, _ssh=False): 1233 return os.path.join(self.root_github, fake) 1234 1235 def setUp(self): 1236 self.boiler = [ 1237 Stub(bootstrap, 'finish', Pass), 1238 Stub(bootstrap.GSUtil, 'copy_file', Pass), 1239 Stub(bootstrap, 'repository', self.fake_repo), 1240 Stub(bootstrap, 'setup_credentials', Pass), 1241 Stub(bootstrap, 'setup_logging', FakeLogging()), 1242 Stub(bootstrap, 'start', Pass), 1243 Stub(os, 'environ', fake_environment(set_job=False)), 1244 ] 1245 self.root_github = tempfile.mkdtemp() 1246 self.root_workspace = tempfile.mkdtemp() 1247 self.root_git_cache = tempfile.mkdtemp() 1248 self.ocwd = os.getcwd() 1249 fakerepo = self.fake_repo(self.REPO) 1250 subprocess.check_call(['git', 'init', fakerepo]) 1251 os.chdir(fakerepo) 1252 subprocess.check_call(['git', 'config', 'user.name', 'foo']) 1253 subprocess.check_call(['git', 'config', 'user.email', 'foo@bar.baz']) 1254 subprocess.check_call(['touch', self.MASTER]) 1255 subprocess.check_call(['git', 'add', self.MASTER]) 1256 subprocess.check_call(['cp', '-r', bootstrap.test_infra('jenkins/fake'), fakerepo]) 1257 subprocess.check_call(['git', 'add', 'fake']) 1258 subprocess.check_call(['git', 'commit', '-m', 'Initial commit']) 1259 subprocess.check_call(['git', 'checkout', 'master']) 1260 1261 def tearDown(self): 1262 for stub in self.boiler: 1263 with stub: # Leaving with restores things 1264 pass 1265 os.chdir(self.ocwd) 1266 subprocess.check_call(['rm', '-rf', self.root_github]) 1267 subprocess.check_call(['rm', '-rf', self.root_workspace]) 1268 subprocess.check_call(['rm', '-rf', self.root_git_cache]) 1269 1270 def test_git_cache(self): 1271 subprocess.check_call(['git', 'checkout', '-b', self.BRANCH]) 1272 subprocess.check_call(['git', 'rm', self.MASTER]) 1273 subprocess.check_call(['touch', self.BRANCH_FILE]) 1274 subprocess.check_call(['git', 'add', self.BRANCH_FILE]) 1275 subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH]) 1276 test_bootstrap( 1277 job='fake-branch', 1278 repo=self.REPO, 1279 branch=self.BRANCH, 1280 pull=None, 1281 root=self.root_workspace, 1282 git_cache=self.root_git_cache) 1283 # Verify that the cache was populated by running a simple git command 1284 # in the git cache directory. 1285 subprocess.check_call( 1286 ['git', '--git-dir=%s/%s' % (self.root_git_cache, self.REPO), 'log']) 1287 1288 def test_pr(self): 1289 subprocess.check_call(['git', 'checkout', 'master']) 1290 subprocess.check_call(['git', 'checkout', '-b', 'unknown-pr-branch']) 1291 subprocess.check_call(['git', 'rm', self.MASTER]) 1292 subprocess.check_call(['touch', self.PR_FILE]) 1293 subprocess.check_call(['git', 'add', self.PR_FILE]) 1294 subprocess.check_call(['git', 'commit', '-m', 'Create branch for PR %d' % self.PR_NUM]) 1295 subprocess.check_call(['git', 'tag', self.PR_TAG]) 1296 os.chdir('/tmp') 1297 test_bootstrap( 1298 job='fake-pr', 1299 repo=self.REPO, 1300 branch=None, 1301 pull=self.PR_NUM, 1302 root=self.root_workspace) 1303 1304 def test_branch(self): 1305 subprocess.check_call(['git', 'checkout', '-b', self.BRANCH]) 1306 subprocess.check_call(['git', 'rm', self.MASTER]) 1307 subprocess.check_call(['touch', self.BRANCH_FILE]) 1308 subprocess.check_call(['git', 'add', self.BRANCH_FILE]) 1309 subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH]) 1310 1311 os.chdir('/tmp') 1312 test_bootstrap( 1313 job='fake-branch', 1314 repo=self.REPO, 1315 branch=self.BRANCH, 1316 pull=None, 1317 root=self.root_workspace) 1318 1319 def test_branch_ref(self): 1320 """Make sure we check out a specific commit.""" 1321 subprocess.check_call(['git', 'checkout', '-b', self.BRANCH]) 1322 subprocess.check_call(['git', 'rm', self.MASTER]) 1323 subprocess.check_call(['touch', self.BRANCH_FILE]) 1324 subprocess.check_call(['git', 'add', self.BRANCH_FILE]) 1325 subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH]) 1326 sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() 1327 subprocess.check_call(['rm', self.BRANCH_FILE]) 1328 subprocess.check_call(['git', 'add', self.BRANCH_FILE]) 1329 subprocess.check_call(['git', 'commit', '-m', 'Delete %s' % self.BRANCH]) 1330 1331 os.chdir('/tmp') 1332 # Supplying the commit exactly works. 1333 test_bootstrap( 1334 job='fake-branch', 1335 repo=self.REPO, 1336 branch='%s:%s' % (self.BRANCH, sha), 1337 pull=None, 1338 root=self.root_workspace) 1339 # Supplying the commit through repo works. 1340 test_bootstrap( 1341 job='fake-branch', 1342 repo="%s=%s:%s" % (self.REPO, self.BRANCH, sha), 1343 branch=None, 1344 pull=None, 1345 root=self.root_workspace) 1346 # Using branch head fails. 1347 with self.assertRaises(SystemExit): 1348 test_bootstrap( 1349 job='fake-branch', 1350 repo=self.REPO, 1351 branch=self.BRANCH, 1352 pull=None, 1353 root=self.root_workspace) 1354 with self.assertRaises(SystemExit): 1355 test_bootstrap( 1356 job='fake-branch', 1357 repo="%s=%s" % (self.REPO, self.BRANCH), 1358 branch=None, 1359 pull=None, 1360 root=self.root_workspace) 1361 1362 def test_batch(self): 1363 def head_sha(): 1364 # We can't hardcode the SHAs for the test, so we need to determine 1365 # them after each commit. 1366 return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() 1367 refs = ['master:%s' % head_sha()] 1368 for pr in (123, 456): 1369 subprocess.check_call(['git', 'checkout', '-b', 'refs/pull/%d/head' % pr, 'master']) 1370 subprocess.check_call(['git', 'rm', self.MASTER]) 1371 subprocess.check_call(['touch', self.PR_FILE]) 1372 subprocess.check_call(['git', 'add', self.PR_FILE]) 1373 open('pr_%d.txt' % pr, 'w').write('some text') 1374 subprocess.check_call(['git', 'add', 'pr_%d.txt' % pr]) 1375 subprocess.check_call(['git', 'commit', '-m', 'add some stuff (#%d)' % pr]) 1376 refs.append('%d:%s' % (pr, head_sha())) 1377 os.chdir('/tmp') 1378 pull = ','.join(refs) 1379 print '--pull', pull 1380 subprocess.check_call(['ls']) 1381 test_bootstrap( 1382 job='fake-pr', 1383 repo=self.REPO, 1384 branch=None, 1385 pull=pull, 1386 root=self.root_workspace) 1387 1388 def test_pr_bad(self): 1389 random_pr = 111 1390 with Stub(bootstrap, 'start', Bomb): 1391 with Stub(time, 'sleep', Pass): 1392 with self.assertRaises(subprocess.CalledProcessError): 1393 test_bootstrap( 1394 job='fake-pr', 1395 repo=self.REPO, 1396 branch=None, 1397 pull=random_pr, 1398 root=self.root_workspace) 1399 1400 def test_branch_bad(self): 1401 random_branch = 'something' 1402 with Stub(bootstrap, 'start', Bomb): 1403 with Stub(time, 'sleep', Pass): 1404 with self.assertRaises(subprocess.CalledProcessError): 1405 test_bootstrap( 1406 job='fake-branch', 1407 repo=self.REPO, 1408 branch=random_branch, 1409 pull=None, 1410 root=self.root_workspace) 1411 1412 def test_job_missing(self): 1413 with self.assertRaises(KeyError): 1414 test_bootstrap( 1415 job='this-job-no-exists', 1416 repo=self.REPO, 1417 branch='master', 1418 pull=None, 1419 root=self.root_workspace) 1420 1421 def test_job_fails(self): 1422 with self.assertRaises(SystemExit): 1423 test_bootstrap( 1424 job='fake-failure', 1425 repo=self.REPO, 1426 branch='master', 1427 pull=None, 1428 root=self.root_workspace) 1429 1430 def test_commit_in_meta(self): 1431 sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() 1432 cwd = os.getcwd() 1433 os.chdir(bootstrap.test_infra('.')) 1434 infra_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()[:9] 1435 os.chdir(cwd) 1436 1437 # Commit SHA should in meta 1438 call = lambda *a, **kw: bootstrap._call(5, *a, **kw) 1439 repos = repo({REPO: ('master', ''), 'other-repo': ('other-branch', '')}) 1440 meta = bootstrap.metadata(repos, 'missing-artifacts-dir', call) 1441 self.assertIn('repo-commit', meta) 1442 self.assertEquals(sha, meta['repo-commit']) 1443 self.assertEquals(40, len(meta['repo-commit'])) 1444 self.assertIn('infra-commit', meta) 1445 self.assertEquals(infra_sha, meta['infra-commit']) 1446 self.assertEquals(9, len(meta['infra-commit'])) 1447 self.assertIn(REPO, meta.get('repos')) 1448 self.assertIn('other-repo', meta.get('repos')) 1449 self.assertEquals(REPO, meta.get('repo')) 1450 1451 1452 class ParseArgsTest(unittest.TestCase): 1453 def test_barerepo_both(self): 1454 with self.assertRaises(argparse.ArgumentTypeError): 1455 bootstrap.parse_args(['--bare', '--repo=hello', '--job=j']) 1456 1457 def test_barerepo_neither(self): 1458 with self.assertRaises(argparse.ArgumentTypeError): 1459 bootstrap.parse_args(['--job=j']) 1460 1461 def test_barerepo_bareonly(self): 1462 args = bootstrap.parse_args(['--bare', '--job=j']) 1463 self.assertFalse(args.repo, args) 1464 self.assertTrue(args.bare, args) 1465 1466 def test_barerepo_repoonly(self): 1467 args = bootstrap.parse_args(['--repo=R', '--job=j']) 1468 self.assertFalse(args.bare, args) 1469 self.assertTrue(args.repo, args) 1470 1471 1472 if __name__ == '__main__': 1473 unittest.main()