github.com/abayer/test-infra@v0.0.5/jenkins/bootstrap_test.py (about) 1 #!/usr/bin/env python 2 3 # Copyright 2016 The Kubernetes Authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Tests for bootstrap.""" 18 19 # pylint: disable=protected-access, attribute-defined-outside-init 20 21 import argparse 22 import json 23 import os 24 import select 25 import signal 26 import subprocess 27 import tempfile 28 import time 29 import unittest 30 31 import bootstrap 32 33 34 BRANCH = 'random_branch' 35 BUILD = 'random_build' 36 FAIL = ['/bin/bash', '-c', 'exit 1'] 37 JOB = 'random_job' 38 PASS = ['/bin/bash', '-c', 'exit 0'] 39 PULL = 12345 40 REPO = 'github.com/random_org/random_repo' 41 ROBOT = 'fake-service-account.json' 42 ROOT = '/random/root' 43 UPLOAD = 'fake-gs://fake-bucket' 44 45 46 class Stub(object): 47 """Replace thing.param with replacement until exiting with.""" 48 def __init__(self, thing, param, replacement): 49 self.thing = thing 50 self.param = param 51 self.replacement = replacement 52 self.old = getattr(thing, param) 53 setattr(thing, param, self.replacement) 54 55 def __enter__(self, *a, **kw): 56 return self.replacement 57 58 def __exit__(self, *a, **kw): 59 setattr(self.thing, self.param, self.old) 60 61 62 class FakeCall(object): 63 def __init__(self): 64 self.calls = [] 65 66 def __call__(self, *a, **kw): 67 self.calls.append((a, kw)) 68 69 70 class FakeSubprocess(object): 71 """Keep track of calls.""" 72 def __init__(self): 73 self.calls = [] 74 self.file_data = [] 75 self.output = {} 76 77 def __call__(self, cmd, *a, **kw): 78 self.calls.append((cmd, a, kw)) 79 for arg in cmd: 80 if arg.startswith('/') and os.path.exists(arg): 81 self.file_data.append(open(arg).read()) 82 if kw.get('output') and self.output.get(cmd[0]): 83 return self.output[cmd[0]].pop(0) 84 85 86 # pylint: disable=invalid-name 87 def Pass(*_a, **_kw): 88 """Do nothing.""" 89 pass 90 91 92 def Truth(*_a, **_kw): 93 """Always true.""" 94 return True 95 96 97 def Bomb(*a, **kw): 98 """Always raise.""" 99 raise AssertionError('Should not happen', a, kw) 100 # pylint: enable=invalid-name 101 102 103 class ReadAllTest(unittest.TestCase): 104 endless = 0 105 ended = time.time() - 50 106 number = 0 107 end = -1 108 109 def fileno(self): 110 return self.end 111 112 def readline(self): 113 line = 'line %d\n' % self.number 114 self.number += 1 115 return line 116 117 def test_read_more(self): 118 """Read lines until we clear the buffer, noting there may be more.""" 119 lines = [] 120 total = 10 121 def more_lines(*_a, **_kw): 122 if len(lines) < total: 123 return [self], [], [] 124 return [], [], [] 125 with Stub(select, 'select', more_lines): 126 done = bootstrap.read_all(self.endless, self, lines.append) 127 128 self.assertFalse(done) 129 self.assertEquals(total, len(lines)) 130 expected = ['line %d' % d for d in range(total)] 131 self.assertEquals(expected, lines) 132 133 def test_read_expired(self): 134 """Read nothing as we are expired, noting there may be more.""" 135 lines = [] 136 with Stub(select, 'select', lambda *a, **kw: ([], [], [])): 137 done = bootstrap.read_all(self.ended, self, lines.append) 138 139 self.assertFalse(done) 140 self.assertFalse(lines) 141 142 def test_read_end(self): 143 """Note we reached the end of the stream.""" 144 lines = [] 145 with Stub(select, 'select', lambda *a, **kw: ([self], [], [])): 146 with Stub(self, 'readline', lambda: ''): 147 done = bootstrap.read_all(self.endless, self, lines.append) 148 149 self.assertTrue(done) 150 151 152 class TerminateTest(unittest.TestCase): 153 """Tests for termiante().""" 154 pid = 1234 155 pgid = 5555 156 terminated = False 157 killed = False 158 159 def terminate(self): 160 self.terminated = True 161 162 def kill(self): 163 self.killed = True 164 165 def getpgid(self, pid): 166 self.got = pid 167 return self.pgid 168 169 def killpg(self, pgig, sig): 170 self.killed_pg = (pgig, sig) 171 172 def test_terminate_later(self): 173 """Do nothing if end is in the future.""" 174 timeout = bootstrap.terminate(time.time() + 50, self, False) 175 self.assertFalse(timeout) 176 177 def test_terminate_never(self): 178 """Do nothing if end is zero.""" 179 timeout = bootstrap.terminate(0, self, False) 180 self.assertFalse(timeout) 181 182 def test_terminate_terminate(self): 183 """Terminate pid if after end and kill is false.""" 184 timeout = bootstrap.terminate(time.time() - 50, self, False) 185 self.assertTrue(timeout) 186 self.assertFalse(self.killed) 187 self.assertTrue(self.terminated) 188 189 def test_terminate_kill(self): 190 """Kill process group if after end and kill is true.""" 191 with Stub(os, 'getpgid', self.getpgid), Stub(os, 'killpg', self.killpg): 192 timeout = bootstrap.terminate(time.time() - 50, self, True) 193 self.assertTrue(timeout) 194 self.assertFalse(self.terminated) 195 self.assertTrue(self.killed) 196 self.assertEquals(self.pid, self.got) 197 self.assertEquals(self.killed_pg, (self.pgid, signal.SIGKILL)) 198 199 200 class SubprocessTest(unittest.TestCase): 201 """Tests for call().""" 202 203 def test_stdin(self): 204 """Will write to subprocess.stdin.""" 205 with self.assertRaises(subprocess.CalledProcessError) as cpe: 206 bootstrap._call(0, ['/bin/bash'], stdin='exit 92') 207 self.assertEquals(92, cpe.exception.returncode) 208 209 def test_check_true(self): 210 """Raise on non-zero exit codes if check is set.""" 211 with self.assertRaises(subprocess.CalledProcessError): 212 bootstrap._call(0, FAIL, check=True) 213 214 bootstrap._call(0, PASS, check=True) 215 216 def test_check_default(self): 217 """Default to check=True.""" 218 with self.assertRaises(subprocess.CalledProcessError): 219 bootstrap._call(0, FAIL) 220 221 bootstrap._call(0, PASS) 222 223 @staticmethod 224 def test_check_false(): 225 """Never raise when check is not set.""" 226 bootstrap._call(0, FAIL, check=False) 227 bootstrap._call(0, PASS, check=False) 228 229 def test_output(self): 230 """Output is returned when requested.""" 231 cmd = ['/bin/bash', '-c', 'echo hello world'] 232 self.assertEquals( 233 'hello world\n', bootstrap._call(0, cmd, output=True)) 234 235 def test_zombie(self): 236 with self.assertRaises(subprocess.CalledProcessError): 237 # make a zombie 238 bootstrap._call(0, ['/bin/bash', '-c', 'A=$BASHPID && ( kill -STOP $A ) & exit 1']) 239 240 241 class PullRefsTest(unittest.TestCase): 242 """Tests for pull_ref, branch_ref, ref_has_shas, and pull_numbers.""" 243 244 def test_multiple_no_shas(self): 245 """Test master,1111,2222.""" 246 self.assertEqual( 247 bootstrap.pull_ref('master,123,456'), 248 ([ 249 'master', 250 '+refs/pull/123/head:refs/pr/123', 251 '+refs/pull/456/head:refs/pr/456', 252 ], [ 253 'FETCH_HEAD', 254 'refs/pr/123', 255 'refs/pr/456', 256 ]), 257 ) 258 259 def test_pull_has_shas(self): 260 self.assertTrue(bootstrap.ref_has_shas('master:abcd')) 261 self.assertFalse(bootstrap.ref_has_shas('123')) 262 self.assertFalse(bootstrap.ref_has_shas(123)) 263 self.assertFalse(bootstrap.ref_has_shas(None)) 264 265 def test_pull_numbers(self): 266 self.assertListEqual(bootstrap.pull_numbers(123), ['123']) 267 self.assertListEqual(bootstrap.pull_numbers('master:abcd'), []) 268 self.assertListEqual( 269 bootstrap.pull_numbers('master:abcd,123:qwer,124:zxcv'), 270 ['123', '124']) 271 272 def test_pull_ref(self): 273 self.assertEqual( 274 bootstrap.pull_ref('master:abcd,123:effe'), 275 (['master', '+refs/pull/123/head:refs/pr/123'], ['abcd', 'effe']) 276 ) 277 self.assertEqual( 278 bootstrap.pull_ref('123'), 279 (['+refs/pull/123/merge'], ['FETCH_HEAD']) 280 ) 281 282 def test_branch_ref(self): 283 self.assertEqual( 284 bootstrap.branch_ref('branch:abcd'), 285 (['branch'], ['abcd']) 286 ) 287 self.assertEqual( 288 bootstrap.branch_ref('master'), 289 (['master'], ['FETCH_HEAD']) 290 ) 291 292 293 class ConfigureSshKeyTest(unittest.TestCase): 294 """Tests for configure_ssh_key().""" 295 def test_empty(self): 296 """Do not change environ if no ssh key.""" 297 fake_env = {} 298 with Stub(os, 'environ', fake_env): 299 with bootstrap.configure_ssh_key(''): 300 self.assertFalse(fake_env) 301 302 def test_full(self): 303 fake_env = {} 304 with Stub(os, 'environ', fake_env): 305 with bootstrap.configure_ssh_key('hello there'): 306 self.assertIn('GIT_SSH', fake_env) 307 with open(fake_env['GIT_SSH']) as fp: 308 buf = fp.read() 309 self.assertIn('hello there', buf) 310 self.assertIn('ssh ', buf) 311 self.assertIn(' -i ', buf) 312 self.assertFalse(fake_env) # Resets env 313 314 def test_full_old_value(self): 315 fake_env = {'GIT_SSH': 'random-value'} 316 old_env = dict(fake_env) 317 with Stub(os, 'environ', fake_env): 318 with bootstrap.configure_ssh_key('hello there'): 319 self.assertNotEqual(old_env, fake_env) 320 self.assertEquals(old_env, fake_env) 321 322 323 class CheckoutTest(unittest.TestCase): 324 """Tests for checkout().""" 325 326 def test_clean(self): 327 """checkout cleans and resets if asked to.""" 328 fake = FakeSubprocess() 329 with Stub(os, 'chdir', Pass): 330 bootstrap.checkout(fake, REPO, REPO, None, PULL, clean=True) 331 332 self.assertTrue(any( 333 'clean' in cmd for cmd, _, _ in fake.calls if 'git' in cmd)) 334 self.assertTrue(any( 335 'reset' in cmd for cmd, _, _ in fake.calls if 'git' in cmd)) 336 337 def test_fetch_retries(self): 338 self.tries = 0 339 expected_attempts = 3 340 def third_time_charm(cmd, *_a, **_kw): 341 if 'fetch' not in cmd: # init/checkout are unlikely to fail 342 return 343 self.tries += 1 344 if self.tries != expected_attempts: 345 raise subprocess.CalledProcessError(128, cmd, None) 346 with Stub(os, 'chdir', Pass): 347 with Stub(time, 'sleep', Pass): 348 bootstrap.checkout(third_time_charm, REPO, REPO, None, PULL) 349 self.assertEquals(expected_attempts, self.tries) 350 351 def test_pull_ref(self): 352 """checkout fetches the right ref for a pull.""" 353 fake = FakeSubprocess() 354 with Stub(os, 'chdir', Pass): 355 bootstrap.checkout(fake, REPO, REPO, None, PULL) 356 357 expected_ref = bootstrap.pull_ref(PULL)[0][0] 358 self.assertTrue(any( 359 expected_ref in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd)) 360 361 def test_branch(self): 362 """checkout fetches the right ref for a branch.""" 363 fake = FakeSubprocess() 364 with Stub(os, 'chdir', Pass): 365 bootstrap.checkout(fake, REPO, REPO, BRANCH, None) 366 367 expected_ref = BRANCH 368 self.assertTrue(any( 369 expected_ref in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd)) 370 371 def test_repo(self): 372 """checkout initializes and fetches the right repo.""" 373 fake = FakeSubprocess() 374 with Stub(os, 'chdir', Pass): 375 bootstrap.checkout(fake, REPO, REPO, BRANCH, None) 376 377 expected_uri = 'https://%s' % REPO 378 self.assertTrue(any( 379 expected_uri in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd)) 380 381 def test_branch_xor_pull(self): 382 """Either branch or pull specified, not both.""" 383 with Stub(os, 'chdir', Bomb): 384 with self.assertRaises(ValueError): 385 bootstrap.checkout(Bomb, REPO, REPO, None, None) 386 with self.assertRaises(ValueError): 387 bootstrap.checkout(Bomb, REPO, REPO, BRANCH, PULL) 388 389 def test_happy(self): 390 """checkout sanity check.""" 391 fake = FakeSubprocess() 392 with Stub(os, 'chdir', Pass): 393 bootstrap.checkout(fake, REPO, REPO, BRANCH, None) 394 395 self.assertTrue(any( 396 '--tags' in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd)) 397 self.assertTrue(any( 398 'FETCH_HEAD' in cmd for cmd, _, _ in fake.calls 399 if 'checkout' in cmd)) 400 401 def test_repo_path(self): 402 """checkout repo to different local path.""" 403 fake = FakeSubprocess() 404 repo_path = "foo/bar" 405 with Stub(os, 'chdir', Pass): 406 bootstrap.checkout(fake, REPO, repo_path, BRANCH, None) 407 408 expected_uri = 'https://%s' % REPO 409 self.assertTrue(any( 410 expected_uri in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd)) 411 412 self.assertTrue(any( 413 repo_path in cmd for cmd, _, _ in fake.calls if 'init' in cmd)) 414 415 class ParseReposTest(unittest.TestCase): 416 def test_bare(self): 417 """--bare works.""" 418 args = bootstrap.parse_args(['--job=foo', '--bare']) 419 self.assertFalse(bootstrap.parse_repos(args)) 420 421 def test_pull_branch_none(self): 422 """args.pull and args.branch should be None""" 423 args = bootstrap.parse_args(['--job=foo', '--bare']) 424 self.assertIsNone(args.pull) 425 self.assertIsNone(args.branch) 426 427 def test_plain(self): 428 """"--repo=foo equals foo=master.""" 429 args = bootstrap.parse_args(['--job=foo', '--repo=foo']) 430 self.assertEquals( 431 {'foo': ('master', '')}, 432 bootstrap.parse_repos(args)) 433 434 def test_branch(self): 435 """--repo=foo=branch.""" 436 args = bootstrap.parse_args(['--job=foo', '--repo=foo=this']) 437 self.assertEquals( 438 {'foo': ('this', '')}, 439 bootstrap.parse_repos(args)) 440 441 def test_branch_commit(self): 442 """--repo=foo=branch:commit works.""" 443 args = bootstrap.parse_args(['--job=foo', '--repo=foo=this:abcd']) 444 self.assertEquals( 445 {'foo': ('this:abcd', '')}, 446 bootstrap.parse_repos(args)) 447 448 def test_parse_repos(self): 449 """--repo=foo=111,222 works""" 450 args = bootstrap.parse_args(['--job=foo', '--repo=foo=111,222']) 451 self.assertEquals( 452 {'foo': ('', '111,222')}, 453 bootstrap.parse_repos(args)) 454 455 def test_pull_branch(self): 456 """--repo=foo=master,111,222 works""" 457 args = bootstrap.parse_args(['--job=foo', '--repo=foo=master,111,222']) 458 self.assertEquals( 459 {'foo': ('', 'master,111,222')}, 460 bootstrap.parse_repos(args)) 461 462 def test_pull_release_branch(self): 463 """--repo=foo=release-3.14,&a-fancy%_branch+:abcd,222 works""" 464 args = bootstrap.parse_args(['--job=foo', 465 '--repo=foo=release-3.14,&a-fancy%_branch+:abcd,222']) 466 self.assertEquals( 467 {'foo': ('', 'release-3.14,&a-fancy%_branch+:abcd,222')}, 468 bootstrap.parse_repos(args)) 469 470 def test_pull_branch_commit(self): 471 """--repo=foo=master,111,222 works""" 472 args = bootstrap.parse_args(['--job=foo', 473 '--repo=foo=master:aaa,111:bbb,222:ccc']) 474 self.assertEquals( 475 {'foo': ('', 'master:aaa,111:bbb,222:ccc')}, 476 bootstrap.parse_repos(args)) 477 478 def test_multi_repo(self): 479 """--repo=foo=master,111,222 bar works""" 480 args = bootstrap.parse_args(['--job=foo', 481 '--repo=foo=master:aaa,111:bbb,222:ccc', 482 '--repo=bar']) 483 self.assertEquals( 484 { 485 'foo': ('', 'master:aaa,111:bbb,222:ccc'), 486 'bar': ('master', '')}, 487 bootstrap.parse_repos(args)) 488 489 490 class GSUtilTest(unittest.TestCase): 491 """Tests for GSUtil.""" 492 def test_upload_json(self): 493 fake = FakeSubprocess() 494 gsutil = bootstrap.GSUtil(fake) 495 gsutil.upload_json('fake_path', {'wee': 'fun'}) 496 self.assertTrue(any( 497 'application/json' in a for a in fake.calls[0][0])) 498 self.assertEqual(fake.file_data, ['{\n "wee": "fun"\n}']) 499 500 def test_upload_text_cached(self): 501 fake = FakeSubprocess() 502 gsutil = bootstrap.GSUtil(fake) 503 gsutil.upload_text('fake_path', 'hello world', cached=True) 504 self.assertFalse(any( 505 'Cache-Control' in a and 'max-age' in a 506 for a in fake.calls[0][0])) 507 self.assertEqual(fake.file_data, ['hello world']) 508 509 def test_upload_text_default(self): 510 fake = FakeSubprocess() 511 gsutil = bootstrap.GSUtil(fake) 512 gsutil.upload_text('fake_path', 'hello world') 513 self.assertFalse(any( 514 'Cache-Control' in a and 'max-age' in a 515 for a in fake.calls[0][0])) 516 self.assertEqual(fake.file_data, ['hello world']) 517 518 def test_upload_text_uncached(self): 519 fake = FakeSubprocess() 520 gsutil = bootstrap.GSUtil(fake) 521 gsutil.upload_text('fake_path', 'hello world', cached=False) 522 self.assertTrue(any( 523 'Cache-Control' in a and 'max-age' in a 524 for a in fake.calls[0][0])) 525 self.assertEqual(fake.file_data, ['hello world']) 526 527 def test_upload_text_metalink(self): 528 fake = FakeSubprocess() 529 gsutil = bootstrap.GSUtil(fake) 530 gsutil.upload_text('txt', 'path', additional_headers=['foo: bar']) 531 self.assertTrue(any('foo: bar' in a for a in fake.calls[0][0])) 532 self.assertEqual(fake.file_data, ['path']) 533 534 class FakeGSUtil(object): 535 generation = 123 536 537 def __init__(self): 538 self.cats = [] 539 self.jsons = [] 540 self.stats = [] 541 self.texts = [] 542 543 def cat(self, *a, **kw): 544 self.cats.append((a, kw)) 545 return 'this is not a list' 546 547 def stat(self, *a, **kw): 548 self.stats.append((a, kw)) 549 return 'Generation: %s' % self.generation 550 551 def upload_text(self, *args, **kwargs): 552 self.texts.append((args, kwargs)) 553 554 def upload_json(self, *args, **kwargs): 555 self.jsons.append((args, kwargs)) 556 557 class GubernatorUriTest(unittest.TestCase): 558 def create_path(self, uri): 559 self.fake_path = FakePath() 560 self.fake_path.build_log = uri 561 return self.fake_path 562 563 def test_non_gs(self): 564 uri = 'hello/world' 565 self.assertEquals('hello', bootstrap.gubernator_uri(self.create_path(uri))) 566 567 def test_multiple_gs(self): 568 uri = 'gs://hello/gs://there' 569 self.assertEquals( 570 bootstrap.GUBERNATOR + '/hello/gs:', 571 bootstrap.gubernator_uri(self.create_path(uri))) 572 573 def test_gs(self): 574 uri = 'gs://blah/blah/blah.txt' 575 self.assertEquals( 576 bootstrap.GUBERNATOR + '/blah/blah', 577 bootstrap.gubernator_uri(self.create_path(uri))) 578 579 580 class UploadPodspecTest(unittest.TestCase): 581 """ Tests for maybe_upload_podspec() """ 582 583 def test_missing_env(self): 584 """ Missing env vars return without doing anything. """ 585 # pylint: disable=no-self-use 586 bootstrap.maybe_upload_podspec(None, '', None, {}.get) 587 bootstrap.maybe_upload_podspec(None, '', None, {bootstrap.K8S_ENV: 'foo'}.get) 588 bootstrap.maybe_upload_podspec(None, '', None, {'HOSTNAME': 'blah'}.get) 589 590 def test_upload(self): 591 gsutil = FakeGSUtil() 592 call = FakeSubprocess() 593 594 output = 'type: gamma/example\n' 595 call.output['kubectl'] = [output] 596 artifacts = 'gs://bucket/logs/123/artifacts' 597 bootstrap.maybe_upload_podspec( 598 call, artifacts, gsutil, 599 {bootstrap.K8S_ENV: 'exists', 'HOSTNAME': 'abcd'}.get) 600 601 self.assertEqual( 602 call.calls, 603 [(['kubectl', 'get', '-oyaml', 'pods/abcd'], (), {'output': True})]) 604 self.assertEqual( 605 gsutil.texts, 606 [(('%s/prow_podspec.yaml' % artifacts, output), {})]) 607 608 609 class AppendResultTest(unittest.TestCase): 610 """Tests for append_result().""" 611 612 def test_new_job(self): 613 """Stat fails when the job doesn't exist.""" 614 gsutil = FakeGSUtil() 615 build = 123 616 version = 'v.interesting' 617 success = True 618 def fake_stat(*_a, **_kw): 619 raise subprocess.CalledProcessError(1, ['gsutil'], None) 620 gsutil.stat = fake_stat 621 bootstrap.append_result(gsutil, 'fake_path', build, version, success) 622 cache = gsutil.jsons[0][0][1] 623 self.assertEquals(1, len(cache)) 624 625 def test_collision_cat(self): 626 """cat fails if the cache has been updated.""" 627 gsutil = FakeGSUtil() 628 build = 42 629 version = 'v1' 630 success = False 631 generations = ['555', '444'] 632 orig_stat = gsutil.stat 633 def fake_stat(*a, **kw): 634 gsutil.generation = generations.pop() 635 return orig_stat(*a, **kw) 636 def fake_cat(_, gen): 637 if gen == '555': # Which version is requested? 638 return '[{"hello": 111}]' 639 raise subprocess.CalledProcessError(1, ['gsutil'], None) 640 with Stub(bootstrap, 'random_sleep', Pass): 641 with Stub(gsutil, 'stat', fake_stat): 642 with Stub(gsutil, 'cat', fake_cat): 643 bootstrap.append_result( 644 gsutil, 'fake_path', build, version, success) 645 self.assertIn('generation', gsutil.jsons[-1][1], gsutil.jsons) 646 self.assertEquals('555', gsutil.jsons[-1][1]['generation'], gsutil.jsons) 647 648 def test_collision_upload(self): 649 """Test when upload_json tries to update an old version.""" 650 gsutil = FakeGSUtil() 651 build = 42 652 version = 'v1' 653 success = False 654 generations = [555, 444] 655 orig = gsutil.upload_json 656 def fake_upload(path, cache, generation): 657 if generation == '555': 658 return orig(path, cache, generation=generation) 659 raise subprocess.CalledProcessError(128, ['gsutil'], None) 660 orig_stat = gsutil.stat 661 def fake_stat(*a, **kw): 662 gsutil.generation = generations.pop() 663 return orig_stat(*a, **kw) 664 def fake_cat(*_a, **_kw): 665 return '[{"hello": 111}]' 666 gsutil.stat = fake_stat 667 gsutil.upload_json = fake_upload 668 gsutil.cat = fake_cat 669 with Stub(bootstrap, 'random_sleep', Pass): 670 bootstrap.append_result( 671 gsutil, 'fake_path', build, version, success) 672 self.assertIn('generation', gsutil.jsons[-1][1], gsutil.jsons) 673 self.assertEquals('555', gsutil.jsons[-1][1]['generation'], gsutil.jsons) 674 675 def test_handle_junk(self): 676 gsutil = FakeGSUtil() 677 gsutil.cat = lambda *a, **kw: '!@!$!@$@!$' 678 build = 123 679 version = 'v.interesting' 680 success = True 681 bootstrap.append_result(gsutil, 'fake_path', build, version, success) 682 cache = gsutil.jsons[0][0][1] 683 self.assertEquals(1, len(cache)) 684 self.assertIn(build, cache[0].values()) 685 self.assertIn(version, cache[0].values()) 686 687 def test_passed_is_bool(self): 688 build = 123 689 version = 'v.interesting' 690 def try_run(success): 691 gsutil = FakeGSUtil() 692 bootstrap.append_result(gsutil, 'fake_path', build, version, success) 693 cache = gsutil.jsons[0][0][1] 694 self.assertTrue(isinstance(cache[0]['passed'], bool)) 695 696 try_run(1) 697 try_run(0) 698 try_run(None) 699 try_run('') 700 try_run('hello') 701 try_run('true') 702 703 def test_truncate(self): 704 old = json.dumps({n: True for n in range(100000)}) 705 gsutil = FakeGSUtil() 706 build = 123 707 version = 'v.interesting' 708 success = True 709 bootstrap.append_result(gsutil, 'fake_path', build, version, success) 710 cache = gsutil.jsons[0][0][1] 711 self.assertLess(len(cache), len(old)) 712 713 714 class FinishTest(unittest.TestCase): 715 """Tests for finish().""" 716 def setUp(self): 717 self.stubs = [ 718 Stub(bootstrap.GSUtil, 'upload_artifacts', Pass), 719 Stub(bootstrap, 'append_result', Pass), 720 Stub(os.path, 'isfile', Pass), 721 Stub(os.path, 'isdir', Pass), 722 ] 723 724 def tearDown(self): 725 for stub in self.stubs: 726 with stub: 727 pass 728 729 def test_no_version(self): 730 gsutil = FakeGSUtil() 731 paths = FakePath() 732 success = True 733 artifacts = 'not-a-dir' 734 no_version = '' 735 version = 'should not have found it' 736 repos = repo({REPO: ('master', '')}) 737 with Stub(bootstrap, 'metadata', lambda *a: {'random-meta': version}): 738 bootstrap.finish(gsutil, paths, success, artifacts, 739 BUILD, no_version, repos, FakeCall()) 740 bootstrap.finish(gsutil, paths, success, artifacts, BUILD, no_version, repos, FakeCall()) 741 calls = gsutil.jsons[-1] 742 # json data is second positional argument 743 self.assertNotIn('job-version', calls[0][1]) 744 self.assertNotIn('version', calls[0][1]) 745 self.assertTrue(calls[0][1].get('metadata')) 746 747 def test_metadata_version(self): 748 """Test that we will extract version info from metadata.""" 749 self.check_metadata_version('job-version') 750 self.check_metadata_version('version') 751 752 def check_metadata_version(self, key): 753 gsutil = FakeGSUtil() 754 paths = FakePath() 755 success = True 756 artifacts = 'not-a-dir' 757 no_version = '' 758 version = 'found it' 759 with Stub(bootstrap, 'metadata', lambda *a: {key: version}): 760 bootstrap.finish(gsutil, paths, success, artifacts, BUILD, no_version, REPO, FakeCall()) 761 calls = gsutil.jsons[-1] 762 # Meta is second positional argument 763 self.assertEquals(version, calls[0][1].get('job-version')) 764 self.assertEquals(version, calls[0][1].get('version')) 765 766 def test_ignore_err_up_artifacts(self): 767 paths = FakePath() 768 gsutil = FakeGSUtil() 769 local_artifacts = None 770 build = 123 771 version = 'v1.terrible' 772 success = True 773 calls = [] 774 with Stub(os.path, 'isdir', lambda _: True): 775 with Stub(os, 'walk', lambda d: [(True, True, True)]): 776 def fake_upload(*a, **kw): 777 calls.append((a, kw)) 778 raise subprocess.CalledProcessError(1, ['fakecmd'], None) 779 gsutil.upload_artifacts = fake_upload 780 repos = repo({REPO: ('master', '')}) 781 bootstrap.finish( 782 gsutil, paths, success, local_artifacts, 783 build, version, repos, FakeCall()) 784 self.assertTrue(calls) 785 786 787 def test_ignore_error_uploadtext(self): 788 paths = FakePath() 789 gsutil = FakeGSUtil() 790 local_artifacts = None 791 build = 123 792 version = 'v1.terrible' 793 success = True 794 calls = [] 795 with Stub(os.path, 'isdir', lambda _: True): 796 with Stub(os, 'walk', lambda d: [(True, True, True)]): 797 def fake_upload(*a, **kw): 798 calls.append((a, kw)) 799 raise subprocess.CalledProcessError(1, ['fakecmd'], None) 800 gsutil.upload_artifacts = Pass 801 gsutil.upload_text = fake_upload 802 repos = repo({REPO: ('master', '')}) 803 bootstrap.finish( 804 gsutil, paths, success, local_artifacts, 805 build, version, repos, FakeCall()) 806 self.assertTrue(calls) 807 self.assertGreater(calls, 1) 808 809 def test_skip_upload_artifacts(self): 810 """Do not upload artifacts dir if it doesn't exist.""" 811 paths = FakePath() 812 gsutil = FakeGSUtil() 813 local_artifacts = None 814 build = 123 815 version = 'v1.terrible' 816 success = True 817 calls = [] 818 with Stub(os.path, 'isdir', lambda _: False): 819 with Stub(bootstrap.GSUtil, 'upload_artifacts', Bomb): 820 repos = repo({REPO: ('master', '')}) 821 bootstrap.finish( 822 gsutil, paths, success, local_artifacts, 823 build, version, repos, FakeCall()) 824 self.assertFalse(calls) 825 826 827 class MetadataTest(unittest.TestCase): 828 829 def test_always_set_metadata(self): 830 repos = repo({REPO: ('master', '')}) 831 meta = bootstrap.metadata(repos, 'missing-artifacts-dir', FakeCall()) 832 self.assertIn('repo', meta) 833 self.assertEquals(REPO, meta['repo']) 834 835 def test_multi_repo(self): 836 repos = repo({REPO: ('foo', ''), 'other-repo': ('', '123,456')}) 837 meta = bootstrap.metadata(repos, 'missing-artifacts-dir', FakeCall()) 838 self.assertIn('repo', meta) 839 self.assertEquals(REPO, meta['repo']) 840 self.assertIn(REPO, meta.get('repos')) 841 self.assertEquals('foo', meta['repos'][REPO]) 842 self.assertIn('other-repo', meta.get('repos')) 843 self.assertEquals('123,456', meta['repos']['other-repo']) 844 845 846 SECONDS = 10 847 848 849 def fake_environment(set_home=True, set_node=True, set_job=True, 850 set_jenkins_home=True, set_workspace=True, **kwargs): 851 if set_home: 852 kwargs.setdefault(bootstrap.HOME_ENV, '/fake/home-dir') 853 if set_node: 854 kwargs.setdefault(bootstrap.NODE_ENV, 'fake-node') 855 if set_job: 856 kwargs.setdefault(bootstrap.JOB_ENV, JOB) 857 if set_jenkins_home: 858 kwargs.setdefault(bootstrap.JENKINS_HOME_ENV, '/fake/home-dir') 859 if set_workspace: 860 kwargs.setdefault(bootstrap.WORKSPACE_ENV, '/fake/workspace') 861 return kwargs 862 863 864 class BuildNameTest(unittest.TestCase): 865 """Tests for build_name().""" 866 867 def test_auto(self): 868 """Automatically select a build if not done by user.""" 869 with Stub(os, 'environ', fake_environment()) as fake: 870 bootstrap.build_name(SECONDS) 871 self.assertTrue(fake[bootstrap.BUILD_ENV]) 872 873 def test_manual(self): 874 """Respect user-selected build.""" 875 with Stub(os, 'environ', fake_environment()) as fake: 876 truth = 'erick is awesome' 877 fake[bootstrap.BUILD_ENV] = truth 878 self.assertEquals(truth, fake[bootstrap.BUILD_ENV]) 879 880 def test_unique(self): 881 """New build every minute.""" 882 with Stub(os, 'environ', fake_environment()) as fake: 883 bootstrap.build_name(SECONDS) 884 first = fake[bootstrap.BUILD_ENV] 885 del fake[bootstrap.BUILD_ENV] 886 bootstrap.build_name(SECONDS + 60) 887 self.assertNotEqual(first, fake[bootstrap.BUILD_ENV]) 888 889 890 class SetupCredentialsTest(unittest.TestCase): 891 """Tests for setup_credentials().""" 892 893 def setUp(self): 894 keys = { 895 bootstrap.GCE_KEY_ENV: 'fake-key', 896 bootstrap.SERVICE_ACCOUNT_ENV: 'fake-service-account.json', 897 } 898 self.env = fake_environment(**keys) 899 900 def test_norobot_noupload_noenv(self): 901 """Can avoid setting up credentials.""" 902 del self.env[bootstrap.SERVICE_ACCOUNT_ENV] 903 with Stub(os, 'environ', self.env): 904 bootstrap.setup_credentials(Bomb, None, None) 905 906 def test_upload_no_robot_raises(self): 907 del self.env[bootstrap.SERVICE_ACCOUNT_ENV] 908 with Stub(os, 'environ', self.env): 909 with self.assertRaises(ValueError): 910 bootstrap.setup_credentials(Pass, None, 'gs://fake') 911 912 913 def test_application_credentials(self): 914 """Raise if GOOGLE_APPLICATION_CREDENTIALS does not exist.""" 915 del self.env[bootstrap.SERVICE_ACCOUNT_ENV] 916 with Stub(os, 'environ', self.env) as fake: 917 gac = 'FAKE_CREDS.json' 918 fake['HOME'] = 'kansas' 919 with Stub(os.path, 'isfile', lambda p: p != gac): 920 with self.assertRaises(IOError): 921 bootstrap.setup_credentials(Pass, gac, UPLOAD) 922 923 with Stub(os.path, 'isfile', Truth): 924 call = lambda *a, **kw: 'robot' 925 bootstrap.setup_credentials(call, gac, UPLOAD) 926 # setup_creds should set SERVICE_ACCOUNT_ENV 927 self.assertEquals(gac, fake.get(bootstrap.SERVICE_ACCOUNT_ENV)) 928 # now that SERVICE_ACCOUNT_ENV is set, it should try to activate 929 # this 930 with Stub(os.path, 'isfile', lambda p: p != gac): 931 with self.assertRaises(IOError): 932 bootstrap.setup_credentials(Pass, None, UPLOAD) 933 934 935 class SetupMagicEnvironmentTest(unittest.TestCase): 936 def test_home_workspace_on_jenkins(self): 937 """WORKSPACE/HOME are set correctly for the Jenkins environment.""" 938 env = fake_environment(set_jenkins_home=True, set_workspace=True) 939 cwd = '/fake/random-location' 940 old_home = env[bootstrap.HOME_ENV] 941 old_workspace = env[bootstrap.WORKSPACE_ENV] 942 with Stub(os, 'environ', env): 943 with Stub(os, 'getcwd', lambda: cwd): 944 bootstrap.setup_magic_environment(JOB, FakeCall()) 945 946 self.assertIn(bootstrap.WORKSPACE_ENV, env) 947 self.assertNotEquals(env[bootstrap.HOME_ENV], 948 env[bootstrap.WORKSPACE_ENV]) 949 self.assertNotEquals(old_home, env[bootstrap.HOME_ENV]) 950 self.assertEquals(cwd, env[bootstrap.HOME_ENV]) 951 self.assertEquals(old_workspace, env[bootstrap.WORKSPACE_ENV]) 952 self.assertNotEquals(cwd, env[bootstrap.WORKSPACE_ENV]) 953 954 def test_home_workspace_in_k8s(self): 955 """WORKSPACE/HOME are set correctly for the kubernetes environment.""" 956 env = fake_environment(set_jenkins_home=False, set_workspace=True) 957 cwd = '/fake/random-location' 958 old_home = env[bootstrap.HOME_ENV] 959 old_workspace = env[bootstrap.WORKSPACE_ENV] 960 with Stub(os, 'environ', env): 961 with Stub(os, 'getcwd', lambda: cwd): 962 bootstrap.setup_magic_environment(JOB, FakeCall()) 963 964 self.assertIn(bootstrap.WORKSPACE_ENV, env) 965 self.assertNotEquals(env[bootstrap.HOME_ENV], 966 env[bootstrap.WORKSPACE_ENV]) 967 self.assertEquals(old_home, env[bootstrap.HOME_ENV]) 968 self.assertNotEquals(cwd, env[bootstrap.HOME_ENV]) 969 self.assertEquals(old_workspace, env[bootstrap.WORKSPACE_ENV]) 970 self.assertNotEquals(cwd, env[bootstrap.WORKSPACE_ENV]) 971 972 def test_workspace_always_set(self): 973 """WORKSPACE is set to cwd when unset in initial environment.""" 974 env = fake_environment(set_workspace=False) 975 cwd = '/fake/random-location' 976 with Stub(os, 'environ', env): 977 with Stub(os, 'getcwd', lambda: cwd): 978 bootstrap.setup_magic_environment(JOB, FakeCall()) 979 980 self.assertIn(bootstrap.WORKSPACE_ENV, env) 981 self.assertEquals(cwd, env[bootstrap.HOME_ENV]) 982 self.assertEquals(cwd, env[bootstrap.WORKSPACE_ENV]) 983 984 def test_job_env_mismatch(self): 985 env = fake_environment() 986 with Stub(os, 'environ', env): 987 self.assertNotEquals('this-is-a-job', env[bootstrap.JOB_ENV]) 988 bootstrap.setup_magic_environment('this-is-a-job', FakeCall()) 989 self.assertEquals('this-is-a-job', env[bootstrap.JOB_ENV]) 990 991 def test_expected(self): 992 env = fake_environment() 993 del env[bootstrap.JOB_ENV] 994 del env[bootstrap.NODE_ENV] 995 with Stub(os, 'environ', env): 996 # call is only used to git show the HEAD commit, so give a fake 997 # timestamp in return 998 bootstrap.setup_magic_environment(JOB, lambda *a, **kw: '123456\n') 999 1000 def check(name): 1001 self.assertIn(name, env) 1002 1003 # Some of these are probably silly to check... 1004 # TODO(fejta): remove as many of these from our infra as possible. 1005 check(bootstrap.JOB_ENV) 1006 check(bootstrap.CLOUDSDK_ENV) 1007 check(bootstrap.BOOTSTRAP_ENV) 1008 check(bootstrap.WORKSPACE_ENV) 1009 self.assertNotIn(bootstrap.SERVICE_ACCOUNT_ENV, env) 1010 self.assertEquals(env[bootstrap.SOURCE_DATE_EPOCH_ENV], '123456') 1011 1012 def test_node_present(self): 1013 expected = 'whatever' 1014 env = {bootstrap.NODE_ENV: expected} 1015 with Stub(os, 'environ', env): 1016 self.assertEquals(expected, bootstrap.node()) 1017 self.assertEquals(expected, env[bootstrap.NODE_ENV]) 1018 1019 def test_node_missing(self): 1020 env = {} 1021 with Stub(os, 'environ', env): 1022 expected = bootstrap.node() 1023 self.assertTrue(expected) 1024 self.assertEquals(expected, env[bootstrap.NODE_ENV]) 1025 1026 1027 1028 def test_cloud_sdk_config(self): 1029 cwd = 'now-here' 1030 env = fake_environment() 1031 with Stub(os, 'environ', env): 1032 with Stub(os, 'getcwd', lambda: cwd): 1033 bootstrap.setup_magic_environment(JOB, FakeCall()) 1034 1035 1036 self.assertTrue(env[bootstrap.CLOUDSDK_ENV].startswith(cwd)) 1037 1038 1039 class FakePath(object): 1040 artifacts = 'fake_artifacts' 1041 pr_latest = 'fake_pr_latest' 1042 pr_build_link = 'fake_pr_link' 1043 build_log = 'fake_log_path' 1044 pr_path = 'fake_pr_path' 1045 pr_result_cache = 'fake_pr_result_cache' 1046 latest = 'fake_latest' 1047 result_cache = 'fake_result_cache' 1048 started = 'fake_started.json' 1049 finished = 'fake_finished.json' 1050 def __call__(self, *arg, **kw): 1051 self.arg = arg 1052 self.kw = kw 1053 return self 1054 1055 1056 class FakeLogging(object): 1057 close = Pass 1058 def __call__(self, *_a, **_kw): 1059 return self 1060 1061 1062 class FakeFinish(object): 1063 called = False 1064 result = None 1065 def __call__(self, unused_a, unused_b, success, *a, **kw): 1066 self.called = True 1067 self.result = success 1068 1069 def repo(config): 1070 repos = bootstrap.Repos() 1071 for cur_repo, tup in config.items(): 1072 repos[cur_repo] = tup 1073 return repos 1074 1075 class PRPathsTest(unittest.TestCase): 1076 def test_kubernetes_kubernetes(self): 1077 """Test the kubernetes/kubernetes prefix.""" 1078 path = bootstrap.pr_paths(UPLOAD, repo({'kubernetes/kubernetes': ('', PULL)}), JOB, BUILD) 1079 self.assertTrue(any( 1080 str(PULL) == p for p in path.build_log.split('/'))) 1081 1082 def test_kubernetes(self): 1083 """Test the kubernetes/something prefix.""" 1084 path = bootstrap.pr_paths(UPLOAD, repo({'kubernetes/prefix': ('', PULL)}), JOB, BUILD) 1085 self.assertTrue(any( 1086 'prefix' in p for p in path.build_log.split('/')), path.build_log) 1087 self.assertTrue(any( 1088 str(PULL) in p for p in path.build_log.split('/')), path.build_log) 1089 1090 def test_other(self): 1091 """Test the none kubernetes prefixes.""" 1092 path = bootstrap.pr_paths(UPLOAD, repo({'github.com/random/repo': ('', PULL)}), JOB, BUILD) 1093 self.assertTrue(any( 1094 'random_repo' in p for p in path.build_log.split('/')), path.build_log) 1095 self.assertTrue(any( 1096 str(PULL) in p for p in path.build_log.split('/')), path.build_log) 1097 1098 1099 class FakeArgs(object): 1100 bare = False 1101 clean = False 1102 git_cache = '' 1103 job = JOB 1104 root = ROOT 1105 service_account = ROBOT 1106 ssh = False 1107 timeout = 0 1108 upload = UPLOAD 1109 json = False 1110 scenario = '' 1111 1112 def __init__(self, **kw): 1113 self.branch = BRANCH 1114 self.pull = PULL 1115 self.repo = [REPO] 1116 self.extra_job_args = [] 1117 for key, val in kw.items(): 1118 if not hasattr(self, key): 1119 raise AttributeError(self, key) 1120 setattr(self, key, val) 1121 1122 1123 def test_bootstrap(**kw): 1124 if isinstance(kw.get('repo'), basestring): 1125 kw['repo'] = [kw['repo']] 1126 return bootstrap.bootstrap(FakeArgs(**kw)) 1127 1128 class BootstrapTest(unittest.TestCase): 1129 1130 def setUp(self): 1131 self.boiler = [ 1132 Stub(bootstrap, 'checkout', Pass), 1133 Stub(bootstrap, 'finish', Pass), 1134 Stub(bootstrap.GSUtil, 'copy_file', Pass), 1135 Stub(bootstrap, 'node', lambda: 'fake-node'), 1136 Stub(bootstrap, 'setup_credentials', Pass), 1137 Stub(bootstrap, 'setup_logging', FakeLogging()), 1138 Stub(bootstrap, 'start', Pass), 1139 Stub(bootstrap, '_call', Pass), 1140 Stub(os, 'environ', fake_environment()), 1141 Stub(os, 'chdir', Pass), 1142 Stub(os, 'makedirs', Pass), 1143 ] 1144 1145 def tearDown(self): 1146 for stub in self.boiler: 1147 with stub: # Leaving with restores things 1148 pass 1149 1150 def test_setcreds_setroot_fails(self): 1151 """We should still call setup_credentials even if setup_root blows up.""" 1152 called = set() 1153 with Stub(bootstrap, 'setup_root', Bomb): 1154 with Stub(bootstrap, 'setup_credentials', 1155 lambda *a, **kw: called.add('setup_credentials')): 1156 with Stub(bootstrap, 'finish', lambda *a, **kw: called.add('finish')): 1157 with self.assertRaises(AssertionError): 1158 test_bootstrap() 1159 1160 for needed in ['setup_credentials', 'finish']: 1161 self.assertIn(needed, called) 1162 1163 def test_empty_repo(self): 1164 repo_name = None 1165 with Stub(bootstrap, 'checkout', Bomb): 1166 test_bootstrap(repo=repo_name, branch=None, pull=None, bare=True) 1167 with self.assertRaises(ValueError): 1168 test_bootstrap(repo=repo_name, branch=None, pull=PULL) 1169 with self.assertRaises(ValueError): 1170 test_bootstrap(repo=repo_name, branch=BRANCH, pull=None) 1171 1172 def test_root_not_exists(self): 1173 with Stub(os, 'chdir', FakeCall()) as fake_chdir: 1174 with Stub(os.path, 'exists', lambda p: False): 1175 with Stub(os, 'makedirs', FakeCall()) as fake_makedirs: 1176 test_bootstrap(branch=None) 1177 self.assertTrue(any(ROOT in c[0] for c in fake_chdir.calls), fake_chdir.calls) 1178 self.assertTrue(any(ROOT in c[0] for c in fake_makedirs.calls), fake_makedirs.calls) 1179 1180 def test_root_exists(self): 1181 with Stub(os, 'chdir', FakeCall()) as fake_chdir: 1182 test_bootstrap(branch=None) 1183 self.assertTrue(any(ROOT in c[0] for c in fake_chdir.calls)) 1184 1185 def test_pr_paths(self): 1186 """Use a pr_paths when pull is set.""" 1187 1188 with Stub(bootstrap, 'ci_paths', Bomb): 1189 with Stub(bootstrap, 'pr_paths', FakePath()) as path: 1190 test_bootstrap(branch=None, pull=PULL, root='.') 1191 self.assertEquals(PULL, path.arg[1][REPO][1], (PULL, path.arg)) 1192 1193 def test_ci_paths(self): 1194 """Use a ci_paths when branch is set.""" 1195 1196 with Stub(bootstrap, 'pr_paths', Bomb): 1197 with Stub(bootstrap, 'ci_paths', FakePath()) as path: 1198 test_bootstrap(pull=None, branch=BRANCH) 1199 self.assertFalse(any( 1200 PULL in o for o in (path.arg, path.kw))) 1201 1202 def test_finish_when_start_fails(self): 1203 """Finish is called even if start fails.""" 1204 with Stub(bootstrap, 'finish', FakeFinish()) as fake: 1205 with Stub(bootstrap, 'start', Bomb): 1206 with self.assertRaises(AssertionError): 1207 test_bootstrap(pull=None) 1208 self.assertTrue(fake.called) 1209 self.assertTrue(fake.result is False) # Distinguish from None 1210 1211 def test_finish_when_build_fails(self): 1212 """Finish is called even if the build fails.""" 1213 def call_error(*_a, **_kw): 1214 raise subprocess.CalledProcessError(1, [], '') 1215 with Stub(bootstrap, 'finish', FakeFinish()) as fake: 1216 with Stub(bootstrap, '_call', call_error): 1217 with self.assertRaises(SystemExit): 1218 test_bootstrap(pull=None) 1219 self.assertTrue(fake.called) 1220 self.assertTrue(fake.result is False) # Distinguish from None 1221 1222 def test_happy(self): 1223 with Stub(bootstrap, 'finish', FakeFinish()) as fake: 1224 test_bootstrap(pull=None) 1225 self.assertTrue(fake.called) 1226 self.assertTrue(fake.result) # Distinguish from None 1227 1228 def test_job_env(self): 1229 """bootstrap sets JOB_NAME.""" 1230 with Stub(os, 'environ', fake_environment()) as env: 1231 test_bootstrap(pull=None) 1232 self.assertIn(bootstrap.JOB_ENV, env) 1233 1234 def test_job_script_expands_vars(self): 1235 fake = { 1236 'HELLO': 'awesome', 1237 'WORLD': 'sauce', 1238 } 1239 with Stub(os, 'environ', fake): 1240 actual = bootstrap.job_args( 1241 ['$HELLO ${WORLD}', 'happy', '${MISSING}']) 1242 self.assertEquals(['awesome sauce', 'happy', '${MISSING}'], actual) 1243 1244 1245 class RepositoryTest(unittest.TestCase): 1246 def test_kubernetes_kubernetes(self): 1247 expected = 'https://github.com/kubernetes/kubernetes' 1248 actual = bootstrap.repository('k8s.io/kubernetes', '') 1249 self.assertEquals(expected, actual) 1250 1251 def test_kubernetes_testinfra(self): 1252 expected = 'https://github.com/kubernetes/test-infra' 1253 actual = bootstrap.repository('k8s.io/test-infra', '') 1254 self.assertEquals(expected, actual) 1255 1256 def test_whatever(self): 1257 expected = 'https://foo.com/bar' 1258 actual = bootstrap.repository('foo.com/bar', '') 1259 self.assertEquals(expected, actual) 1260 1261 def test_k8s_k8s_ssh(self): 1262 expected = 'git@github.com:kubernetes/kubernetes' 1263 actual = bootstrap.repository('k8s.io/kubernetes', 'path') 1264 self.assertEquals(expected, actual) 1265 1266 def test_k8s_k8s_ssh_with_colon(self): 1267 expected = 'git@github.com:kubernetes/kubernetes' 1268 actual = bootstrap.repository('github.com:kubernetes/kubernetes', 'path') 1269 self.assertEquals(expected, actual) 1270 1271 def test_whatever_ssh(self): 1272 expected = 'git@foo.com:bar' 1273 actual = bootstrap.repository('foo.com/bar', 'path') 1274 self.assertEquals(expected, actual) 1275 1276 1277 1278 class IntegrationTest(unittest.TestCase): 1279 REPO = 'hello/world' 1280 MASTER = 'fake-master-file' 1281 BRANCH_FILE = 'fake-branch-file' 1282 PR_FILE = 'fake-pr-file' 1283 BRANCH = 'another-branch' 1284 PR_NUM = 42 1285 PR_TAG = bootstrap.pull_ref(PR_NUM)[0][0].strip('+') 1286 1287 def fake_repo(self, fake, _ssh=False): 1288 return os.path.join(self.root_github, fake) 1289 1290 def setUp(self): 1291 self.boiler = [ 1292 Stub(bootstrap, 'finish', Pass), 1293 Stub(bootstrap.GSUtil, 'copy_file', Pass), 1294 Stub(bootstrap, 'repository', self.fake_repo), 1295 Stub(bootstrap, 'setup_credentials', Pass), 1296 Stub(bootstrap, 'setup_logging', FakeLogging()), 1297 Stub(bootstrap, 'start', Pass), 1298 Stub(os, 'environ', fake_environment(set_job=False)), 1299 ] 1300 self.root_github = tempfile.mkdtemp() 1301 self.root_workspace = tempfile.mkdtemp() 1302 self.root_git_cache = tempfile.mkdtemp() 1303 self.ocwd = os.getcwd() 1304 fakerepo = self.fake_repo(self.REPO) 1305 subprocess.check_call(['git', 'init', fakerepo]) 1306 os.chdir(fakerepo) 1307 subprocess.check_call(['git', 'config', 'user.name', 'foo']) 1308 subprocess.check_call(['git', 'config', 'user.email', 'foo@bar.baz']) 1309 subprocess.check_call(['touch', self.MASTER]) 1310 subprocess.check_call(['git', 'add', self.MASTER]) 1311 subprocess.check_call(['cp', '-r', bootstrap.test_infra('jenkins/fake'), fakerepo]) 1312 subprocess.check_call(['git', 'add', 'fake']) 1313 subprocess.check_call(['git', 'commit', '-m', 'Initial commit']) 1314 subprocess.check_call(['git', 'checkout', 'master']) 1315 1316 def tearDown(self): 1317 for stub in self.boiler: 1318 with stub: # Leaving with restores things 1319 pass 1320 os.chdir(self.ocwd) 1321 subprocess.check_call(['rm', '-rf', self.root_github]) 1322 subprocess.check_call(['rm', '-rf', self.root_workspace]) 1323 subprocess.check_call(['rm', '-rf', self.root_git_cache]) 1324 1325 def test_git_cache(self): 1326 subprocess.check_call(['git', 'checkout', '-b', self.BRANCH]) 1327 subprocess.check_call(['git', 'rm', self.MASTER]) 1328 subprocess.check_call(['touch', self.BRANCH_FILE]) 1329 subprocess.check_call(['git', 'add', self.BRANCH_FILE]) 1330 subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH]) 1331 test_bootstrap( 1332 job='fake-branch', 1333 repo=self.REPO, 1334 branch=self.BRANCH, 1335 pull=None, 1336 root=self.root_workspace, 1337 git_cache=self.root_git_cache) 1338 # Verify that the cache was populated by running a simple git command 1339 # in the git cache directory. 1340 subprocess.check_call( 1341 ['git', '--git-dir=%s/%s' % (self.root_git_cache, self.REPO), 'log']) 1342 1343 def test_pr(self): 1344 subprocess.check_call(['git', 'checkout', 'master']) 1345 subprocess.check_call(['git', 'checkout', '-b', 'unknown-pr-branch']) 1346 subprocess.check_call(['git', 'rm', self.MASTER]) 1347 subprocess.check_call(['touch', self.PR_FILE]) 1348 subprocess.check_call(['git', 'add', self.PR_FILE]) 1349 subprocess.check_call(['git', 'commit', '-m', 'Create branch for PR %d' % self.PR_NUM]) 1350 subprocess.check_call(['git', 'tag', self.PR_TAG]) 1351 os.chdir('/tmp') 1352 test_bootstrap( 1353 job='fake-pr', 1354 repo=self.REPO, 1355 branch=None, 1356 pull=self.PR_NUM, 1357 root=self.root_workspace) 1358 1359 def test_branch(self): 1360 subprocess.check_call(['git', 'checkout', '-b', self.BRANCH]) 1361 subprocess.check_call(['git', 'rm', self.MASTER]) 1362 subprocess.check_call(['touch', self.BRANCH_FILE]) 1363 subprocess.check_call(['git', 'add', self.BRANCH_FILE]) 1364 subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH]) 1365 1366 os.chdir('/tmp') 1367 test_bootstrap( 1368 job='fake-branch', 1369 repo=self.REPO, 1370 branch=self.BRANCH, 1371 pull=None, 1372 root=self.root_workspace) 1373 1374 def test_branch_ref(self): 1375 """Make sure we check out a specific commit.""" 1376 subprocess.check_call(['git', 'checkout', '-b', self.BRANCH]) 1377 subprocess.check_call(['git', 'rm', self.MASTER]) 1378 subprocess.check_call(['touch', self.BRANCH_FILE]) 1379 subprocess.check_call(['git', 'add', self.BRANCH_FILE]) 1380 subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH]) 1381 sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() 1382 subprocess.check_call(['rm', self.BRANCH_FILE]) 1383 subprocess.check_call(['git', 'add', self.BRANCH_FILE]) 1384 subprocess.check_call(['git', 'commit', '-m', 'Delete %s' % self.BRANCH]) 1385 1386 os.chdir('/tmp') 1387 # Supplying the commit exactly works. 1388 test_bootstrap( 1389 job='fake-branch', 1390 repo=self.REPO, 1391 branch='%s:%s' % (self.BRANCH, sha), 1392 pull=None, 1393 root=self.root_workspace) 1394 # Supplying the commit through repo works. 1395 test_bootstrap( 1396 job='fake-branch', 1397 repo="%s=%s:%s" % (self.REPO, self.BRANCH, sha), 1398 branch=None, 1399 pull=None, 1400 root=self.root_workspace) 1401 # Using branch head fails. 1402 with self.assertRaises(SystemExit): 1403 test_bootstrap( 1404 job='fake-branch', 1405 repo=self.REPO, 1406 branch=self.BRANCH, 1407 pull=None, 1408 root=self.root_workspace) 1409 with self.assertRaises(SystemExit): 1410 test_bootstrap( 1411 job='fake-branch', 1412 repo="%s=%s" % (self.REPO, self.BRANCH), 1413 branch=None, 1414 pull=None, 1415 root=self.root_workspace) 1416 1417 def test_batch(self): 1418 def head_sha(): 1419 # We can't hardcode the SHAs for the test, so we need to determine 1420 # them after each commit. 1421 return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() 1422 refs = ['master:%s' % head_sha()] 1423 master_commit_date = int(subprocess.check_output( 1424 ['git', 'show', '-s', '--format=format:%ct', head_sha()])) 1425 for pr in (123, 456): 1426 subprocess.check_call(['git', 'checkout', '-b', 'refs/pull/%d/head' % pr, 'master']) 1427 subprocess.check_call(['git', 'rm', self.MASTER]) 1428 subprocess.check_call(['touch', self.PR_FILE]) 1429 subprocess.check_call(['git', 'add', self.PR_FILE]) 1430 open('pr_%d.txt' % pr, 'w').write('some text') 1431 subprocess.check_call(['git', 'add', 'pr_%d.txt' % pr]) 1432 subprocess.check_call(['git', 'commit', '-m', 'add some stuff (#%d)' % pr]) 1433 refs.append('%d:%s' % (pr, head_sha())) 1434 os.chdir('/tmp') 1435 pull = ','.join(refs) 1436 print '--pull', pull 1437 subprocess.check_call(['ls']) 1438 test_bootstrap( 1439 job='fake-pr', 1440 repo=self.REPO, 1441 branch=None, 1442 pull=pull, 1443 root=self.root_workspace) 1444 head_commit_date = int(subprocess.check_output( 1445 ['git', 'show', '-s', '--format=format:%ct', 'test'])) 1446 # Since there were 2 PRs merged, we expect the timestamp of the latest 1447 # commit on the 'test' branch to be 2 more than master. 1448 self.assertEqual(head_commit_date, master_commit_date + 2) 1449 1450 def test_pr_bad(self): 1451 random_pr = 111 1452 with Stub(bootstrap, 'start', Bomb): 1453 with Stub(time, 'sleep', Pass): 1454 with self.assertRaises(subprocess.CalledProcessError): 1455 test_bootstrap( 1456 job='fake-pr', 1457 repo=self.REPO, 1458 branch=None, 1459 pull=random_pr, 1460 root=self.root_workspace) 1461 1462 def test_branch_bad(self): 1463 random_branch = 'something' 1464 with Stub(bootstrap, 'start', Bomb): 1465 with Stub(time, 'sleep', Pass): 1466 with self.assertRaises(subprocess.CalledProcessError): 1467 test_bootstrap( 1468 job='fake-branch', 1469 repo=self.REPO, 1470 branch=random_branch, 1471 pull=None, 1472 root=self.root_workspace) 1473 1474 def test_job_missing(self): 1475 with self.assertRaises(ValueError): 1476 test_bootstrap( 1477 job='this-job-no-exists', 1478 repo=self.REPO, 1479 branch='master', 1480 pull=None, 1481 root=self.root_workspace) 1482 1483 def test_job_fails(self): 1484 with self.assertRaises(SystemExit): 1485 test_bootstrap( 1486 job='fake-failure', 1487 repo=self.REPO, 1488 branch='master', 1489 pull=None, 1490 root=self.root_workspace) 1491 1492 def test_commit_in_meta(self): 1493 sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() 1494 cwd = os.getcwd() 1495 os.chdir(bootstrap.test_infra('.')) 1496 infra_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()[:9] 1497 os.chdir(cwd) 1498 1499 # Commit SHA should in meta 1500 call = lambda *a, **kw: bootstrap._call(5, *a, **kw) 1501 repos = repo({REPO: ('master', ''), 'other-repo': ('other-branch', '')}) 1502 meta = bootstrap.metadata(repos, 'missing-artifacts-dir', call) 1503 self.assertIn('repo-commit', meta) 1504 self.assertEquals(sha, meta['repo-commit']) 1505 self.assertEquals(40, len(meta['repo-commit'])) 1506 self.assertIn('infra-commit', meta) 1507 self.assertEquals(infra_sha, meta['infra-commit']) 1508 self.assertEquals(9, len(meta['infra-commit'])) 1509 self.assertIn(REPO, meta.get('repos')) 1510 self.assertIn('other-repo', meta.get('repos')) 1511 self.assertEquals(REPO, meta.get('repo')) 1512 1513 1514 class ParseArgsTest(unittest.TestCase): 1515 def test_barerepo_both(self): 1516 with self.assertRaises(argparse.ArgumentTypeError): 1517 bootstrap.parse_args(['--bare', '--repo=hello', '--job=j']) 1518 1519 def test_barerepo_neither(self): 1520 with self.assertRaises(argparse.ArgumentTypeError): 1521 bootstrap.parse_args(['--job=j']) 1522 1523 def test_barerepo_bareonly(self): 1524 args = bootstrap.parse_args(['--bare', '--job=j']) 1525 self.assertFalse(args.repo, args) 1526 self.assertTrue(args.bare, args) 1527 1528 def test_barerepo_repoonly(self): 1529 args = bootstrap.parse_args(['--repo=R', '--job=j']) 1530 self.assertFalse(args.bare, args) 1531 self.assertTrue(args.repo, args) 1532 1533 1534 def test_extra_job_args(self): 1535 args = bootstrap.parse_args(['--repo=R', '--job=j', '--', '--foo=bar', '--baz=quux']) 1536 self.assertEquals(args.extra_job_args, ['--foo=bar', '--baz=quux']) 1537 1538 1539 if __name__ == '__main__': 1540 unittest.main()