github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/jenkins/bootstrap_test.py (about)

     1  #!/usr/bin/env python
     2  
     3  # Copyright 2016 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  """Tests for bootstrap."""
    18  
    19  # pylint: disable=protected-access, attribute-defined-outside-init
    20  
    21  import argparse
    22  import json
    23  import os
    24  import select
    25  import signal
    26  import subprocess
    27  import tempfile
    28  import time
    29  import unittest
    30  
    31  import bootstrap
    32  
    33  
    34  BRANCH = 'random_branch'
    35  BUILD = 'random_build'
    36  FAIL = ['/bin/bash', '-c', 'exit 1']
    37  JOB = 'random_job'
    38  PASS = ['/bin/bash', '-c', 'exit 0']
    39  PULL = 12345
    40  REPO = 'github.com/random_org/random_repo'
    41  ROBOT = 'fake-service-account.json'
    42  ROOT = '/random/root'
    43  UPLOAD = 'fake-gs://fake-bucket'
    44  
    45  
    46  class Stub(object):
    47      """Replace thing.param with replacement until exiting with."""
    48      def __init__(self, thing, param, replacement):
    49          self.thing = thing
    50          self.param = param
    51          self.replacement = replacement
    52          self.old = getattr(thing, param)
    53          setattr(thing, param, self.replacement)
    54  
    55      def __enter__(self, *a, **kw):
    56          return self.replacement
    57  
    58      def __exit__(self, *a, **kw):
    59          setattr(self.thing, self.param, self.old)
    60  
    61  
    62  class FakeCall(object):
    63      def __init__(self):
    64          self.calls = []
    65  
    66      def __call__(self, *a, **kw):
    67          self.calls.append((a, kw))
    68  
    69  
    70  class FakeSubprocess(object):
    71      """Keep track of calls."""
    72      def __init__(self):
    73          self.calls = []
    74  
    75      def __call__(self, cmd, *a, **kw):
    76          self.calls.append((cmd, a, kw))
    77  
    78  
    79  # pylint: disable=invalid-name
    80  def Pass(*_a, **_kw):
    81      """Do nothing."""
    82      pass
    83  
    84  
    85  def Truth(*_a, **_kw):
    86      """Always true."""
    87      return True
    88  
    89  
    90  def Bomb(*a, **kw):
    91      """Always raise."""
    92      raise AssertionError('Should not happen', a, kw)
    93  # pylint: enable=invalid-name
    94  
    95  
    96  class ReadAllTest(unittest.TestCase):
    97      endless = 0
    98      ended = time.time() - 50
    99      number = 0
   100      end = -1
   101  
   102      def fileno(self):
   103          return self.end
   104  
   105      def readline(self):
   106          line = 'line %d\n' % self.number
   107          self.number += 1
   108          return line
   109  
   110      def test_read_more(self):
   111          """Read lines until we clear the buffer, noting there may be more."""
   112          lines = []
   113          total = 10
   114          def more_lines(*_a, **_kw):
   115              if len(lines) < total:
   116                  return [self], [], []
   117              return [], [], []
   118          with Stub(select, 'select', more_lines):
   119              done = bootstrap.read_all(self.endless, self, lines.append)
   120  
   121          self.assertFalse(done)
   122          self.assertEquals(total, len(lines))
   123          expected = ['line %d' % d for d in range(total)]
   124          self.assertEquals(expected, lines)
   125  
   126      def test_read_expired(self):
   127          """Read nothing as we are expired, noting there may be more."""
   128          lines = []
   129          with Stub(select, 'select', lambda *a, **kw: ([], [], [])):
   130              done = bootstrap.read_all(self.ended, self, lines.append)
   131  
   132          self.assertFalse(done)
   133          self.assertFalse(lines)
   134  
   135      def test_read_end(self):
   136          """Note we reached the end of the stream."""
   137          lines = []
   138          with Stub(select, 'select', lambda *a, **kw: ([self], [], [])):
   139              with Stub(self, 'readline', lambda: ''):
   140                  done = bootstrap.read_all(self.endless, self, lines.append)
   141  
   142          self.assertTrue(done)
   143  
   144  
   145  class TerminateTest(unittest.TestCase):
   146      """Tests for termiante()."""
   147      pid = 1234
   148      pgid = 5555
   149      terminated = False
   150      killed = False
   151  
   152      def terminate(self):
   153          self.terminated = True
   154  
   155      def kill(self):
   156          self.killed = True
   157  
   158      def getpgid(self, pid):
   159          self.got = pid
   160          return self.pgid
   161  
   162      def killpg(self, pgig, sig):
   163          self.killed_pg = (pgig, sig)
   164  
   165      def test_terminate_later(self):
   166          """Do nothing if end is in the future."""
   167          timeout = bootstrap.terminate(time.time() + 50, self, False)
   168          self.assertFalse(timeout)
   169  
   170      def test_terminate_never(self):
   171          """Do nothing if end is zero."""
   172          timeout = bootstrap.terminate(0, self, False)
   173          self.assertFalse(timeout)
   174  
   175      def test_terminate_terminate(self):
   176          """Terminate pid if after end and kill is false."""
   177          timeout = bootstrap.terminate(time.time() - 50, self, False)
   178          self.assertTrue(timeout)
   179          self.assertFalse(self.killed)
   180          self.assertTrue(self.terminated)
   181  
   182      def test_terminate_kill(self):
   183          """Kill process group if after end and kill is true."""
   184          with Stub(os, 'getpgid', self.getpgid), Stub(os, 'killpg', self.killpg):
   185              timeout = bootstrap.terminate(time.time() - 50, self, True)
   186          self.assertTrue(timeout)
   187          self.assertFalse(self.terminated)
   188          self.assertTrue(self.killed)
   189          self.assertEquals(self.pid, self.got)
   190          self.assertEquals(self.killed_pg, (self.pgid, signal.SIGKILL))
   191  
   192  
   193  class SubprocessTest(unittest.TestCase):
   194      """Tests for call()."""
   195  
   196      def test_stdin(self):
   197          """Will write to subprocess.stdin."""
   198          with self.assertRaises(subprocess.CalledProcessError) as cpe:
   199              bootstrap._call(0, ['/bin/bash'], stdin='exit 92')
   200          self.assertEquals(92, cpe.exception.returncode)
   201  
   202      def test_check_true(self):
   203          """Raise on non-zero exit codes if check is set."""
   204          with self.assertRaises(subprocess.CalledProcessError):
   205              bootstrap._call(0, FAIL, check=True)
   206  
   207          bootstrap._call(0, PASS, check=True)
   208  
   209      def test_check_default(self):
   210          """Default to check=True."""
   211          with self.assertRaises(subprocess.CalledProcessError):
   212              bootstrap._call(0, FAIL)
   213  
   214          bootstrap._call(0, PASS)
   215  
   216      @staticmethod
   217      def test_check_false():
   218          """Never raise when check is not set."""
   219          bootstrap._call(0, FAIL, check=False)
   220          bootstrap._call(0, PASS, check=False)
   221  
   222      def test_output(self):
   223          """Output is returned when requested."""
   224          cmd = ['/bin/bash', '-c', 'echo hello world']
   225          self.assertEquals(
   226              'hello world\n', bootstrap._call(0, cmd, output=True))
   227  
   228      def test_zombie(self):
   229          with self.assertRaises(subprocess.CalledProcessError):
   230              # make a zombie
   231              bootstrap._call(0, ['/bin/bash', '-c', 'A=$BASHPID && ( kill -STOP $A ) & exit 1'])
   232  
   233  
   234  class PullRefsTest(unittest.TestCase):
   235      """Tests for pull_ref, branch_ref, ref_has_shas, and pull_numbers."""
   236  
   237      def test_multiple_no_shas(self):
   238          """Test master,1111,2222."""
   239          self.assertEqual(
   240              bootstrap.pull_ref('master,123,456'),
   241              ([
   242                  'master',
   243                  '+refs/pull/123/head:refs/pr/123',
   244                  '+refs/pull/456/head:refs/pr/456',
   245              ], [
   246                  'FETCH_HEAD',
   247                  'refs/pr/123',
   248                  'refs/pr/456',
   249              ]),
   250          )
   251  
   252      def test_pull_has_shas(self):
   253          self.assertTrue(bootstrap.ref_has_shas('master:abcd'))
   254          self.assertFalse(bootstrap.ref_has_shas('123'))
   255          self.assertFalse(bootstrap.ref_has_shas(123))
   256          self.assertFalse(bootstrap.ref_has_shas(None))
   257  
   258      def test_pull_numbers(self):
   259          self.assertListEqual(bootstrap.pull_numbers(123), ['123'])
   260          self.assertListEqual(bootstrap.pull_numbers('master:abcd'), [])
   261          self.assertListEqual(
   262              bootstrap.pull_numbers('master:abcd,123:qwer,124:zxcv'),
   263              ['123', '124'])
   264  
   265      def test_pull_ref(self):
   266          self.assertEqual(
   267              bootstrap.pull_ref('master:abcd,123:effe'),
   268              (['master', '+refs/pull/123/head:refs/pr/123'], ['abcd', 'effe'])
   269          )
   270          self.assertEqual(
   271              bootstrap.pull_ref('123'),
   272              (['+refs/pull/123/merge'], ['FETCH_HEAD'])
   273          )
   274  
   275      def test_branch_ref(self):
   276          self.assertEqual(
   277              bootstrap.branch_ref('branch:abcd'),
   278              (['branch'], ['abcd'])
   279          )
   280          self.assertEqual(
   281              bootstrap.branch_ref('master'),
   282              (['master'], ['FETCH_HEAD'])
   283          )
   284  
   285  
   286  class ChooseSshKeyTest(unittest.TestCase):
   287      """Tests for choose_ssh_key()."""
   288      def test_empty(self):
   289          """Do not change environ if no ssh key."""
   290          fake_env = {}
   291          with Stub(os, 'environ', fake_env):
   292              with bootstrap.choose_ssh_key(''):
   293                  self.assertFalse(fake_env)
   294  
   295      def test_full(self):
   296          fake_env = {}
   297          with Stub(os, 'environ', fake_env):
   298              with bootstrap.choose_ssh_key('hello there'):
   299                  self.assertIn('GIT_SSH', fake_env)
   300                  with open(fake_env['GIT_SSH']) as fp:
   301                      buf = fp.read()
   302                  self.assertIn('hello there', buf)
   303                  self.assertIn('ssh ', buf)
   304                  self.assertIn(' -i ', buf)
   305              self.assertFalse(fake_env)  # Resets env
   306  
   307      def test_full_old_value(self):
   308          fake_env = {'GIT_SSH': 'random-value'}
   309          old_env = dict(fake_env)
   310          with Stub(os, 'environ', fake_env):
   311              with bootstrap.choose_ssh_key('hello there'):
   312                  self.assertNotEqual(old_env, fake_env)
   313              self.assertEquals(old_env, fake_env)
   314  
   315  
   316  class CheckoutTest(unittest.TestCase):
   317      """Tests for checkout()."""
   318  
   319      def test_clean(self):
   320          """checkout cleans and resets if asked to."""
   321          fake = FakeSubprocess()
   322          with Stub(os, 'chdir', Pass):
   323              bootstrap.checkout(fake, REPO, None, PULL, clean=True)
   324  
   325          self.assertTrue(any(
   326              'clean' in cmd for cmd, _, _ in fake.calls if 'git' in cmd))
   327          self.assertTrue(any(
   328              'reset' in cmd for cmd, _, _ in fake.calls if 'git' in cmd))
   329  
   330      def test_fetch_retries(self):
   331          self.tries = 0
   332          expected_attempts = 3
   333          def third_time_charm(cmd, *_a, **_kw):
   334              if 'fetch' not in cmd:  # init/checkout are unlikely to fail
   335                  return
   336              self.tries += 1
   337              if self.tries != expected_attempts:
   338                  raise subprocess.CalledProcessError(128, cmd, None)
   339          with Stub(os, 'chdir', Pass):
   340              with Stub(time, 'sleep', Pass):
   341                  bootstrap.checkout(third_time_charm, REPO, None, PULL)
   342          self.assertEquals(expected_attempts, self.tries)
   343  
   344      def test_pull_ref(self):
   345          """checkout fetches the right ref for a pull."""
   346          fake = FakeSubprocess()
   347          with Stub(os, 'chdir', Pass):
   348              bootstrap.checkout(fake, REPO, None, PULL)
   349  
   350          expected_ref = bootstrap.pull_ref(PULL)[0][0]
   351          self.assertTrue(any(
   352              expected_ref in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd))
   353  
   354      def test_branch(self):
   355          """checkout fetches the right ref for a branch."""
   356          fake = FakeSubprocess()
   357          with Stub(os, 'chdir', Pass):
   358              bootstrap.checkout(fake, REPO, BRANCH, None)
   359  
   360          expected_ref = BRANCH
   361          self.assertTrue(any(
   362              expected_ref in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd))
   363  
   364      def test_repo(self):
   365          """checkout initializes and fetches the right repo."""
   366          fake = FakeSubprocess()
   367          with Stub(os, 'chdir', Pass):
   368              bootstrap.checkout(fake, REPO, BRANCH, None)
   369  
   370          expected_uri = 'https://%s' % REPO
   371          self.assertTrue(any(
   372              expected_uri in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd))
   373  
   374      def test_branch_xor_pull(self):
   375          """Either branch or pull specified, not both."""
   376          with Stub(os, 'chdir', Bomb):
   377              with self.assertRaises(ValueError):
   378                  bootstrap.checkout(Bomb, REPO, None, None)
   379              with self.assertRaises(ValueError):
   380                  bootstrap.checkout(Bomb, REPO, BRANCH, PULL)
   381  
   382      def test_happy(self):
   383          """checkout sanity check."""
   384          fake = FakeSubprocess()
   385          with Stub(os, 'chdir', Pass):
   386              bootstrap.checkout(fake, REPO, BRANCH, None)
   387  
   388          self.assertTrue(any(
   389              '--tags' in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd))
   390          self.assertTrue(any(
   391              'FETCH_HEAD' in cmd for cmd, _, _ in fake.calls
   392              if 'checkout' in cmd))
   393  
   394  class ParseReposTest(unittest.TestCase):
   395      def test_bare(self):
   396          """--bare works."""
   397          args = bootstrap.parse_args(['--job=foo', '--bare'])
   398          self.assertFalse(bootstrap.parse_repos(args))
   399  
   400      def test_pull_branch_none(self):
   401          """args.pull and args.branch should be None"""
   402          args = bootstrap.parse_args(['--job=foo', '--bare'])
   403          self.assertIsNone(args.pull)
   404          self.assertIsNone(args.branch)
   405  
   406      def test_plain(self):
   407          """"--repo=foo equals foo=master."""
   408          args = bootstrap.parse_args(['--job=foo', '--repo=foo'])
   409          self.assertEquals(
   410              {'foo': ('master', '')},
   411              bootstrap.parse_repos(args))
   412  
   413      def test_branch(self):
   414          """--repo=foo=branch."""
   415          args = bootstrap.parse_args(['--job=foo', '--repo=foo=this'])
   416          self.assertEquals(
   417              {'foo': ('this', '')},
   418              bootstrap.parse_repos(args))
   419  
   420      def test_branch_commit(self):
   421          """--repo=foo=branch:commit works."""
   422          args = bootstrap.parse_args(['--job=foo', '--repo=foo=this:abcd'])
   423          self.assertEquals(
   424              {'foo': ('this:abcd', '')},
   425              bootstrap.parse_repos(args))
   426  
   427      def test_parse_repos(self):
   428          """--repo=foo=111,222 works"""
   429          args = bootstrap.parse_args(['--job=foo', '--repo=foo=111,222'])
   430          self.assertEquals(
   431              {'foo': ('', '111,222')},
   432              bootstrap.parse_repos(args))
   433  
   434      def test_pull_branch(self):
   435          """--repo=foo=master,111,222 works"""
   436          args = bootstrap.parse_args(['--job=foo', '--repo=foo=master,111,222'])
   437          self.assertEquals(
   438              {'foo': ('', 'master,111,222')},
   439              bootstrap.parse_repos(args))
   440  
   441      def test_pull_release_branch(self):
   442          """--repo=foo=release-3.14,&a-fancy%_branch+:abcd,222 works"""
   443          args = bootstrap.parse_args(['--job=foo',
   444                                       '--repo=foo=release-3.14,&a-fancy%_branch+:abcd,222'])
   445          self.assertEquals(
   446              {'foo': ('', 'release-3.14,&a-fancy%_branch+:abcd,222')},
   447              bootstrap.parse_repos(args))
   448  
   449      def test_pull_branch_commit(self):
   450          """--repo=foo=master,111,222 works"""
   451          args = bootstrap.parse_args(['--job=foo',
   452                                       '--repo=foo=master:aaa,111:bbb,222:ccc'])
   453          self.assertEquals(
   454              {'foo': ('', 'master:aaa,111:bbb,222:ccc')},
   455              bootstrap.parse_repos(args))
   456  
   457      def test_multi_repo(self):
   458          """--repo=foo=master,111,222 bar works"""
   459          args = bootstrap.parse_args(['--job=foo',
   460                                       '--repo=foo=master:aaa,111:bbb,222:ccc',
   461                                       '--repo=bar'])
   462          self.assertEquals(
   463              {
   464                  'foo': ('', 'master:aaa,111:bbb,222:ccc'),
   465                  'bar': ('master', '')},
   466              bootstrap.parse_repos(args))
   467  
   468  
   469  class GSUtilTest(unittest.TestCase):
   470      """Tests for GSUtil."""
   471      def test_upload_json(self):
   472          fake = FakeSubprocess()
   473          gsutil = bootstrap.GSUtil(fake)
   474          gsutil.upload_json('fake_path', {'wee': 'fun'})
   475          self.assertTrue(any(
   476              'application/json' in a for a in fake.calls[0][0]))
   477          self.assertIn('stdin', fake.calls[0][2])  # kwargs
   478  
   479      def test_upload_text_cached(self):
   480          fake = FakeSubprocess()
   481          gsutil = bootstrap.GSUtil(fake)
   482          gsutil.upload_text('fake_path', 'hello world', cached=True)
   483          self.assertFalse(any(
   484              'Cache-Control' in a and 'max-age' in a
   485              for a in fake.calls[0][0]))
   486          self.assertIn('stdin', fake.calls[0][2])  # kwargs
   487  
   488      def test_upload_text_default(self):
   489          fake = FakeSubprocess()
   490          gsutil = bootstrap.GSUtil(fake)
   491          gsutil.upload_text('fake_path', 'hello world')
   492          self.assertFalse(any(
   493              'Cache-Control' in a and 'max-age' in a
   494              for a in fake.calls[0][0]))
   495          self.assertIn('stdin', fake.calls[0][2])  # kwargs
   496  
   497      def test_upload_text_uncached(self):
   498          fake = FakeSubprocess()
   499          gsutil = bootstrap.GSUtil(fake)
   500          gsutil.upload_text('fake_path', 'hello world', cached=False)
   501          self.assertTrue(any(
   502              'Cache-Control' in a and 'max-age' in a
   503              for a in fake.calls[0][0]))
   504          self.assertIn('stdin', fake.calls[0][2])  # kwargs
   505  
   506      def test_upload_text_metalink(self):
   507          fake = FakeSubprocess()
   508          gsutil = bootstrap.GSUtil(fake)
   509          gsutil.upload_text('txt', 'path', additional_headers=['foo: bar'])
   510          self.assertTrue(any('foo: bar' in a for a in fake.calls[0][0]))
   511  
   512  class FakeGSUtil(object):
   513      generation = 123
   514  
   515      def __init__(self):
   516          self.cats = []
   517          self.jsons = []
   518          self.stats = []
   519          self.texts = []
   520  
   521      def cat(self, *a, **kw):
   522          self.cats.append((a, kw))
   523          return 'this is not a list'
   524  
   525      def stat(self, *a, **kw):
   526          self.stats.append((a, kw))
   527          return 'Generation: %s' % self.generation
   528  
   529      def upload_text(self, *args, **kwargs):
   530          self.texts.append((args, kwargs))
   531  
   532      def upload_json(self, *args, **kwargs):
   533          self.jsons.append((args, kwargs))
   534  
   535  class GubernatorUriTest(unittest.TestCase):
   536      def create_path(self, uri):
   537          self.fake_path = FakePath()
   538          self.fake_path.build_log = uri
   539          return self.fake_path
   540  
   541      def test_non_gs(self):
   542          uri = 'hello/world'
   543          self.assertEquals('hello', bootstrap.gubernator_uri(self.create_path(uri)))
   544  
   545      def test_multiple_gs(self):
   546          uri = 'gs://hello/gs://there'
   547          self.assertEquals(
   548              bootstrap.GUBERNATOR + '/hello/gs:',
   549              bootstrap.gubernator_uri(self.create_path(uri)))
   550  
   551      def test_gs(self):
   552          uri = 'gs://blah/blah/blah.txt'
   553          self.assertEquals(
   554              bootstrap.GUBERNATOR + '/blah/blah',
   555              bootstrap.gubernator_uri(self.create_path(uri)))
   556  
   557  
   558  
   559  class AppendResultTest(unittest.TestCase):
   560      """Tests for append_result()."""
   561  
   562      def test_new_job(self):
   563          """Stat fails when the job doesn't exist."""
   564          gsutil = FakeGSUtil()
   565          build = 123
   566          version = 'v.interesting'
   567          success = True
   568          def fake_stat(*_a, **_kw):
   569              raise subprocess.CalledProcessError(1, ['gsutil'], None)
   570          gsutil.stat = fake_stat
   571          bootstrap.append_result(gsutil, 'fake_path', build, version, success)
   572          cache = gsutil.jsons[0][0][1]
   573          self.assertEquals(1, len(cache))
   574  
   575      def test_collision_cat(self):
   576          """cat fails if the cache has been updated."""
   577          gsutil = FakeGSUtil()
   578          build = 42
   579          version = 'v1'
   580          success = False
   581          generations = ['555', '444']
   582          orig_stat = gsutil.stat
   583          def fake_stat(*a, **kw):
   584              gsutil.generation = generations.pop()
   585              return orig_stat(*a, **kw)
   586          def fake_cat(_, gen):
   587              if gen == '555':  # Which version is requested?
   588                  return '[{"hello": 111}]'
   589              raise subprocess.CalledProcessError(1, ['gsutil'], None)
   590          with Stub(bootstrap, 'random_sleep', Pass):
   591              with Stub(gsutil, 'stat', fake_stat):
   592                  with Stub(gsutil, 'cat', fake_cat):
   593                      bootstrap.append_result(
   594                          gsutil, 'fake_path', build, version, success)
   595          self.assertIn('generation', gsutil.jsons[-1][1], gsutil.jsons)
   596          self.assertEquals('555', gsutil.jsons[-1][1]['generation'], gsutil.jsons)
   597  
   598      def test_collision_upload(self):
   599          """Test when upload_json tries to update an old version."""
   600          gsutil = FakeGSUtil()
   601          build = 42
   602          version = 'v1'
   603          success = False
   604          generations = [555, 444]
   605          orig = gsutil.upload_json
   606          def fake_upload(path, cache, generation):
   607              if generation == '555':
   608                  return orig(path, cache, generation=generation)
   609              raise subprocess.CalledProcessError(128, ['gsutil'], None)
   610          orig_stat = gsutil.stat
   611          def fake_stat(*a, **kw):
   612              gsutil.generation = generations.pop()
   613              return orig_stat(*a, **kw)
   614          def fake_cat(*_a, **_kw):
   615              return '[{"hello": 111}]'
   616          gsutil.stat = fake_stat
   617          gsutil.upload_json = fake_upload
   618          gsutil.cat = fake_cat
   619          with Stub(bootstrap, 'random_sleep', Pass):
   620              bootstrap.append_result(
   621                  gsutil, 'fake_path', build, version, success)
   622          self.assertIn('generation', gsutil.jsons[-1][1], gsutil.jsons)
   623          self.assertEquals('555', gsutil.jsons[-1][1]['generation'], gsutil.jsons)
   624  
   625      def test_handle_junk(self):
   626          gsutil = FakeGSUtil()
   627          gsutil.cat = lambda *a, **kw: '!@!$!@$@!$'
   628          build = 123
   629          version = 'v.interesting'
   630          success = True
   631          bootstrap.append_result(gsutil, 'fake_path', build, version, success)
   632          cache = gsutil.jsons[0][0][1]
   633          self.assertEquals(1, len(cache))
   634          self.assertIn(build, cache[0].values())
   635          self.assertIn(version, cache[0].values())
   636  
   637      def test_passed_is_bool(self):
   638          build = 123
   639          version = 'v.interesting'
   640          def try_run(success):
   641              gsutil = FakeGSUtil()
   642              bootstrap.append_result(gsutil, 'fake_path', build, version, success)
   643              cache = gsutil.jsons[0][0][1]
   644              self.assertTrue(isinstance(cache[0]['passed'], bool))
   645  
   646          try_run(1)
   647          try_run(0)
   648          try_run(None)
   649          try_run('')
   650          try_run('hello')
   651          try_run('true')
   652  
   653      def test_truncate(self):
   654          old = json.dumps({n: True for n in range(100000)})
   655          gsutil = FakeGSUtil()
   656          build = 123
   657          version = 'v.interesting'
   658          success = True
   659          bootstrap.append_result(gsutil, 'fake_path', build, version, success)
   660          cache = gsutil.jsons[0][0][1]
   661          self.assertLess(len(cache), len(old))
   662  
   663  
   664  class FinishTest(unittest.TestCase):
   665      """Tests for finish()."""
   666      def setUp(self):
   667          self.stubs = [
   668              Stub(bootstrap.GSUtil, 'upload_artifacts', Pass),
   669              Stub(bootstrap, 'append_result', Pass),
   670              Stub(os.path, 'isfile', Pass),
   671              Stub(os.path, 'isdir', Pass),
   672          ]
   673  
   674      def tearDown(self):
   675          for stub in self.stubs:
   676              with stub:
   677                  pass
   678  
   679      def test_no_version(self):
   680          gsutil = FakeGSUtil()
   681          paths = FakePath()
   682          success = True
   683          artifacts = 'not-a-dir'
   684          no_version = ''
   685          version = 'should not have found it'
   686          repos = repo({REPO: ('master', '')})
   687          with Stub(bootstrap, 'metadata', lambda *a: {'random-meta': version}):
   688              bootstrap.finish(gsutil, paths, success, artifacts,
   689                               BUILD, no_version, repos, FakeCall())
   690          bootstrap.finish(gsutil, paths, success, artifacts, BUILD, no_version, repos, FakeCall())
   691          calls = gsutil.jsons[-1]
   692          # json data is second positional argument
   693          self.assertNotIn('job-version', calls[0][1])
   694          self.assertNotIn('version', calls[0][1])
   695          self.assertTrue(calls[0][1].get('metadata'))
   696  
   697      def test_metadata_version(self):
   698          """Test that we will extract version info from metadata."""
   699          self.check_metadata_version('job-version')
   700          self.check_metadata_version('version')
   701  
   702      def check_metadata_version(self, key):
   703          gsutil = FakeGSUtil()
   704          paths = FakePath()
   705          success = True
   706          artifacts = 'not-a-dir'
   707          no_version = ''
   708          version = 'found it'
   709          with Stub(bootstrap, 'metadata', lambda *a: {key: version}):
   710              bootstrap.finish(gsutil, paths, success, artifacts, BUILD, no_version, REPO, FakeCall())
   711          calls = gsutil.jsons[-1]
   712          # Meta is second positional argument
   713          self.assertEquals(version, calls[0][1].get('job-version'))
   714          self.assertEquals(version, calls[0][1].get('version'))
   715  
   716      def test_ignore_err_up_artifacts(self):
   717          paths = FakePath()
   718          gsutil = FakeGSUtil()
   719          local_artifacts = None
   720          build = 123
   721          version = 'v1.terrible'
   722          success = True
   723          calls = []
   724          with Stub(os.path, 'isdir', lambda _: True):
   725              with Stub(os, 'walk', lambda d: [(True, True, True)]):
   726                  def fake_upload(*a, **kw):
   727                      calls.append((a, kw))
   728                      raise subprocess.CalledProcessError(1, ['fakecmd'], None)
   729                  gsutil.upload_artifacts = fake_upload
   730                  repos = repo({REPO: ('master', '')})
   731                  bootstrap.finish(
   732                      gsutil, paths, success, local_artifacts,
   733                      build, version, repos, FakeCall())
   734                  self.assertTrue(calls)
   735  
   736  
   737      def test_ignore_error_uploadtext(self):
   738          paths = FakePath()
   739          gsutil = FakeGSUtil()
   740          local_artifacts = None
   741          build = 123
   742          version = 'v1.terrible'
   743          success = True
   744          calls = []
   745          with Stub(os.path, 'isdir', lambda _: True):
   746              with Stub(os, 'walk', lambda d: [(True, True, True)]):
   747                  def fake_upload(*a, **kw):
   748                      calls.append((a, kw))
   749                      raise subprocess.CalledProcessError(1, ['fakecmd'], None)
   750                  gsutil.upload_artifacts = Pass
   751                  gsutil.upload_text = fake_upload
   752                  repos = repo({REPO: ('master', '')})
   753                  bootstrap.finish(
   754                      gsutil, paths, success, local_artifacts,
   755                      build, version, repos, FakeCall())
   756                  self.assertTrue(calls)
   757                  self.assertGreater(calls, 1)
   758  
   759      def test_skip_upload_artifacts(self):
   760          """Do not upload artifacts dir if it doesn't exist."""
   761          paths = FakePath()
   762          gsutil = FakeGSUtil()
   763          local_artifacts = None
   764          build = 123
   765          version = 'v1.terrible'
   766          success = True
   767          calls = []
   768          with Stub(os.path, 'isdir', lambda _: False):
   769              with Stub(bootstrap.GSUtil, 'upload_artifacts', Bomb):
   770                  repos = repo({REPO: ('master', '')})
   771                  bootstrap.finish(
   772                      gsutil, paths, success, local_artifacts,
   773                      build, version, repos, FakeCall())
   774                  self.assertFalse(calls)
   775  
   776  
   777  class MetadataTest(unittest.TestCase):
   778  
   779      def test_always_set_metadata(self):
   780          repos = repo({REPO: ('master', '')})
   781          meta = bootstrap.metadata(repos, 'missing-artifacts-dir', FakeCall())
   782          self.assertIn('repo', meta)
   783          self.assertEquals(REPO, meta['repo'])
   784  
   785      def test_multi_repo(self):
   786          repos = repo({REPO: ('foo', ''), 'other-repo': ('', '123,456')})
   787          meta = bootstrap.metadata(repos, 'missing-artifacts-dir', FakeCall())
   788          self.assertIn('repo', meta)
   789          self.assertEquals(REPO, meta['repo'])
   790          self.assertIn(REPO, meta.get('repos'))
   791          self.assertEquals('foo', meta['repos'][REPO])
   792          self.assertIn('other-repo', meta.get('repos'))
   793          self.assertEquals('123,456', meta['repos']['other-repo'])
   794  
   795  
   796  SECONDS = 10
   797  
   798  
   799  def fake_environment(set_home=True, set_node=True, set_job=True,
   800                       set_jenkins_home=True, set_workspace=True, **kwargs):
   801      if set_home:
   802          kwargs.setdefault(bootstrap.HOME_ENV, '/fake/home-dir')
   803      if set_node:
   804          kwargs.setdefault(bootstrap.NODE_ENV, 'fake-node')
   805      if set_job:
   806          kwargs.setdefault(bootstrap.JOB_ENV, JOB)
   807      if set_jenkins_home:
   808          kwargs.setdefault(bootstrap.JENKINS_HOME_ENV, '/fake/home-dir')
   809      if set_workspace:
   810          kwargs.setdefault(bootstrap.WORKSPACE_ENV, '/fake/workspace')
   811      return kwargs
   812  
   813  
   814  class BuildNameTest(unittest.TestCase):
   815      """Tests for build_name()."""
   816  
   817      def test_auto(self):
   818          """Automatically select a build if not done by user."""
   819          with Stub(os, 'environ', fake_environment()) as fake:
   820              bootstrap.build_name(SECONDS)
   821              self.assertTrue(fake[bootstrap.BUILD_ENV])
   822  
   823      def test_manual(self):
   824          """Respect user-selected build."""
   825          with Stub(os, 'environ', fake_environment()) as fake:
   826              truth = 'erick is awesome'
   827              fake[bootstrap.BUILD_ENV] = truth
   828              self.assertEquals(truth, fake[bootstrap.BUILD_ENV])
   829  
   830      def test_unique(self):
   831          """New build every minute."""
   832          with Stub(os, 'environ', fake_environment()) as fake:
   833              bootstrap.build_name(SECONDS)
   834              first = fake[bootstrap.BUILD_ENV]
   835              del fake[bootstrap.BUILD_ENV]
   836              bootstrap.build_name(SECONDS + 60)
   837              self.assertNotEqual(first, fake[bootstrap.BUILD_ENV])
   838  
   839  
   840  class SetupCredentialsTest(unittest.TestCase):
   841      """Tests for setup_credentials()."""
   842  
   843      def setUp(self):
   844          keys = {
   845              bootstrap.GCE_KEY_ENV: 'fake-key',
   846              bootstrap.SERVICE_ACCOUNT_ENV: 'fake-service-account.json',
   847          }
   848          self.env = fake_environment(**keys)
   849  
   850      def test_norobot_noupload_noenv(self):
   851          """Can avoid setting up credentials."""
   852          del self.env[bootstrap.SERVICE_ACCOUNT_ENV]
   853          with Stub(os, 'environ', self.env):
   854              bootstrap.setup_credentials(Bomb, None, None)
   855  
   856      def test_upload_no_robot_raises(self):
   857          del self.env[bootstrap.SERVICE_ACCOUNT_ENV]
   858          with Stub(os, 'environ', self.env):
   859              with self.assertRaises(ValueError):
   860                  bootstrap.setup_credentials(Pass, None, 'gs://fake')
   861  
   862  
   863      def test_application_credentials(self):
   864          """Raise if GOOGLE_APPLICATION_CREDENTIALS does not exist."""
   865          del self.env[bootstrap.SERVICE_ACCOUNT_ENV]
   866          with Stub(os, 'environ', self.env) as fake:
   867              gac = 'FAKE_CREDS.json'
   868              fake['HOME'] = 'kansas'
   869              with Stub(os.path, 'isfile', lambda p: p != gac):
   870                  with self.assertRaises(IOError):
   871                      bootstrap.setup_credentials(Pass, gac, UPLOAD)
   872  
   873              with Stub(os.path, 'isfile', Truth):
   874                  call = lambda *a, **kw: 'robot'
   875                  bootstrap.setup_credentials(call, gac, UPLOAD)
   876              # setup_creds should set SERVICE_ACCOUNT_ENV
   877              self.assertEquals(gac, fake.get(bootstrap.SERVICE_ACCOUNT_ENV))
   878              # now that SERVICE_ACCOUNT_ENV is set, it should try to activate
   879              # this
   880              with Stub(os.path, 'isfile', lambda p: p != gac):
   881                  with self.assertRaises(IOError):
   882                      bootstrap.setup_credentials(Pass, None, UPLOAD)
   883  
   884  
   885  class SetupMagicEnvironmentTest(unittest.TestCase):
   886      def test_home_workspace_on_jenkins(self):
   887          """WORKSPACE/HOME are set correctly for the Jenkins environment."""
   888          env = fake_environment(set_jenkins_home=True, set_workspace=True)
   889          cwd = '/fake/random-location'
   890          old_home = env[bootstrap.HOME_ENV]
   891          old_workspace = env[bootstrap.WORKSPACE_ENV]
   892          with Stub(os, 'environ', env):
   893              with Stub(os, 'getcwd', lambda: cwd):
   894                  bootstrap.setup_magic_environment(JOB)
   895  
   896          self.assertIn(bootstrap.WORKSPACE_ENV, env)
   897          self.assertNotEquals(env[bootstrap.HOME_ENV],
   898                               env[bootstrap.WORKSPACE_ENV])
   899          self.assertNotEquals(old_home, env[bootstrap.HOME_ENV])
   900          self.assertEquals(cwd, env[bootstrap.HOME_ENV])
   901          self.assertEquals(old_workspace, env[bootstrap.WORKSPACE_ENV])
   902          self.assertNotEquals(cwd, env[bootstrap.WORKSPACE_ENV])
   903  
   904      def test_home_workspace_in_k8s(self):
   905          """WORKSPACE/HOME are set correctly for the kubernetes environment."""
   906          env = fake_environment(set_jenkins_home=False, set_workspace=True)
   907          cwd = '/fake/random-location'
   908          old_home = env[bootstrap.HOME_ENV]
   909          old_workspace = env[bootstrap.WORKSPACE_ENV]
   910          with Stub(os, 'environ', env):
   911              with Stub(os, 'getcwd', lambda: cwd):
   912                  bootstrap.setup_magic_environment(JOB)
   913  
   914          self.assertIn(bootstrap.WORKSPACE_ENV, env)
   915          self.assertNotEquals(env[bootstrap.HOME_ENV],
   916                               env[bootstrap.WORKSPACE_ENV])
   917          self.assertEquals(old_home, env[bootstrap.HOME_ENV])
   918          self.assertNotEquals(cwd, env[bootstrap.HOME_ENV])
   919          self.assertEquals(old_workspace, env[bootstrap.WORKSPACE_ENV])
   920          self.assertNotEquals(cwd, env[bootstrap.WORKSPACE_ENV])
   921  
   922      def test_workspace_always_set(self):
   923          """WORKSPACE is set to cwd when unset in initial environment."""
   924          env = fake_environment(set_workspace=False)
   925          cwd = '/fake/random-location'
   926          with Stub(os, 'environ', env):
   927              with Stub(os, 'getcwd', lambda: cwd):
   928                  bootstrap.setup_magic_environment(JOB)
   929  
   930          self.assertIn(bootstrap.WORKSPACE_ENV, env)
   931          self.assertEquals(cwd, env[bootstrap.HOME_ENV])
   932          self.assertEquals(cwd, env[bootstrap.WORKSPACE_ENV])
   933  
   934      def test_job_env_mismatch(self):
   935          env = fake_environment()
   936          with Stub(os, 'environ', env):
   937              self.assertNotEquals('this-is-a-job', env[bootstrap.JOB_ENV])
   938              bootstrap.setup_magic_environment('this-is-a-job')
   939              self.assertEquals('this-is-a-job', env[bootstrap.JOB_ENV])
   940  
   941      def test_expected(self):
   942          env = fake_environment()
   943          del env[bootstrap.JOB_ENV]
   944          del env[bootstrap.NODE_ENV]
   945          with Stub(os, 'environ', env):
   946              bootstrap.setup_magic_environment(JOB)
   947  
   948          def check(name):
   949              self.assertIn(name, env)
   950  
   951          # Some of these are probably silly to check...
   952          # TODO(fejta): remove as many of these from our infra as possible.
   953          check(bootstrap.JOB_ENV)
   954          check(bootstrap.CLOUDSDK_ENV)
   955          check(bootstrap.BOOTSTRAP_ENV)
   956          check(bootstrap.WORKSPACE_ENV)
   957          self.assertNotIn(bootstrap.SERVICE_ACCOUNT_ENV, env)
   958  
   959      def test_node_present(self):
   960          expected = 'whatever'
   961          env = {bootstrap.NODE_ENV: expected}
   962          with Stub(os, 'environ', env):
   963              self.assertEquals(expected, bootstrap.node())
   964          self.assertEquals(expected, env[bootstrap.NODE_ENV])
   965  
   966      def test_node_missing(self):
   967          env = {}
   968          with Stub(os, 'environ', env):
   969              expected = bootstrap.node()
   970              self.assertTrue(expected)
   971          self.assertEquals(expected, env[bootstrap.NODE_ENV])
   972  
   973  
   974  
   975      def test_cloud_sdk_config(self):
   976          cwd = 'now-here'
   977          env = fake_environment()
   978          with Stub(os, 'environ', env):
   979              with Stub(os, 'getcwd', lambda: cwd):
   980                  bootstrap.setup_magic_environment(JOB)
   981  
   982  
   983          self.assertTrue(env[bootstrap.CLOUDSDK_ENV].startswith(cwd))
   984  
   985  
   986  class FakePath(object):
   987      artifacts = 'fake_artifacts'
   988      pr_latest = 'fake_pr_latest'
   989      pr_build_link = 'fake_pr_link'
   990      build_log = 'fake_log_path'
   991      pr_path = 'fake_pr_path'
   992      pr_result_cache = 'fake_pr_result_cache'
   993      latest = 'fake_latest'
   994      result_cache = 'fake_result_cache'
   995      started = 'fake_started.json'
   996      finished = 'fake_finished.json'
   997      def __call__(self, *arg, **kw):
   998          self.arg = arg
   999          self.kw = kw
  1000          return self
  1001  
  1002  
  1003  class FakeLogging(object):
  1004      close = Pass
  1005      def __call__(self, *_a, **_kw):
  1006          return self
  1007  
  1008  
  1009  class FakeFinish(object):
  1010      called = False
  1011      result = None
  1012      def __call__(self, unused_a, unused_b, success, *a, **kw):
  1013          self.called = True
  1014          self.result = success
  1015  
  1016  def repo(config):
  1017      repos = bootstrap.Repos()
  1018      for cur_repo, tup in config.items():
  1019          repos[cur_repo] = tup
  1020      return repos
  1021  
  1022  class PRPathsTest(unittest.TestCase):
  1023      def test_kubernetes_kubernetes(self):
  1024          """Test the kubernetes/kubernetes prefix."""
  1025          path = bootstrap.pr_paths(UPLOAD, repo({'kubernetes/kubernetes': ('', PULL)}), JOB, BUILD)
  1026          self.assertTrue(any(
  1027              str(PULL) == p for p in path.build_log.split('/')))
  1028  
  1029      def test_kubernetes(self):
  1030          """Test the kubernetes/something prefix."""
  1031          path = bootstrap.pr_paths(UPLOAD, repo({'kubernetes/prefix': ('', PULL)}), JOB, BUILD)
  1032          self.assertTrue(any(
  1033              'prefix' in p for p in path.build_log.split('/')), path.build_log)
  1034          self.assertTrue(any(
  1035              str(PULL) in p for p in path.build_log.split('/')), path.build_log)
  1036  
  1037      def test_other(self):
  1038          """Test the none kubernetes prefixes."""
  1039          path = bootstrap.pr_paths(UPLOAD, repo({'github.com/random/repo': ('', PULL)}), JOB, BUILD)
  1040          self.assertTrue(any(
  1041              'random_repo' in p for p in path.build_log.split('/')), path.build_log)
  1042          self.assertTrue(any(
  1043              str(PULL) in p for p in path.build_log.split('/')), path.build_log)
  1044  
  1045  
  1046  class FakeArgs(object):
  1047      bare = False
  1048      clean = False
  1049      git_cache = ''
  1050      job = JOB
  1051      root = ROOT
  1052      service_account = ROBOT
  1053      ssh = False
  1054      timeout = 0
  1055      upload = UPLOAD
  1056      json = False
  1057  
  1058      def __init__(self, **kw):
  1059          self.branch = BRANCH
  1060          self.pull = PULL
  1061          self.repo = [REPO]
  1062          for key, val in kw.items():
  1063              if not hasattr(self, key):
  1064                  raise AttributeError(self, key)
  1065              setattr(self, key, val)
  1066  
  1067  
  1068  def test_bootstrap(**kw):
  1069      if isinstance(kw.get('repo'), basestring):
  1070          kw['repo'] = [kw['repo']]
  1071      return bootstrap.bootstrap(FakeArgs(**kw))
  1072  
  1073  class BootstrapTest(unittest.TestCase):
  1074  
  1075      def setUp(self):
  1076          self.boiler = [
  1077              Stub(bootstrap, 'checkout', Pass),
  1078              Stub(bootstrap, 'finish', Pass),
  1079              Stub(bootstrap.GSUtil, 'copy_file', Pass),
  1080              Stub(bootstrap, 'node', lambda: 'fake-node'),
  1081              Stub(bootstrap, 'setup_credentials', Pass),
  1082              Stub(bootstrap, 'setup_logging', FakeLogging()),
  1083              Stub(bootstrap, 'start', Pass),
  1084              Stub(bootstrap, '_call', Pass),
  1085              Stub(os, 'environ', fake_environment()),
  1086              Stub(os, 'chdir', Pass),
  1087              Stub(os, 'makedirs', Pass),
  1088          ]
  1089  
  1090      def tearDown(self):
  1091          for stub in self.boiler:
  1092              with stub:  # Leaving with restores things
  1093                  pass
  1094  
  1095      def test_setcreds_setroot_fails(self):
  1096          """We should still call setup_credentials even if setup_root blows up."""
  1097          called = set()
  1098          with Stub(bootstrap, 'setup_root', Bomb):
  1099              with Stub(bootstrap, 'setup_credentials',
  1100                        lambda *a, **kw: called.add('setup_credentials')):
  1101                  with Stub(bootstrap, 'finish', lambda *a, **kw: called.add('finish')):
  1102                      with self.assertRaises(AssertionError):
  1103                          test_bootstrap()
  1104  
  1105          for needed in ['setup_credentials', 'finish']:
  1106              self.assertIn(needed, called)
  1107  
  1108      def test_empty_repo(self):
  1109          repo_name = None
  1110          with Stub(bootstrap, 'checkout', Bomb):
  1111              test_bootstrap(repo=repo_name, branch=None, pull=None, bare=True)
  1112          with self.assertRaises(ValueError):
  1113              test_bootstrap(repo=repo_name, branch=None, pull=PULL)
  1114          with self.assertRaises(ValueError):
  1115              test_bootstrap(repo=repo_name, branch=BRANCH, pull=None)
  1116  
  1117      def test_root_not_exists(self):
  1118          with Stub(os, 'chdir', FakeCall()) as fake_chdir:
  1119              with Stub(os.path, 'exists', lambda p: False):
  1120                  with Stub(os, 'makedirs', FakeCall()) as fake_makedirs:
  1121                      test_bootstrap(branch=None)
  1122          self.assertTrue(any(ROOT in c[0] for c in fake_chdir.calls), fake_chdir.calls)
  1123          self.assertTrue(any(ROOT in c[0] for c in fake_makedirs.calls), fake_makedirs.calls)
  1124  
  1125      def test_root_exists(self):
  1126          with Stub(os, 'chdir', FakeCall()) as fake_chdir:
  1127              test_bootstrap(branch=None)
  1128          self.assertTrue(any(ROOT in c[0] for c in fake_chdir.calls))
  1129  
  1130      def test_pr_paths(self):
  1131          """Use a pr_paths when pull is set."""
  1132  
  1133          with Stub(bootstrap, 'ci_paths', Bomb):
  1134              with Stub(bootstrap, 'pr_paths', FakePath()) as path:
  1135                  test_bootstrap(branch=None, pull=PULL, root='.')
  1136              self.assertEquals(PULL, path.arg[1][REPO][1], (PULL, path.arg))
  1137  
  1138      def test_ci_paths(self):
  1139          """Use a ci_paths when branch is set."""
  1140  
  1141          with Stub(bootstrap, 'pr_paths', Bomb):
  1142              with Stub(bootstrap, 'ci_paths', FakePath()) as path:
  1143                  test_bootstrap(pull=None, branch=BRANCH)
  1144              self.assertFalse(any(
  1145                  PULL in o for o in (path.arg, path.kw)))
  1146  
  1147      def test_finish_when_start_fails(self):
  1148          """Finish is called even if start fails."""
  1149          with Stub(bootstrap, 'finish', FakeFinish()) as fake:
  1150              with Stub(bootstrap, 'start', Bomb):
  1151                  with self.assertRaises(AssertionError):
  1152                      test_bootstrap(pull=None)
  1153          self.assertTrue(fake.called)
  1154          self.assertTrue(fake.result is False)  # Distinguish from None
  1155  
  1156      def test_finish_when_build_fails(self):
  1157          """Finish is called even if the build fails."""
  1158          def call_error(*_a, **_kw):
  1159              raise subprocess.CalledProcessError(1, [], '')
  1160          with Stub(bootstrap, 'finish', FakeFinish()) as fake:
  1161              with Stub(bootstrap, '_call', call_error):
  1162                  with self.assertRaises(SystemExit):
  1163                      test_bootstrap(pull=None)
  1164          self.assertTrue(fake.called)
  1165          self.assertTrue(fake.result is False)  # Distinguish from None
  1166  
  1167      def test_happy(self):
  1168          with Stub(bootstrap, 'finish', FakeFinish()) as fake:
  1169              test_bootstrap(pull=None)
  1170          self.assertTrue(fake.called)
  1171          self.assertTrue(fake.result)  # Distinguish from None
  1172  
  1173      def test_job_env(self):
  1174          """bootstrap sets JOB_NAME."""
  1175          with Stub(os, 'environ', fake_environment()) as env:
  1176              test_bootstrap(pull=None)
  1177          self.assertIn(bootstrap.JOB_ENV, env)
  1178  
  1179      def test_job_script_expands_vars(self):
  1180          fake = {
  1181              'HELLO': 'awesome',
  1182              'WORLD': 'sauce',
  1183          }
  1184          with Stub(os, 'environ', fake):
  1185              actual = bootstrap.job_args(
  1186                  ['$HELLO ${WORLD}', 'happy', '${MISSING}'])
  1187          self.assertEquals(['awesome sauce', 'happy', '${MISSING}'], actual)
  1188  
  1189  
  1190  class RepositoryTest(unittest.TestCase):
  1191      def test_kubernetes_kubernetes(self):
  1192          expected = 'https://github.com/kubernetes/kubernetes'
  1193          actual = bootstrap.repository('k8s.io/kubernetes', '')
  1194          self.assertEquals(expected, actual)
  1195  
  1196      def test_kubernetes_testinfra(self):
  1197          expected = 'https://github.com/kubernetes/test-infra'
  1198          actual = bootstrap.repository('k8s.io/test-infra', '')
  1199          self.assertEquals(expected, actual)
  1200  
  1201      def test_whatever(self):
  1202          expected = 'https://foo.com/bar'
  1203          actual = bootstrap.repository('foo.com/bar', '')
  1204          self.assertEquals(expected, actual)
  1205  
  1206      def test_k8s_k8s_ssh(self):
  1207          expected = 'git@github.com:kubernetes/kubernetes'
  1208          actual = bootstrap.repository('k8s.io/kubernetes', 'path')
  1209          self.assertEquals(expected, actual)
  1210  
  1211      def test_k8s_k8s_ssh_with_colon(self):
  1212          expected = 'git@github.com:kubernetes/kubernetes'
  1213          actual = bootstrap.repository('github.com:kubernetes/kubernetes', 'path')
  1214          self.assertEquals(expected, actual)
  1215  
  1216      def test_whatever_ssh(self):
  1217          expected = 'git@foo.com:bar'
  1218          actual = bootstrap.repository('foo.com/bar', 'path')
  1219          self.assertEquals(expected, actual)
  1220  
  1221  
  1222  
  1223  class IntegrationTest(unittest.TestCase):
  1224      REPO = 'hello/world'
  1225      MASTER = 'fake-master-file'
  1226      BRANCH_FILE = 'fake-branch-file'
  1227      PR_FILE = 'fake-pr-file'
  1228      BRANCH = 'another-branch'
  1229      PR_NUM = 42
  1230      PR_TAG = bootstrap.pull_ref(PR_NUM)[0][0].strip('+')
  1231  
  1232      def fake_repo(self, fake, _ssh=False):
  1233          return os.path.join(self.root_github, fake)
  1234  
  1235      def setUp(self):
  1236          self.boiler = [
  1237              Stub(bootstrap, 'finish', Pass),
  1238              Stub(bootstrap.GSUtil, 'copy_file', Pass),
  1239              Stub(bootstrap, 'repository', self.fake_repo),
  1240              Stub(bootstrap, 'setup_credentials', Pass),
  1241              Stub(bootstrap, 'setup_logging', FakeLogging()),
  1242              Stub(bootstrap, 'start', Pass),
  1243              Stub(os, 'environ', fake_environment(set_job=False)),
  1244          ]
  1245          self.root_github = tempfile.mkdtemp()
  1246          self.root_workspace = tempfile.mkdtemp()
  1247          self.root_git_cache = tempfile.mkdtemp()
  1248          self.ocwd = os.getcwd()
  1249          fakerepo = self.fake_repo(self.REPO)
  1250          subprocess.check_call(['git', 'init', fakerepo])
  1251          os.chdir(fakerepo)
  1252          subprocess.check_call(['git', 'config', 'user.name', 'foo'])
  1253          subprocess.check_call(['git', 'config', 'user.email', 'foo@bar.baz'])
  1254          subprocess.check_call(['touch', self.MASTER])
  1255          subprocess.check_call(['git', 'add', self.MASTER])
  1256          subprocess.check_call(['cp', '-r', bootstrap.test_infra('jenkins/fake'), fakerepo])
  1257          subprocess.check_call(['git', 'add', 'fake'])
  1258          subprocess.check_call(['git', 'commit', '-m', 'Initial commit'])
  1259          subprocess.check_call(['git', 'checkout', 'master'])
  1260  
  1261      def tearDown(self):
  1262          for stub in self.boiler:
  1263              with stub:  # Leaving with restores things
  1264                  pass
  1265          os.chdir(self.ocwd)
  1266          subprocess.check_call(['rm', '-rf', self.root_github])
  1267          subprocess.check_call(['rm', '-rf', self.root_workspace])
  1268          subprocess.check_call(['rm', '-rf', self.root_git_cache])
  1269  
  1270      def test_git_cache(self):
  1271          subprocess.check_call(['git', 'checkout', '-b', self.BRANCH])
  1272          subprocess.check_call(['git', 'rm', self.MASTER])
  1273          subprocess.check_call(['touch', self.BRANCH_FILE])
  1274          subprocess.check_call(['git', 'add', self.BRANCH_FILE])
  1275          subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH])
  1276          test_bootstrap(
  1277              job='fake-branch',
  1278              repo=self.REPO,
  1279              branch=self.BRANCH,
  1280              pull=None,
  1281              root=self.root_workspace,
  1282              git_cache=self.root_git_cache)
  1283          # Verify that the cache was populated by running a simple git command
  1284          # in the git cache directory.
  1285          subprocess.check_call(
  1286              ['git', '--git-dir=%s/%s' % (self.root_git_cache, self.REPO), 'log'])
  1287  
  1288      def test_pr(self):
  1289          subprocess.check_call(['git', 'checkout', 'master'])
  1290          subprocess.check_call(['git', 'checkout', '-b', 'unknown-pr-branch'])
  1291          subprocess.check_call(['git', 'rm', self.MASTER])
  1292          subprocess.check_call(['touch', self.PR_FILE])
  1293          subprocess.check_call(['git', 'add', self.PR_FILE])
  1294          subprocess.check_call(['git', 'commit', '-m', 'Create branch for PR %d' % self.PR_NUM])
  1295          subprocess.check_call(['git', 'tag', self.PR_TAG])
  1296          os.chdir('/tmp')
  1297          test_bootstrap(
  1298              job='fake-pr',
  1299              repo=self.REPO,
  1300              branch=None,
  1301              pull=self.PR_NUM,
  1302              root=self.root_workspace)
  1303  
  1304      def test_branch(self):
  1305          subprocess.check_call(['git', 'checkout', '-b', self.BRANCH])
  1306          subprocess.check_call(['git', 'rm', self.MASTER])
  1307          subprocess.check_call(['touch', self.BRANCH_FILE])
  1308          subprocess.check_call(['git', 'add', self.BRANCH_FILE])
  1309          subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH])
  1310  
  1311          os.chdir('/tmp')
  1312          test_bootstrap(
  1313              job='fake-branch',
  1314              repo=self.REPO,
  1315              branch=self.BRANCH,
  1316              pull=None,
  1317              root=self.root_workspace)
  1318  
  1319      def test_branch_ref(self):
  1320          """Make sure we check out a specific commit."""
  1321          subprocess.check_call(['git', 'checkout', '-b', self.BRANCH])
  1322          subprocess.check_call(['git', 'rm', self.MASTER])
  1323          subprocess.check_call(['touch', self.BRANCH_FILE])
  1324          subprocess.check_call(['git', 'add', self.BRANCH_FILE])
  1325          subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH])
  1326          sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
  1327          subprocess.check_call(['rm', self.BRANCH_FILE])
  1328          subprocess.check_call(['git', 'add', self.BRANCH_FILE])
  1329          subprocess.check_call(['git', 'commit', '-m', 'Delete %s' % self.BRANCH])
  1330  
  1331          os.chdir('/tmp')
  1332          # Supplying the commit exactly works.
  1333          test_bootstrap(
  1334              job='fake-branch',
  1335              repo=self.REPO,
  1336              branch='%s:%s' % (self.BRANCH, sha),
  1337              pull=None,
  1338              root=self.root_workspace)
  1339          # Supplying the commit through repo works.
  1340          test_bootstrap(
  1341              job='fake-branch',
  1342              repo="%s=%s:%s" % (self.REPO, self.BRANCH, sha),
  1343              branch=None,
  1344              pull=None,
  1345              root=self.root_workspace)
  1346          # Using branch head fails.
  1347          with self.assertRaises(SystemExit):
  1348              test_bootstrap(
  1349                  job='fake-branch',
  1350                  repo=self.REPO,
  1351                  branch=self.BRANCH,
  1352                  pull=None,
  1353                  root=self.root_workspace)
  1354          with self.assertRaises(SystemExit):
  1355              test_bootstrap(
  1356                  job='fake-branch',
  1357                  repo="%s=%s" % (self.REPO, self.BRANCH),
  1358                  branch=None,
  1359                  pull=None,
  1360                  root=self.root_workspace)
  1361  
  1362      def test_batch(self):
  1363          def head_sha():
  1364              # We can't hardcode the SHAs for the test, so we need to determine
  1365              # them after each commit.
  1366              return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
  1367          refs = ['master:%s' % head_sha()]
  1368          for pr in (123, 456):
  1369              subprocess.check_call(['git', 'checkout', '-b', 'refs/pull/%d/head' % pr, 'master'])
  1370              subprocess.check_call(['git', 'rm', self.MASTER])
  1371              subprocess.check_call(['touch', self.PR_FILE])
  1372              subprocess.check_call(['git', 'add', self.PR_FILE])
  1373              open('pr_%d.txt' % pr, 'w').write('some text')
  1374              subprocess.check_call(['git', 'add', 'pr_%d.txt' % pr])
  1375              subprocess.check_call(['git', 'commit', '-m', 'add some stuff (#%d)' % pr])
  1376              refs.append('%d:%s' % (pr, head_sha()))
  1377          os.chdir('/tmp')
  1378          pull = ','.join(refs)
  1379          print '--pull', pull
  1380          subprocess.check_call(['ls'])
  1381          test_bootstrap(
  1382              job='fake-pr',
  1383              repo=self.REPO,
  1384              branch=None,
  1385              pull=pull,
  1386              root=self.root_workspace)
  1387  
  1388      def test_pr_bad(self):
  1389          random_pr = 111
  1390          with Stub(bootstrap, 'start', Bomb):
  1391              with Stub(time, 'sleep', Pass):
  1392                  with self.assertRaises(subprocess.CalledProcessError):
  1393                      test_bootstrap(
  1394                          job='fake-pr',
  1395                          repo=self.REPO,
  1396                          branch=None,
  1397                          pull=random_pr,
  1398                          root=self.root_workspace)
  1399  
  1400      def test_branch_bad(self):
  1401          random_branch = 'something'
  1402          with Stub(bootstrap, 'start', Bomb):
  1403              with Stub(time, 'sleep', Pass):
  1404                  with self.assertRaises(subprocess.CalledProcessError):
  1405                      test_bootstrap(
  1406                          job='fake-branch',
  1407                          repo=self.REPO,
  1408                          branch=random_branch,
  1409                          pull=None,
  1410                          root=self.root_workspace)
  1411  
  1412      def test_job_missing(self):
  1413          with self.assertRaises(KeyError):
  1414              test_bootstrap(
  1415                  job='this-job-no-exists',
  1416                  repo=self.REPO,
  1417                  branch='master',
  1418                  pull=None,
  1419                  root=self.root_workspace)
  1420  
  1421      def test_job_fails(self):
  1422          with self.assertRaises(SystemExit):
  1423              test_bootstrap(
  1424                  job='fake-failure',
  1425                  repo=self.REPO,
  1426                  branch='master',
  1427                  pull=None,
  1428                  root=self.root_workspace)
  1429  
  1430      def test_commit_in_meta(self):
  1431          sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
  1432          cwd = os.getcwd()
  1433          os.chdir(bootstrap.test_infra('.'))
  1434          infra_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()[:9]
  1435          os.chdir(cwd)
  1436  
  1437          # Commit SHA should in meta
  1438          call = lambda *a, **kw: bootstrap._call(5, *a, **kw)
  1439          repos = repo({REPO: ('master', ''), 'other-repo': ('other-branch', '')})
  1440          meta = bootstrap.metadata(repos, 'missing-artifacts-dir', call)
  1441          self.assertIn('repo-commit', meta)
  1442          self.assertEquals(sha, meta['repo-commit'])
  1443          self.assertEquals(40, len(meta['repo-commit']))
  1444          self.assertIn('infra-commit', meta)
  1445          self.assertEquals(infra_sha, meta['infra-commit'])
  1446          self.assertEquals(9, len(meta['infra-commit']))
  1447          self.assertIn(REPO, meta.get('repos'))
  1448          self.assertIn('other-repo', meta.get('repos'))
  1449          self.assertEquals(REPO, meta.get('repo'))
  1450  
  1451  
  1452  class ParseArgsTest(unittest.TestCase):
  1453      def test_barerepo_both(self):
  1454          with self.assertRaises(argparse.ArgumentTypeError):
  1455              bootstrap.parse_args(['--bare', '--repo=hello', '--job=j'])
  1456  
  1457      def test_barerepo_neither(self):
  1458          with self.assertRaises(argparse.ArgumentTypeError):
  1459              bootstrap.parse_args(['--job=j'])
  1460  
  1461      def test_barerepo_bareonly(self):
  1462          args = bootstrap.parse_args(['--bare', '--job=j'])
  1463          self.assertFalse(args.repo, args)
  1464          self.assertTrue(args.bare, args)
  1465  
  1466      def test_barerepo_repoonly(self):
  1467          args = bootstrap.parse_args(['--repo=R', '--job=j'])
  1468          self.assertFalse(args.bare, args)
  1469          self.assertTrue(args.repo, args)
  1470  
  1471  
  1472  if __name__ == '__main__':
  1473      unittest.main()