github.com/jenkins-x/test-infra@v0.0.7/scenarios/kubernetes_bazel.py (about)

     1  #!/usr/bin/env python
     2  
     3  # Copyright 2017 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  """Runs bazel build/test for current repo."""
    18  
    19  import argparse
    20  import os
    21  import subprocess
    22  import sys
    23  
    24  ORIG_CWD = os.getcwd()
    25  
    26  def test_infra(*paths):
    27      """Return path relative to root of test-infra repo."""
    28      return os.path.join(ORIG_CWD, os.path.dirname(__file__), '..', *paths)
    29  
    30  def check(*cmd):
    31      """Log and run the command, raising on errors."""
    32      print >>sys.stderr, 'Run:', cmd
    33      subprocess.check_call(cmd)
    34  
    35  def check_output(*cmd):
    36      """Log and run the command, raising on errors, return output"""
    37      print >>sys.stderr, 'Run:', cmd
    38      return subprocess.check_output(cmd)
    39  
    40  
    41  class Bazel(object):
    42      def __init__(self, batch):
    43          self.batch = batch
    44  
    45      def check(self, *cmd):
    46          """wrapper for check('bazel', *cmd) that respects batch"""
    47          if self.batch:
    48              check('bazel', '--batch', *cmd)
    49          else:
    50              check('bazel', *cmd)
    51  
    52      def check_output(self, *cmd):
    53          """wrapper for check_output('bazel', *cmd) that respects batch"""
    54          if self.batch:
    55              return check_output('bazel', '--batch', *cmd)
    56          return check_output('bazel', *cmd)
    57  
    58      def query(self, kind, selected_pkgs, changed_pkgs):
    59          """
    60          Run a bazel query against target kind, include targets from args.
    61  
    62          Returns a list of kind objects from bazel query.
    63          """
    64  
    65          # Changes are calculated and no packages found, return empty list.
    66          if changed_pkgs == []:
    67              return []
    68  
    69          selection = '//...'
    70          if selected_pkgs:
    71              # targets without a '-' operator prefix are implicitly additive
    72              # when specifying build targets
    73              selection = selected_pkgs[0]
    74              for pkg in selected_pkgs[1:]:
    75                  if pkg.startswith('-'):
    76                      selection += ' '+pkg
    77                  else:
    78                      selection += ' +'+pkg
    79  
    80  
    81          changes = '//...'
    82          if changed_pkgs:
    83              changes = 'set(%s)' % ' '.join(changed_pkgs)
    84  
    85          query_pat = 'kind(%s, rdeps(%s, %s)) except attr(\'tags\', \'manual\', //...)'
    86          return filter(None, self.check_output(
    87              'query',
    88              '--keep_going',
    89              '--noshow_progress',
    90              query_pat % (kind, selection, changes)
    91          ).split('\n'))
    92  
    93  
    94  def upload_string(gcs_path, text):
    95      """Uploads text to gcs_path"""
    96      cmd = ['gsutil', '-q', '-h', 'Content-Type:text/plain', 'cp', '-', gcs_path]
    97      print >>sys.stderr, 'Run:', cmd, 'stdin=%s'%text
    98      proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
    99      proc.communicate(input=text)
   100  
   101  def echo_result(res):
   102      """echo error message bazed on value of res"""
   103      echo_map = {
   104          0:'Success',
   105          1:'Build failed',
   106          2:'Bad environment or flags',
   107          3:'Build passed, tests failed or timed out',
   108          4:'Build passed, no tests found',
   109          5:'Interrupted'
   110      }
   111      print echo_map.get(res, 'Unknown exit code : %s' % res)
   112  
   113  def get_version():
   114      """Return kubernetes version"""
   115      with open('bazel-genfiles/version') as fp:
   116          return fp.read().strip()
   117  
   118  def get_changed(base, pull):
   119      """Get affected packages between base sha and pull sha."""
   120      diff = check_output(
   121          'git', 'diff', '--name-only',
   122          '--diff-filter=d', '%s...%s' % (base, pull))
   123      return check_output(
   124          'bazel', 'query',
   125          '--noshow_progress',
   126          'set(%s)' % diff).split('\n')
   127  
   128  def clean_file_in_dir(dirname, filename):
   129      """Recursively remove all file with filename in dirname."""
   130      for parent, _, filenames in os.walk(dirname):
   131          for name in filenames:
   132              if name == filename:
   133                  os.remove(os.path.join(parent, name))
   134  
   135  def main(args):
   136      """Trigger a bazel build/test run, and upload results."""
   137      # pylint:disable=too-many-branches, too-many-statements, too-many-locals
   138      if args.install:
   139          for install in args.install:
   140              if not os.path.isfile(install):
   141                  raise ValueError('Invalid install path: %s' % install)
   142              check('pip', 'install', '-r', install)
   143  
   144      bazel = Bazel(args.batch)
   145  
   146      bazel.check('version')
   147      res = 0
   148      try:
   149          affected = None
   150          if args.affected:
   151              base = os.getenv('PULL_BASE_SHA', '')
   152              pull = os.getenv('PULL_PULL_SHA', 'HEAD')
   153              if not base:
   154                  raise ValueError('PULL_BASE_SHA must be set!')
   155              affected = get_changed(base, pull)
   156  
   157          build_pkgs = []
   158          manual_build_targets = []
   159          test_pkgs = []
   160          manual_test_targets = []
   161          if args.build:
   162              build_pkgs = args.build.split(' ')
   163          if args.manual_build:
   164              manual_build_targets = args.manual_build.split(' ')
   165          if args.test:
   166              test_pkgs = args.test.split(' ')
   167          if args.manual_test:
   168              manual_test_targets = args.manual_test.split(' ')
   169  
   170          buildables = []
   171          if build_pkgs or manual_build_targets or affected:
   172              buildables = bazel.query('.*_binary', build_pkgs, affected) + manual_build_targets
   173  
   174          if buildables:
   175              bazel.check('build', *buildables)
   176          else:
   177              # Call bazel build regardless, to establish bazel symlinks
   178              bazel.check('build')
   179  
   180          # clean up previous test.xml
   181          clean_file_in_dir('./bazel-testlogs', 'test.xml')
   182  
   183          if args.release:
   184              bazel.check('build', *args.release.split(' '))
   185  
   186          if test_pkgs or manual_test_targets or affected:
   187              tests = bazel.query('test', test_pkgs, affected) + manual_test_targets
   188              if tests:
   189                  if args.test_args:
   190                      tests = args.test_args + tests
   191                  bazel.check('test', *tests)
   192      except subprocess.CalledProcessError as exp:
   193          res = exp.returncode
   194  
   195      if args.release and res == 0:
   196          version = get_version()
   197          if not version:
   198              print 'Kubernetes version missing; not uploading ci artifacts.'
   199              res = 1
   200          else:
   201              try:
   202                  if args.version_suffix:
   203                      version += args.version_suffix
   204                  gcs_build = '%s/%s' % (args.gcs, version)
   205                  bazel.check('run', '//:push-build', '--', gcs_build)
   206                  # log push-build location to path child jobs can find
   207                  # (gs://<shared-bucket>/$PULL_REFS/bazel-build-location.txt)
   208                  pull_refs = os.getenv('PULL_REFS', '')
   209                  gcs_shared = os.path.join(args.gcs_shared, pull_refs, 'bazel-build-location.txt')
   210                  if pull_refs:
   211                      upload_string(gcs_shared, gcs_build)
   212                  if args.publish_version:
   213                      upload_string(args.publish_version, version)
   214              except subprocess.CalledProcessError as exp:
   215                  res = exp.returncode
   216  
   217      # Coalesce test results into one file for upload.
   218      check(test_infra('hack/coalesce.py'))
   219  
   220      echo_result(res)
   221      if res != 0:
   222          sys.exit(res)
   223  
   224  
   225  def create_parser():
   226      """Create argparser."""
   227      parser = argparse.ArgumentParser()
   228      parser.add_argument(
   229          '--affected', action='store_true',
   230          help='If build/test affected targets. Filtered by --build and --test flags.')
   231      parser.add_argument(
   232          '--build', help='Bazel build target patterns, split by one space')
   233      parser.add_argument(
   234          '--manual-build',
   235          help='Bazel build targets that should always be manually included, split by one space'
   236      )
   237      # TODO(krzyzacy): Convert to bazel build rules
   238      parser.add_argument(
   239          '--install', action="append", help='Python dependency(s) that need to be installed')
   240      parser.add_argument(
   241          '--release', help='Run bazel build, and push release build to --gcs bucket')
   242      parser.add_argument(
   243          '--gcs-shared',
   244          default="gs://kubernetes-jenkins/shared-results/",
   245          help='If $PULL_REFS is set push build location to this bucket')
   246      parser.add_argument(
   247          '--publish-version',
   248          help='publish GCS file here with the build version, like ci/latest.txt',
   249      )
   250      parser.add_argument(
   251          '--test', help='Bazel test target patterns, split by one space')
   252      parser.add_argument(
   253          '--manual-test',
   254          help='Bazel test targets that should always be manually included, split by one space'
   255      )
   256      parser.add_argument(
   257          '--test-args', action="append", help='Bazel test args')
   258      parser.add_argument(
   259          '--gcs',
   260          default='gs://kubernetes-release-dev/bazel',
   261          help='GCS path for where to push build')
   262      parser.add_argument(
   263          '--version-suffix',
   264          help='version suffix for build pushing')
   265      parser.add_argument(
   266          '--batch', action='store_true', help='run Bazel in batch mode')
   267      return parser
   268  
   269  def parse_args(args=None):
   270      """Return parsed args."""
   271      parser = create_parser()
   272      return parser.parse_args(args)
   273  
   274  if __name__ == '__main__':
   275      main(parse_args())