github.com/niedbalski/juju@v0.0.0-20190215020005-8ff100488e47/acceptancetests/jujuci.py (about)

     1  #!/usr/bin/python
     2  """Access Juju CI artifacts and data."""
     3  
     4  from __future__ import print_function
     5  
     6  from argparse import ArgumentParser
     7  import base64
     8  from collections import namedtuple
     9  import fnmatch
    10  import json
    11  import os
    12  import shutil
    13  import sys
    14  import traceback
    15  import urllib
    16  import urllib2
    17  
    18  try:
    19      from lsb_release import get_distro_information
    20  except ImportError:
    21      def get_distro_information():
    22          raise NotImplementedError('Not supported on this platform!')
    23  
    24  from utility import (
    25      extract_deb,
    26      get_deb_arch,
    27      print_now,
    28      )
    29  
    30  
    31  __metaclass__ = type
    32  
    33  
    34  JENKINS_URL = 'http://jenkins:8080'
    35  BUILD_REVISION = 'build-revision'
    36  PUBLISH_REVISION = 'publish-revision'
    37  
    38  Artifact = namedtuple('Artifact', ['file_name', 'location'])
    39  
    40  
    41  Credentials = namedtuple('Credentials', ['user', 'password'])
    42  
    43  
    44  class CredentialsMissing(Exception):
    45      """Raised when no credentials are supplied."""
    46  
    47  
    48  def get_jenkins_json(credentials, url):
    49      req = urllib2.Request(url)
    50      encoded = base64.encodestring(
    51          '{}:{}'.format(*credentials)).replace('\n', '')
    52      req.add_header('Authorization', 'Basic {}'.format(encoded))
    53      build_data = urllib2.urlopen(req)
    54      return json.load(build_data)
    55  
    56  
    57  def get_build_data(jenkins_url, credentials, job_name,
    58                     build='lastSuccessfulBuild'):
    59      """Return a dict of the build data for a job build number."""
    60      url = '%s/job/%s/%s/api/json' % (jenkins_url, job_name, build)
    61      return get_jenkins_json(credentials, url)
    62  
    63  
    64  def get_job_data(jenkins_url, credentials, job_name):
    65      """Return a dict of the job data for a job name."""
    66      url = '%s/job/%s/api/json' % (jenkins_url, job_name)
    67      return get_jenkins_json(credentials, url)
    68  
    69  
    70  def make_artifact(build_data, artifact):
    71      location = '%sartifact/%s' % (build_data['url'], artifact['relativePath'])
    72      return Artifact(artifact['fileName'], location)
    73  
    74  
    75  def find_artifacts(build_data, glob='*'):
    76      found = []
    77      for artifact in build_data['artifacts']:
    78          file_name = artifact['fileName']
    79          if fnmatch.fnmatch(file_name, glob):
    80              found.append(make_artifact(build_data, artifact))
    81      return found
    82  
    83  
    84  def list_artifacts(credentials, job_name, build, glob, verbose=False):
    85      build_data = get_build_data(JENKINS_URL, credentials, job_name, build)
    86      artifacts = find_artifacts(build_data, glob)
    87      for artifact in artifacts:
    88          if verbose:
    89              print_now(artifact.location)
    90          else:
    91              print_now(artifact.file_name)
    92  
    93  
    94  def retrieve_artifact(credentials, url, local_path):
    95      auth_location = url.replace('http://',
    96                                  'http://{}:{}@'.format(*credentials))
    97      urllib.urlretrieve(auth_location, local_path)
    98  
    99  
   100  def acquire_binary(package_path, workspace):
   101      bin_dir = os.path.join(workspace, 'extracted-bin')
   102      extract_deb(package_path, bin_dir)
   103      for root, dirs, files in os.walk(bin_dir):
   104          if 'juju' in files and os.path.basename(root) == 'bin':
   105              return os.path.join(root, 'juju')
   106  
   107  
   108  def get_artifacts(credentials, job_name, build, glob, path,
   109                    archive=False, dry_run=False, verbose=False):
   110      full_path = os.path.expanduser(path)
   111      if archive:
   112          if verbose:
   113              print_now('Cleaning %s' % full_path)
   114          if not os.path.isdir(full_path):
   115              raise ValueError('%s does not exist' % full_path)
   116          shutil.rmtree(full_path)
   117          os.makedirs(full_path)
   118      build_data = get_build_data(JENKINS_URL, credentials, job_name, build)
   119      artifacts = find_artifacts(build_data, glob)
   120      for artifact in artifacts:
   121          local_path = os.path.abspath(
   122              os.path.join(full_path, artifact.file_name))
   123          if verbose:
   124              print_now('Retrieving %s => %s' % (artifact.location, local_path))
   125          else:
   126              print_now(artifact.file_name)
   127          if not dry_run:
   128              retrieve_artifact(credentials, artifact.location, local_path)
   129      return artifacts
   130  
   131  
   132  def setup_workspace(workspace_dir, dry_run=False, verbose=False):
   133      """Clean the workspace directory and create an artifacts sub directory."""
   134      for root, dirs, files in os.walk(workspace_dir):
   135          for name in files:
   136              print_now('Removing %s' % name)
   137              if not dry_run:
   138                  os.remove(os.path.join(root, name))
   139          for name in dirs:
   140              print_now('Removing %s' % name)
   141              if not dry_run:
   142                  shutil.rmtree(os.path.join(root, name))
   143      artifacts_path = os.path.join(workspace_dir, 'artifacts')
   144      print_now('Creating artifacts dir.')
   145      if not dry_run:
   146          os.mkdir(artifacts_path)
   147      # "touch empty" to convince jenkins there is an archive.
   148      empty_path = os.path.join(artifacts_path, 'empty')
   149      if not dry_run:
   150          with open(empty_path, 'w'):
   151              pass
   152  
   153  
   154  def add_artifacts(workspace_dir, globs, dry_run=False, verbose=False):
   155      """Find files beneath the workspace_dir and move them to the artifacts.
   156  
   157      The list of globs can match the full file name, part of a name, or
   158      a sub directory: eg: buildvars.json, *.deb, tmp/*.deb.
   159      """
   160      workspace_dir = os.path.realpath(workspace_dir)
   161      artifacts_dir = os.path.join(workspace_dir, 'artifacts')
   162      for root, dirs, files in os.walk(workspace_dir):
   163          # create a pseudo-relative path to make glob matches easy.
   164          relative = os.path.relpath(root, workspace_dir)
   165          if relative == '.':
   166              relative = ''
   167          if 'artifacts' in dirs:
   168              dirs.remove('artifacts')
   169          for file_name in files:
   170              file_path = os.path.join(root, file_name)
   171              file_relative_path = os.path.join(relative, file_name)
   172              for glob in globs:
   173                  if fnmatch.fnmatch(file_relative_path, glob):
   174                      if verbose:
   175                          print_now("Adding artifact %s" % file_relative_path)
   176                      if not dry_run:
   177                          shutil.move(file_path, artifacts_dir)
   178                      break
   179  
   180  
   181  def add_build_job_glob(parser):
   182      """Added the --build, job, and glob arguments to the parser."""
   183      parser.add_argument(
   184          '-b', '--build', default='lastSuccessfulBuild',
   185          help="The specific build to examine (default: lastSuccessfulBuild).")
   186      parser.add_argument(
   187          'job', help="The job that collected the artifacts.")
   188      parser.add_argument(
   189          'glob', nargs='?', default='*',
   190          help="The glob pattern to match artifact file names.")
   191  
   192  
   193  def add_credential_args(parser):
   194      parser.add_argument(
   195          '--user', default=os.environ.get('JENKINS_USER'))
   196      parser.add_argument(
   197          '--password', default=os.environ.get('JENKINS_PASSWORD'))
   198  
   199  
   200  def parse_args(args=None):
   201      """Return the argument parser for this program."""
   202      parser = ArgumentParser("List and get artifacts from Juju CI.")
   203      parser.add_argument(
   204          '-d', '--dry-run', action='store_true', default=False,
   205          help='Do not make changes.')
   206      parser.add_argument(
   207          '-v', '--verbose', action='store_true', default=False,
   208          help='Increase verbosity.')
   209      subparsers = parser.add_subparsers(help='sub-command help', dest="command")
   210      parser_list = subparsers.add_parser(
   211          'list', help='list artifacts for a job build')
   212      add_build_job_glob(parser_list)
   213      add_credential_args(parser_list)
   214      parser_workspace = subparsers.add_parser(
   215          'setup-workspace', help='Setup a workspace for building.')
   216      parser_workspace.add_argument(
   217          'path', help="The path to the existing workspace directory.")
   218      parsed_args = parser.parse_args(args)
   219      credentials = get_credentials(parsed_args)
   220      return parsed_args, credentials
   221  
   222  
   223  def get_credentials(args):
   224      if 'user' not in args:
   225          return None
   226      if None in (args.user, args.password):
   227          raise CredentialsMissing(
   228              'Jenkins username and/or password not supplied.')
   229          return None
   230      return Credentials(args.user, args.password)
   231  
   232  
   233  class Namer:
   234      """A base class that has distro and arch info used to name things."""
   235  
   236      @classmethod
   237      def factory(cls):
   238          dist_info = get_distro_information()
   239          return cls(get_deb_arch(), dist_info['RELEASE'], dist_info['CODENAME'])
   240  
   241      def __init__(self, arch, distro_release, distro_series):
   242          self.arch = arch
   243          self.distro_release = distro_release
   244          self.distro_series = distro_series
   245  
   246  
   247  class PackageNamer(Namer):
   248      """A class knows the names of packages."""
   249  
   250      def get_release_package_suffix(self):
   251          return '-0ubuntu1~{distro_release}.1~juju1_{arch}.deb'.format(
   252              distro_release=self.distro_release, arch=self.arch)
   253  
   254      def get_release_package(self, version):
   255          return (
   256              'juju-core_{version}{suffix}'
   257              ).format(version=version, suffix=self.get_release_package_suffix())
   258  
   259      def get_certification_package(self, version):
   260          return (
   261              'juju-core_{version}~{distro_release}.1_{arch}.deb'
   262              ).format(version=version, distro_release=self.distro_release,
   263                       arch=self.arch)
   264  
   265  
   266  class JobNamer(Namer):
   267      """A class knows the names of jobs."""
   268  
   269      def get_build_binary_job(self):
   270          return 'build-binary-{distro_series}-{arch}'.format(
   271              distro_series=self.distro_series, arch=self.arch)
   272  
   273  
   274  def main(argv):
   275      """Manage list and get files from Juju CI builds."""
   276      try:
   277          args, credentials = parse_args(argv)
   278      except CredentialsMissing as e:
   279          print_now(e)
   280          sys.exit(2)
   281      try:
   282          if args.command == 'list':
   283              list_artifacts(
   284                  credentials, args.job, args.build, args.glob,
   285                  verbose=args.verbose)
   286          elif args.command == 'setup-workspace':
   287              setup_workspace(
   288                  args.path, dry_run=args.dry_run, verbose=args.verbose)
   289      except Exception as e:
   290          print_now(e)
   291          if args.verbose:
   292              traceback.print_tb(sys.exc_info()[2])
   293          return 2
   294      if args.verbose:
   295          print_now("Done.")
   296      return 0
   297  
   298  
   299  if __name__ == '__main__':
   300      sys.exit(main(sys.argv[1:]))