github.com/kayoticsully/syncthing@v0.8.9-0.20140724133906-c45a2fdc03f8/assets/bootstrap-3.1.1/test-infra/s3_cache.py (about)

     1  #!/usr/bin/env python2.7
     2  from __future__ import absolute_import, unicode_literals, print_function, division
     3  
     4  from sys import argv
     5  from os import environ, stat, remove as _delete_file
     6  from os.path import isfile, dirname, basename, abspath
     7  from hashlib import sha256
     8  from subprocess import check_call as run
     9  
    10  from boto.s3.connection import S3Connection
    11  from boto.s3.key import Key
    12  from boto.exception import S3ResponseError
    13  
    14  
    15  NEED_TO_UPLOAD_MARKER = '.need-to-upload'
    16  BYTES_PER_MB = 1024 * 1024
    17  try:
    18      BUCKET_NAME = environ['TWBS_S3_BUCKET']
    19  except KeyError:
    20      raise SystemExit("TWBS_S3_BUCKET environment variable not set!")
    21  
    22  
    23  def _sha256_of_file(filename):
    24      hasher = sha256()
    25      with open(filename, 'rb') as input_file:
    26          hasher.update(input_file.read())
    27      file_hash = hasher.hexdigest()
    28      print('sha256({}) = {}'.format(filename, file_hash))
    29      return file_hash
    30  
    31  
    32  def _delete_file_quietly(filename):
    33      try:
    34          _delete_file(filename)
    35      except (OSError, IOError):
    36          pass
    37  
    38  
    39  def _tarball_size(directory):
    40      kib = stat(_tarball_filename_for(directory)).st_size // BYTES_PER_MB
    41      return "{} MiB".format(kib)
    42  
    43  
    44  def _tarball_filename_for(directory):
    45      return abspath('./{}.tar.gz'.format(basename(directory)))
    46  
    47  
    48  def _create_tarball(directory):
    49      print("Creating tarball of {}...".format(directory))
    50      run(['tar', '-czf', _tarball_filename_for(directory), '-C', dirname(directory), basename(directory)])
    51  
    52  
    53  def _extract_tarball(directory):
    54      print("Extracting tarball of {}...".format(directory))
    55      run(['tar', '-xzf', _tarball_filename_for(directory), '-C', dirname(directory)])
    56  
    57  
    58  def download(directory):
    59      _delete_file_quietly(NEED_TO_UPLOAD_MARKER)
    60      try:
    61          print("Downloading {} tarball from S3...".format(friendly_name))
    62          key.get_contents_to_filename(_tarball_filename_for(directory))
    63      except S3ResponseError as err:
    64          open(NEED_TO_UPLOAD_MARKER, 'a').close()
    65          print(err)
    66          raise SystemExit("Cached {} download failed!".format(friendly_name))
    67      print("Downloaded {}.".format(_tarball_size(directory)))
    68      _extract_tarball(directory)
    69      print("{} successfully installed from cache.".format(friendly_name))
    70  
    71  
    72  def upload(directory):
    73      _create_tarball(directory)
    74      print("Uploading {} tarball to S3... ({})".format(friendly_name, _tarball_size(directory)))
    75      key.set_contents_from_filename(_tarball_filename_for(directory))
    76      print("{} cache successfully updated.".format(friendly_name))
    77      _delete_file_quietly(NEED_TO_UPLOAD_MARKER)
    78  
    79  
    80  if __name__ == '__main__':
    81      # Uses environment variables:
    82      #   AWS_ACCESS_KEY_ID -- AWS Access Key ID
    83      #   AWS_SECRET_ACCESS_KEY -- AWS Secret Access Key
    84      argv.pop(0)
    85      if len(argv) != 4:
    86          raise SystemExit("USAGE: s3_cache.py <download | upload> <friendly name> <dependencies file> <directory>")
    87      mode, friendly_name, dependencies_file, directory = argv
    88  
    89      conn = S3Connection()
    90      bucket = conn.lookup(BUCKET_NAME, validate=False)
    91      if bucket is None:
    92          raise SystemExit("Could not access bucket!")
    93  
    94      dependencies_file_hash = _sha256_of_file(dependencies_file)
    95  
    96      key = Key(bucket, dependencies_file_hash)
    97      key.storage_class = 'REDUCED_REDUNDANCY'
    98  
    99      if mode == 'download':
   100          download(directory)
   101      elif mode == 'upload':
   102          if isfile(NEED_TO_UPLOAD_MARKER):  # FIXME
   103              upload(directory)
   104          else:
   105              print("No need to upload anything.")
   106      else:
   107          raise SystemExit("Unrecognized mode {!r}".format(mode))