github.com/rohankumardubey/proxyfs@v0.0.0-20210108201508-653efa9ab00e/regression_test.py (about)

     1  #!/usr/bin/env python
     2  
     3  """
     4  A script to lint and test ProxyFS project code.
     5  """
     6  
     7  from __future__ import print_function, unicode_literals
     8  from threading import Timer
     9  
    10  import os
    11  import argparse
    12  import functools
    13  import logging
    14  import platform
    15  import contextlib
    16  import subprocess
    17  import shutil
    18  import sys
    19  import tempfile
    20  import time
    21  
    22  
    23  PACKAGES = ["blunder",
    24              "cleanproxyfs",
    25              "conf",
    26              "dlm",
    27              "fs",
    28              "fsworkout",
    29              "fuse",
    30              "headhunter",
    31              "httpserver",
    32              "inode",
    33              "inodeworkout",
    34              "jrpcfs",
    35              "logger",
    36              "mkproxyfs", "mkproxyfs/mkproxyfs",
    37              "pfs-stress",
    38              "pfsconfjson", "pfsconfjsonpacked",
    39              "pfsworkout",
    40              "platform",
    41              "proxyfsd", "proxyfsd/proxyfsd",
    42              "ramswift", "ramswift/ramswift",
    43              "stats",
    44              "statslogger",
    45              "swiftclient",
    46              "utils"]
    47  
    48  
    49  COLORS = {"bright red": '1;31', "bright green": '1;32'}
    50  
    51  
    52  @contextlib.contextmanager
    53  def return_to_wd():
    54      curdir = os.getcwd()
    55      try:
    56          yield
    57      finally:
    58          os.chdir(curdir)
    59  
    60  
    61  @contextlib.contextmanager
    62  def self_cleaning_tempdir(*args, **kwargs):
    63      our_tempdir = tempfile.mkdtemp(*args, **kwargs)
    64      try:
    65          yield our_tempdir
    66      finally:
    67          shutil.rmtree(our_tempdir, ignore_errors=True)
    68  
    69  
    70  def proxyfs_binary_path(binary):
    71      try:
    72          gopath = os.environ["GOPATH"]
    73      except KeyError:
    74          color_print("$GOPATH must be set", 'bright red')
    75          os.exit(1)
    76      return os.path.join(gopath, "bin", binary)
    77  
    78  
    79  def color_print(content, color=None):
    80      print("\x1b[{color}m{content}\x1b[0m".format(content=content,
    81                                                   color=COLORS[color]))
    82  
    83  
    84  def proxyfs_package_path(package):
    85      try:
    86          gopath = os.environ["GOPATH"]
    87      except KeyError:
    88          color_print("$GOPATH must be set", 'bright red')
    89          os.exit(1)
    90      return os.path.join(gopath, "src/github.com/swiftstack/ProxyFS", package)
    91  
    92  
    93  def color_print(content, color=None):
    94      print("\x1b[{color}m{content}\x1b[0m".format(content=content,
    95                                                   color=COLORS[color]))
    96  
    97  
    98  def report(task, success=False):
    99      printer = color_print if sys.stdout.isatty() else lambda *a, **kw: print(*a)
   100      if success:
   101          printer("{} {}".format(task, "succeeded!"), color="bright green")
   102      else:
   103          printer("{} {}".format(task, "failed!"), color="bright red")
   104  
   105  
   106  class GoCommand(object):
   107      def __init__(self, command, project_path,
   108                   options=None, report_as=None, skip=False):
   109          self.command = command
   110          self.project_path = project_path
   111          self.options = options or []
   112          self.report_as = report_as
   113          self.skip = skip
   114  
   115      def execute(self, package):
   116          if self.skip:
   117              return None
   118          package_path = "{}{}".format(self.project_path, package)
   119          command_line = ["go", self.command] + self.options + [package_path]
   120          logging.info(' '.join("'{}'".format(s) if ' ' in s else s
   121                                for s in command_line))
   122          success = not bool(subprocess.call(command_line))
   123          return success
   124  
   125  
   126  def get_go_commands(options, project_path, skip_tests=False):
   127      commands = [
   128          GoCommand('fmt', project_path),
   129          GoCommand('get', project_path, ['-t', '-u'], skip=not options.get),
   130          # Source code has to be `go install`ed before `stringer` can run on
   131          # it, so we install before generate, which mysteriously does work?
   132          # see https://github.com/golang/go/issues/10249
   133          GoCommand('install', project_path, ['-gcflags', '-N -l'], report_as="`go install`"),
   134          GoCommand('generate', project_path, report_as="`go generate`"),
   135      ]
   136      if not options.deb_builder:
   137          commands.append(
   138              GoCommand('test', project_path,
   139                        filter(lambda x: x, [options.bench, options.cover]),
   140                        report_as="`go test`",
   141                        skip=skip_tests)
   142          )
   143      commands.append(GoCommand('vet', project_path, report_as="`go vet`"))
   144      return commands
   145  
   146  
   147  def execute_go_commands(commands, packages):
   148      reports = []
   149      failures = 0
   150      for command in commands:
   151          successes = filter(lambda success: success is not None,
   152                             [command.execute(package) for package in packages])
   153          success = all(successes)
   154          failures += not success
   155          if command.report_as is not None:
   156              reports.append(functools.partial(report, command.report_as,
   157                                               success=success))
   158      for reporter in reports:
   159          reporter()
   160      return failures
   161  
   162  
   163  def test_pfs_middleware(options):
   164      failures = 0
   165      with return_to_wd():
   166          proxyfs_dir = os.path.dirname(os.path.abspath(__file__))
   167          pfs_middleware_dir = os.path.join(proxyfs_dir, "pfs_middleware")
   168          os.chdir(pfs_middleware_dir)
   169          failures += bool(subprocess.call((['python', 'setup.py', 'test'])))
   170      report("test_pfs_middleware()", not failures)
   171      return failures
   172  
   173  
   174  def build_proxyfs(options):
   175      commands = get_go_commands(options, "github.com/swiftstack/ProxyFS/")
   176      if options.packages is None:
   177          selected_packages = PACKAGES
   178      else:
   179          selected_packages = options.packages
   180      return execute_go_commands(commands, selected_packages)
   181  
   182  
   183  def build_dependencies(options):
   184      failures = 0
   185      proxyfs_dir = os.path.dirname(os.path.abspath(__file__))
   186      stringer_path = 'golang.org/x/tools/cmd/stringer'
   187      full_stringer_path = os.path.join(proxyfs_dir, "vendor", stringer_path)
   188      print("Building " + stringer_path)
   189      install_cmd = ("go", "install")
   190      with return_to_wd():
   191          os.chdir(full_stringer_path)
   192          success = not(bool(subprocess.call(install_cmd)))
   193          failures += not success
   194      report("build_dependencies()", not failures)
   195      return failures
   196  
   197  
   198  def build_jrpcclient(options):
   199      proxyfs_dir = os.path.dirname(os.path.abspath(__file__))
   200      jrpcclient_dir = os.path.join(proxyfs_dir, "jrpcclient")
   201      command = ['./regression_test.py']
   202      if options.no_install:
   203          command.append('--no-install')
   204      if options.deb_builder:
   205          command.append('--deb-builder')
   206      if options.just_build_libs:
   207          command.append('--just-build-libs')
   208      return bool(subprocess.call(command, cwd=jrpcclient_dir))
   209  
   210  
   211  def build_vfs(options):
   212      proxyfs_dir = os.path.dirname(os.path.abspath(__file__))
   213      vfs_dir = os.path.join(proxyfs_dir, "vfs")
   214      failures = 0
   215      clean_cmd = ('make', 'clean')
   216      failures += bool(subprocess.call(clean_cmd, cwd=vfs_dir))
   217      if failures:
   218          return failures
   219      failures += bool(subprocess.call('make', cwd=vfs_dir))
   220      if failures:
   221          return failures
   222      distro = platform.linux_distribution()[0]
   223      if not options.no_install:
   224          if options.deb_builder:
   225              if 'centos' in distro.lower():
   226                  install_cmd = ('make', 'installcentos')
   227              else:
   228                  install_cmd = ('make', 'install')
   229          else:
   230              if 'centos' in distro.lower():
   231                  install_cmd = ('sudo', '-E', 'make', 'installcentos')
   232              else:
   233                  install_cmd = ('sudo', '-E', 'make', 'install')
   234          failures += bool(subprocess.call(install_cmd, cwd=vfs_dir))
   235      return failures
   236  
   237  
   238  def main(options):
   239      failures = 0
   240  
   241      if not options.just_libs and not options.just_build_libs:
   242  
   243          go_version = subprocess.check_output((['go', 'version']))
   244          color_print(go_version[:-1], "bright green")
   245  
   246          if not options.quiet:
   247              logging.basicConfig(format="%(message)s", level=logging.INFO)
   248  
   249          failures += build_dependencies(options)
   250          if failures:
   251              return failures
   252  
   253          failures += build_proxyfs(options)
   254          if failures:
   255              return failures
   256  
   257      if platform.system() != "Darwin":
   258  
   259          if not options.no_libs:
   260  
   261              failures += build_jrpcclient(options)
   262              if failures:
   263                  return failures
   264  
   265              failures += build_vfs(options)
   266              if failures:
   267                  return failures
   268  
   269      return failures
   270  
   271  
   272  if __name__ == "__main__":
   273      arg_parser = argparse.ArgumentParser(description=__doc__)
   274      arg_parser.add_argument('--bench', '-bench',
   275                              action='store_const', const='-bench=.',
   276                              help="include benchmark measurements in test output")
   277      arg_parser.add_argument('--cover', '-cover',
   278                              action='store_const', const='-cover',
   279                              help="include coverage statistics in test output")
   280      arg_parser.add_argument('--get', '-get', action='store_true',
   281                              help="invoke `go get` to retrieve new dependencies")
   282      arg_parser.add_argument('--packages', '-p', action='store', nargs='*',
   283                              help="specific packages to process")
   284      arg_parser.add_argument('--no-install', action='store_true',
   285                              help="When building C libraries, do not attempt "
   286                                   "to install resulting objects")
   287      arg_parser.add_argument('--deb-builder', action='store_true',
   288                              help="Modify commands to run inside "
   289                                   "swift-deb-builder")
   290      arg_parser.add_argument('--quiet', '-q', action='store_true',
   291                              help="suppress printing of what commands are being run")
   292      libs_group = arg_parser.add_mutually_exclusive_group()
   293      libs_group.add_argument('--no-libs', action='store_true',
   294                              help="don't build C libraries or run C tests")
   295      libs_group.add_argument('--just-build-libs', action='store_true',
   296                              help="only build C libraries")
   297      libs_group.add_argument('--just-libs', action='store_true',
   298                              help="only build C libraries and run C tests")
   299      options = arg_parser.parse_args()
   300  
   301      exit(main(options))