github.com/mit-dci/lit@v0.0.0-20221102210550-8c3d3b49f2ce/test/runtests.py (about)

     1  #!/usr/bin/env python3
     2  
     3  import os
     4  import sys
     5  import time
     6  import importlib.util as imu
     7  
     8  import testlib
     9  
    10  # Loads a python module from a path, giving it a particular name.
    11  def __load_mod_from_file(path, name):
    12      spec = imu.spec_from_file_location(name, path)
    13      mod = imu.module_from_spec(spec)
    14      spec.loader.exec_module(mod)
    15      return mod
    16  
    17  # Parses a line like this:
    18  #
    19  #   foobar 3 run_test_foo
    20  #
    21  # (The last argument is optional.)
    22  def __parse_test_def(line):
    23      parts = line.split(' ')
    24      if len(parts) != 2 and len(parts) != 3:
    25          return None
    26      name = parts[0]
    27      nodes = parts[1]
    28      func = 'run_test'
    29      if len(parts) == 3:
    30          func = parts[2]
    31      return {
    32          'test_name': name.strip(),
    33          'func_name': func.strip(),
    34          'node_cnt': int(nodes.strip()), # Raises if fails.
    35      }
    36  
    37  def test_name_from_filename(name):
    38      if name.startswith('itest_') and name.endswith('.py'):
    39          return name[6:-3] # Should just cut off the ends.
    40      else:
    41          raise ValueError('')
    42  
    43  def parse_tests_file(path):
    44      tdefs = []
    45      with open(path, 'r') as f:
    46          for l in f.readlines():
    47              if len(l) == 0 or l.startswith('#'):
    48                  continue
    49              t = __parse_test_def(l)
    50              if t is not None:
    51                  tdefs.append(t)
    52      return tdefs
    53  
    54  def load_tests_from_file(path):
    55      tests = []
    56      pdir = os.path.dirname(path)
    57      mods = {}
    58      for t in parse_tests_file(path):
    59          tname = t['test_name']
    60          mod = None
    61          if tname in mods:
    62              mod = mods[tname]
    63          else:
    64              fname = 'itest_%s.py' % tname
    65              modname = 'testmod_' + tname
    66              mod = __load_mod_from_file(fname, modname)
    67              mods[tname] = mod
    68          pretty = tname
    69          tfn = getattr(mod, t['func_name'])
    70          if tfn.__name__ != 'run_test':
    71              pretty += ':' + tfn.__name__
    72          tests.append({
    73              'name': tname,
    74              'pretty_name': pretty,
    75              'test_func': tfn,
    76              'node_cnt': t['node_cnt']
    77          })
    78      return tests
    79  
    80  def run_test_list(tests):
    81      ok = 0
    82      failed = []
    83  
    84      # First, just run the tests.
    85      for t in tests:
    86          pname = t['pretty_name']
    87  
    88          print('==============================')
    89          tfunc = t['test_func']
    90          print('Running test:', pname)
    91  
    92          # Do this before the bottom frame so we have a clue how long startup
    93          # took and where the fail was.
    94          env = None
    95          try:
    96              testlib.clean_data_dir() # IMPORTANT!
    97              print('------------------------------')
    98              env = testlib.TestEnv(t['node_cnt'])
    99          except Exception as e:
   100              print('Error initing env, this is a test framework bug:', e)
   101              break
   102          print('==============================')
   103  
   104          # This is where the test actually runs.
   105          try:
   106              tfunc(env)
   107              env.shutdown()
   108              print('------------------------------')
   109              print('Success:', pname)
   110              ok += 1
   111              time.sleep(0.1) # Wait for things to exit, just to be sure.
   112          except BaseException as e:
   113              env.shutdown()
   114              print('------------------------------')
   115              print('Failure:', pname)
   116              print('\nError:', e)
   117              failed.append(t)
   118              if type(e) is KeyboardInterrupt:
   119                  break
   120              # TODO Report failures and why.
   121  
   122      print('==============================')
   123  
   124      # Collect results.
   125      res = {
   126          'ok': ok,
   127          'fail': len(failed),
   128          'ignored': len(tests) - ok - len(failed),
   129          'failed': failed
   130      }
   131      return res
   132  
   133  if __name__ == '__main__':
   134      os.makedirs('_data', exist_ok=True)
   135      tests = load_tests_from_file('tests.txt')
   136  
   137      # If given arguments, run these instead.  Doesn't do them in given order, sadly.
   138      if len(sys.argv) > 1:
   139          to_run = []
   140          for t in tests:
   141              if t['name'] in sys.argv[1:]:
   142                  to_run.append(t)
   143          tests = to_run
   144  
   145      res = run_test_list(tests)
   146      print('Success:', res['ok'])
   147      print('Failure:', res['fail'])
   148      print('Ignored:', res['ignored'])
   149      if res['fail'] > 0:
   150          print('\nFailures:')
   151          for f in res['failed']:
   152              print(' - %s' % f['pretty_name'])
   153          sys.exit(1)
   154      else:
   155          sys.exit(0)