github.com/google/grumpy@v0.0.0-20171122020858-3ec87959189c/tools/benchcmp (about)

     1  #!/usr/bin/env python
     2  
     3  # Copyright 2016 Google Inc. All Rights Reserved.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  """Runs two benchmark programs and compares their results."""
    18  
    19  import argparse
    20  import subprocess
    21  import sys
    22  
    23  
    24  parser = argparse.ArgumentParser()
    25  parser.add_argument('prog1')
    26  parser.add_argument('prog2')
    27  parser.add_argument('--runs', default=1, type=int,
    28                      help='number of times to run each program')
    29  
    30  
    31  def main(args):
    32    results1 = _RunBenchmark(args.prog1)
    33    benchmarks = set(results1.keys())
    34    results2 = {}
    35    for _ in xrange(args.runs - 1):
    36      _MergeResults(results1, _RunBenchmark(args.prog1), benchmarks)
    37      _MergeResults(results2, _RunBenchmark(args.prog2), benchmarks)
    38    _MergeResults(results2, _RunBenchmark(args.prog2), benchmarks)
    39    for b in sorted(benchmarks):
    40      print b, '{:+.1%}'.format(results2[b] / results1[b] - 1)
    41  
    42  
    43  def _MergeResults(merged, results, benchmarks):
    44    benchmarks = set(benchmarks)
    45    for k, v in results.iteritems():
    46      if k not in benchmarks:
    47        _Die('unmatched benchmark: {}', k)
    48      merged[k] = max(merged.get(k, 0), v)
    49      benchmarks.remove(k)
    50    if benchmarks:
    51      _Die('missing benchmark(s): {}', ', '.join(benchmarks))
    52  
    53  
    54  def _RunBenchmark(prog):
    55    """Executes prog and returns a dict mapping benchmark name -> result."""
    56    try:
    57      p = subprocess.Popen([prog], shell=True, stdout=subprocess.PIPE)
    58    except OSError as e:
    59      _Die(e)
    60    out, _ = p.communicate()
    61    if p.returncode:
    62      _Die('{} exited with status: {}', prog, p.returncode)
    63    results = {}
    64    for line in out.splitlines():
    65      line = line.strip()
    66      if not line:
    67        continue
    68      parts = line.split()
    69      if len(parts) != 3:
    70        _Die('invalid benchmark output: {}', line)
    71      name, status, result = parts
    72      if status != 'PASSED':
    73        _Die('benchmark failed: {}', line)
    74      try:
    75        result = float(result)
    76      except ValueError:
    77        _Die('invalid benchmark result: {}', line)
    78      results[name] = result
    79    return results
    80  
    81  
    82  def _Die(msg, *args):
    83    if args:
    84      msg = msg.format(*args)
    85    print >> sys.stderr, msg
    86    sys.exit(1)
    87  
    88  
    89  if __name__ == '__main__':
    90    main(parser.parse_args())