github.com/google/grumpy@v0.0.0-20171122020858-3ec87959189c/tools/benchcmp (about) 1 #!/usr/bin/env python 2 3 # Copyright 2016 Google Inc. All Rights Reserved. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Runs two benchmark programs and compares their results.""" 18 19 import argparse 20 import subprocess 21 import sys 22 23 24 parser = argparse.ArgumentParser() 25 parser.add_argument('prog1') 26 parser.add_argument('prog2') 27 parser.add_argument('--runs', default=1, type=int, 28 help='number of times to run each program') 29 30 31 def main(args): 32 results1 = _RunBenchmark(args.prog1) 33 benchmarks = set(results1.keys()) 34 results2 = {} 35 for _ in xrange(args.runs - 1): 36 _MergeResults(results1, _RunBenchmark(args.prog1), benchmarks) 37 _MergeResults(results2, _RunBenchmark(args.prog2), benchmarks) 38 _MergeResults(results2, _RunBenchmark(args.prog2), benchmarks) 39 for b in sorted(benchmarks): 40 print b, '{:+.1%}'.format(results2[b] / results1[b] - 1) 41 42 43 def _MergeResults(merged, results, benchmarks): 44 benchmarks = set(benchmarks) 45 for k, v in results.iteritems(): 46 if k not in benchmarks: 47 _Die('unmatched benchmark: {}', k) 48 merged[k] = max(merged.get(k, 0), v) 49 benchmarks.remove(k) 50 if benchmarks: 51 _Die('missing benchmark(s): {}', ', '.join(benchmarks)) 52 53 54 def _RunBenchmark(prog): 55 """Executes prog and returns a dict mapping benchmark name -> result.""" 56 try: 57 p = subprocess.Popen([prog], shell=True, stdout=subprocess.PIPE) 58 except OSError as e: 59 _Die(e) 60 out, _ = p.communicate() 61 if p.returncode: 62 _Die('{} exited with status: {}', prog, p.returncode) 63 results = {} 64 for line in out.splitlines(): 65 line = line.strip() 66 if not line: 67 continue 68 parts = line.split() 69 if len(parts) != 3: 70 _Die('invalid benchmark output: {}', line) 71 name, status, result = parts 72 if status != 'PASSED': 73 _Die('benchmark failed: {}', line) 74 try: 75 result = float(result) 76 except ValueError: 77 _Die('invalid benchmark result: {}', line) 78 results[name] = result 79 return results 80 81 82 def _Die(msg, *args): 83 if args: 84 msg = msg.format(*args) 85 print >> sys.stderr, msg 86 sys.exit(1) 87 88 89 if __name__ == '__main__': 90 main(parser.parse_args())