github.com/bytedance/sonic@v1.11.7-0.20240517092252-d2edb31b167b/scripts/bench.py (about)

     1  #!/usr/bin/env python3
     2  
     3  # Copyright 2022 ByteDance Inc.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  import tempfile
    18  import os
    19  import subprocess
    20  import argparse
    21  
    22  gbench_prefix = "SONIC_NO_ASYNC_GC=1 %s test -benchmem -run=none "
    23  
    24  def run(cmd):
    25      print(cmd)
    26      if os.system(cmd):
    27          print ("Failed to run cmd: %s"%(cmd))
    28          exit(1)
    29  
    30  def run_s(cmd):
    31      print (cmd)
    32      try:
    33          res = os.popen(cmd)
    34      except subprocess.CalledProcessError as e:
    35          if e.returncode:
    36              print (e.output)
    37              exit(1)
    38      return res.read()
    39  
    40  def run_r(cmd):
    41      print (cmd)
    42      try:
    43          cmds = cmd.split(' ')
    44          data = subprocess.check_output(cmds, stderr=subprocess.STDOUT)
    45      except subprocess.CalledProcessError as e:
    46          if e.returncode:
    47              print (e.output)
    48              exit(1)
    49      return data.decode("utf-8") 
    50  
    51  def compare(args):
    52      # detech current branch.
    53      # result = run_r("git branch")
    54      current_branch = run_s("git status | head -n1 | sed 's/On branch //'")
    55      # for br in result.split('\n'):
    56      #     if br.startswith("* "):
    57      #         current_branch = br.lstrip('* ')
    58      #         break
    59  
    60      if not current_branch:
    61          print ("Failed to detech current branch")
    62          return None
    63      
    64      # get the current diff
    65      (fd, diff) = tempfile.mkstemp()
    66      run("git diff > %s"%diff)
    67  
    68      # early return if currrent is main branch.
    69      print ("Current branch: %s"%(current_branch))
    70      if current_branch == "main":
    71          print ("Cannot compare at the main branch.Please build a new branch")
    72          return None
    73  
    74      # benchmark current branch    
    75      (fd, target) = tempfile.mkstemp(".target.txt")
    76      run("%s %s ./... 2>&1 | tee %s" %(gbench_prefix % "go", args, target))
    77      
    78     
    79      # trying to switch to the latest main branch
    80      run("git checkout -- .")
    81      if current_branch != "main":
    82          run("git checkout main")
    83      run("git pull --allow-unrelated-histories origin main")
    84      # benchmark main branch
    85      (fd, main) = tempfile.mkstemp(".main.txt")
    86      if gobin != "go":
    87          # change GOROOT
    88          run("export GOROOT=%s" % gobin)
    89          run("%s %s ./... 2>&1 | tee %s" %(gbench_prefix % (gobin+"/bin/go")), args, main)
    90      else:
    91          run("%s %s ./... 2>&1 | tee %s" %(gbench_prefix % "go", args, main))
    92  
    93      # diff the result
    94      # benchstat = "go get golang.org/x/perf/cmd/benchstat && go install golang.org/x/perf/cmd/benchstat"
    95      run( "benchstat -sort=delta %s %s"%(main, target))
    96      run("git checkout -- .")
    97  
    98      # restore branch
    99      if current_branch != "main":
   100          run("git checkout %s"%(current_branch))
   101          
   102      run("patch -p1 < %s" % (diff))
   103      return target
   104  
   105  def main():
   106      argparser = argparse.ArgumentParser(description='Tools to test the performance. Example: ./bench.py -b Decoder_Generic_Sonic -c')
   107      argparser.add_argument('-b', '--bench', dest='filter', required=False,
   108          help='Specify the filter for golang benchmark')
   109      argparser.add_argument('-c', '--compare', dest='compare', action='store_true', required=False,
   110          help='Compare with the main benchmarking')
   111      argparser.add_argument('-t', '--times', dest='times', required=False,
   112          help='benchmark the times')
   113      argparser.add_argument('-r', '--repeat_times', dest='count', required=False,
   114          help='benchmark the count')
   115      argparser.add_argument('-go', '--go_bin_path', dest='gobin', required=False,
   116          help='benchmark the count')
   117      args = argparser.parse_args()
   118      
   119      global gobin 
   120      if args.gobin:
   121          gobin = args.gobin
   122      else:
   123          gobin = "go"
   124      
   125      if args.filter:
   126          gbench_args = "-bench=%s"%(args.filter)
   127      else:
   128          gbench_args = "-bench=."
   129          
   130      if args.times:
   131          gbench_args += " -benchtime=%s"%(args.times)
   132          
   133      if args.count:
   134          gbench_args += " -count=%s"%(args.count)
   135      else:
   136          gbench_args += " -count=10"
   137  
   138      if args.compare:
   139          target = compare(gbench_args)
   140      else:
   141          target = None
   142  
   143      if not target:
   144          (fd, target) = tempfile.mkstemp(".target.txt")
   145          run("%s %s ./... 2>&1 | tee %s" %(gbench_prefix, gbench_args, target))
   146  
   147  if __name__ == "__main__":
   148      main()