github.com/sagansystems/goofys-app@v0.19.1-0.20180410053237-b2302fdf5af9/bench/bench.sh (about)

     1  #!/bin/bash
     2  
     3  : ${TRAVIS:="false"}
     4  : ${FAST:="false"}
     5  : ${test:=""}
     6  : ${CACHE:="false"}
     7  
     8  iter=10
     9  
    10  if [ "$TRAVIS" != "false" ]; then
    11      set -o xtrace
    12      iter=1
    13  fi
    14  
    15  if [ "$FAST" != "false" ]; then
    16      iter=1
    17  fi
    18  
    19  set -o errexit
    20  set -o nounset
    21  
    22  if [ $# -lt 2 ]; then
    23      echo "Usage: $0 <mount cmd> <dir>"
    24      exit 1
    25  fi
    26  
    27  cmd=$1
    28  mnt=$2
    29  if [ $# -gt 2 ]; then
    30      t=$3
    31  else
    32      t=
    33  fi
    34  
    35  prefix=$mnt/test_dir
    36  
    37  MOUNTED=0
    38  
    39  if [[ "$cmd" == riofs* ]]; then
    40      RIOFS="true"
    41  else
    42      RIOFS="false"
    43  fi
    44  
    45  $cmd >& mount.log &
    46  PID=$!
    47  
    48  function cleanup {
    49      if [ $MOUNTED == 1 ]; then
    50          popd >/dev/null
    51          if [ "$TRAVIS" != "false" ]; then
    52              rmdir $prefix
    53          else
    54              rmdir $prefix >& /dev/null || true # riofs doesn't support rmdir
    55          fi
    56      fi
    57  
    58      if [ "$PID" != "" ]; then
    59          kill $PID >& /dev/null || true
    60          fusermount -u $mnt >& /dev/null || true
    61          sleep 1
    62      fi
    63  }
    64  
    65  function cleanup_err {
    66      err=$?
    67      cat mount.log
    68      if [ $MOUNTED == 1 ]; then
    69          popd >&/dev/null || true
    70          rmdir $prefix >&/dev/null || true
    71      fi
    72  
    73      if [ "$PID" != "" ]; then
    74          kill $PID >& /dev/null || true
    75          fusermount -u $mnt >& /dev/null || true
    76      fi
    77  
    78      return $err
    79  }
    80  
    81  trap cleanup EXIT
    82  trap cleanup_err ERR
    83  
    84  if [ "$TRAVIS" == "false" -a "$cmd" != "cat" ]; then
    85      for i in $(seq 1 10); do
    86          if grep -q $mnt /proc/mounts; then
    87              break
    88          fi
    89          sleep 1
    90      done
    91      if ! grep -q $mnt /proc/mounts; then
    92          echo "$mnt not mounted by $cmd"
    93          cat mount.log
    94          exit 1
    95      fi
    96      MOUNTED=1
    97  else
    98      # in travis we mount things externally so we know we are mounted
    99      MOUNTED=1
   100  fi
   101  
   102  mkdir -p "$prefix"
   103  pushd "$prefix" >/dev/null
   104  
   105  SUDO=
   106  if [ $(id -u) != 0 ]; then
   107      SUDO=sudo
   108  fi
   109  
   110  function drop_cache {
   111      if [ "$TRAVIS" == "false" ]; then
   112          (echo 3 | $SUDO tee /proc/sys/vm/drop_caches) > /dev/null
   113      fi
   114  }
   115  
   116  export TIMEFORMAT=%R
   117  
   118  function run_test {
   119      test=$1
   120      drop_cache
   121      sleep 2
   122      if [ "$CACHE" == "false" ]; then
   123          # make sure riofs cache get cleared
   124          if [ -d /tmp/cache ]; then
   125              cache=$(ls -1 /tmp/cache)
   126              rm -Rf /tmp/cache 2>/dev/null || true
   127              mkdir -p /tmp/cache/$cache
   128          fi
   129      fi
   130      echo -n "$test "
   131      if [ $# -gt 1 ]; then
   132          time $test $2
   133      else
   134          time $test
   135      fi
   136  }
   137  
   138  function get_howmany {
   139      if [ "$TRAVIS" != "false" ]; then
   140          howmany=10
   141      else
   142          if [ $# == 0 ]; then
   143              howmany=100
   144          else
   145              howmany=$1
   146          fi
   147      fi
   148  }
   149  
   150  function create_files {
   151      get_howmany $@
   152  
   153      for i in $(seq 1 $howmany); do
   154          echo $i > file$i
   155      done
   156  }
   157  
   158  function ls_files {
   159      get_howmany $@
   160      # people usually use ls in the terminal when color is on
   161      numfiles=$(ls -1 --color=always | wc -l)
   162      if [ "$numfiles" != "$howmany" ]; then
   163          echo "$numfiles != $howmany"
   164          false
   165      fi
   166  }
   167  
   168  function rm_files {
   169      get_howmany $@
   170  
   171      for i in $(seq 1 $howmany); do
   172          rm file$i >&/dev/null || true
   173      done
   174  }
   175  
   176  function find_files {
   177      numfiles=$(find | wc -l)
   178  
   179      if [ "$numfiles" != 820 ]; then
   180          echo "$numfiles != 820"
   181          rm_tree
   182          exit 1
   183      fi
   184  }
   185  
   186  function create_tree_parallel {
   187      (for i in $(seq 1 9); do
   188          mkdir $i
   189          for j in $(seq 1 9); do
   190              mkdir $i/$j
   191  
   192              for k in $(seq 1 9); do
   193                   touch $i/$j/$k & true
   194               done
   195           done
   196      done
   197      wait)
   198  }
   199  
   200  function rm_tree {
   201      for i in $(seq 1 9); do
   202          if [ "$TRAVIS" != "false" ]; then
   203              rm -Rf $i
   204          else
   205              rm -Rf $i >& /dev/null || true # riofs doesn't support rmdir
   206          fi
   207      done
   208  }
   209  
   210  function create_files_parallel {
   211      get_howmany $@
   212  
   213      (for i in $(seq 1 $howmany); do
   214          echo $i > file$i & true
   215      done
   216      wait)
   217  }
   218  
   219  function rm_files_parallel {
   220      get_howmany $@
   221  
   222      (for i in $(seq 1 $howmany); do
   223          rm file$i & true
   224      done
   225      wait)
   226  }
   227  
   228  function write_large_file {
   229      count=1000
   230      if [ "$FAST" == "true" ]; then
   231          count=100
   232      fi
   233      dd if=/dev/zero of=largefile bs=1MB count=$count oflag=nocache status=none
   234  }
   235  
   236  function read_large_file {
   237      dd if=largefile of=/dev/null bs=1MB iflag=nocache status=none
   238  }
   239  
   240  function read_first_byte {
   241      dd if=largefile of=/dev/null bs=1 count=1 iflag=nocache status=none
   242  }
   243  
   244  if [ "$t" = "" -o "$t" = "create" ]; then
   245      for i in $(seq 1 $iter); do
   246          run_test create_files
   247          run_test rm_files
   248      done
   249  fi
   250  
   251  if [ "$t" = "" -o "$t" = "create_parallel" ]; then
   252      for i in $(seq 1 $iter); do
   253          run_test create_files_parallel
   254          run_test rm_files_parallel
   255      done
   256  fi
   257  
   258  function write_md5 {
   259      seed=$(dd if=/dev/urandom bs=128 count=1 status=none | base64 -w 0)
   260      random_cmd="openssl enc -aes-256-ctr -pass pass:$seed -nosalt"
   261      count=1000
   262      if [ "$FAST" == "true" ]; then
   263          count=100
   264      fi
   265      MD5=$(dd if=/dev/zero bs=1MB count=$count status=none | $random_cmd | \
   266          tee >(md5sum) >(dd of=largefile bs=1MB oflag=nocache status=none) >/dev/null | cut -f 1 '-d ')
   267      if [ "$RIOFS" == "true" ]; then
   268          # riofs doesn't wait for flush, so we need to wait for object to show up
   269          # XXX kind of broken due to eventual consistency but it's hte best we can do
   270          while ! aws s3api --endpoint ${ENDPOINT} head-object --bucket ${BUCKET} --key test_dir/largefile >& /dev/null; do sleep 0.1; done
   271      fi
   272  }
   273  
   274  function read_md5 {
   275      READ_MD5=$(md5sum largefile | cut -f 1 '-d ')
   276      if [ "$READ_MD5" != "$MD5" ]; then
   277          echo "$READ_MD5 != $MD5" >&2
   278          rm largefile
   279          exit 1
   280      fi
   281  }
   282  
   283  function rm {
   284      if [ "$RIOFS" == "true" ]; then
   285          while ! /bin/rm $@; do true; done
   286      else
   287          /bin/rm $@
   288      fi
   289  }
   290  
   291  if [ "$t" = "" -o "$t" = "io" ]; then
   292      for i in $(seq 1 $iter); do
   293          run_test write_md5
   294          if [ "$CACHE" != "true" ]; then
   295              run_test read_md5
   296              run_test read_first_byte
   297          fi
   298          rm largefile
   299      done
   300      if [ "$CACHE" = "true" ]; then
   301          write_md5
   302          read_md5
   303          for i in $(seq 1 $iter); do
   304              run_test read_md5
   305              run_test read_first_byte
   306          done
   307          rm largefile
   308      fi
   309  fi
   310  
   311  if [ "$t" = "" -o "$t" = "ls" ]; then
   312      create_files_parallel 1000
   313      for i in $(seq 1 $iter); do
   314          run_test ls_files 1000
   315      done
   316      rm_files 1000
   317  fi
   318  
   319  if [ "$t" = "ls_create" ]; then
   320      create_files_parallel 1000
   321      test=dummy
   322      sleep 10
   323  fi
   324  
   325  if [ "$t" = "ls_ls" ]; then
   326      run_test ls_files 1000
   327  fi
   328  
   329  if [ "$t" = "ls_rm" ]; then
   330      rm_files 1000
   331      test=dummy
   332  fi
   333  
   334  if [ "$t" = "" -o "$t" = "find" ]; then
   335      create_tree_parallel
   336      for i in $(seq 1 $iter); do
   337          run_test find_files
   338      done
   339      rm_tree
   340  fi
   341  
   342  if [ "$t" = "find_create" ]; then
   343      create_tree_parallel
   344      test=dummy
   345      sleep 10
   346  fi
   347  
   348  if [ "$t" = "find_find" ]; then
   349      for i in $(seq 1 $iter); do
   350          run_test find_files
   351      done
   352  fi
   353  
   354  if [ "$t" = "issue231" ]; then
   355      run_test write_md5
   356      (for i in $(seq 1 20); do
   357           run_test read_md5 & true
   358      done; wait);
   359      rm largefile
   360  fi
   361  
   362  
   363  if [ "$t" = "cleanup" ]; then
   364      rm -Rf *
   365      test=dummy
   366  fi
   367  
   368  # for https://github.com/kahing/goofys/issues/64
   369  # quote: There are 5 concurrent transfers gong at a time.
   370  # Data file size is often 100-400MB.
   371  # Regarding the number of transfers, I think it's about 200 files.
   372  # We read from the goofys mounted s3 bucket and write to a local spring webapp using curl.
   373  if [ "$t" = "disable" -o "$t" = "issue64" ]; then
   374      # setup the files
   375      (for i in $(seq 0 9); do
   376          dd if=/dev/zero of=file$i bs=1MB count=300 oflag=nocache status=none & true
   377      done
   378      wait)
   379      if [ $? != 0 ]; then
   380          exit $?
   381      fi
   382  
   383      # 200 files and 5 concurrent transfer means 40 times, do 50 times for good measure
   384      (for i in $(seq 0 9); do
   385          dd if=file$i of=/dev/null bs=1MB iflag=nocache status=none &
   386      done
   387  
   388      for i in $(seq 10 300); do
   389          # wait for 1 to finish, then invoke more
   390          wait -n
   391          running=$(ps -ef | grep ' dd if=' | grep -v grep | sed 's/.*dd if=file\([0-9]\).*/\1/')
   392          for i in $(seq 0 9); do
   393              if echo $running | grep -v -q $i; then
   394                  dd if=file$i of=/dev/null bs=1MB iflag=nocache status=none &
   395                  break
   396              fi
   397          done
   398      done
   399      wait)
   400      if [ $? != 0 ]; then
   401          exit $?
   402      fi
   403      
   404      # cleanup
   405      (for i in $(seq 0 9); do
   406          rm -f file$i & true
   407      done
   408      wait)
   409  fi
   410  
   411  if [ "$test" = "" ]; then
   412      echo "No test was run: $t"
   413      exit 1
   414  fi