github.com/sagansystems/goofys-app@v0.19.1-0.20180410053237-b2302fdf5af9/bench/bench.sh (about) 1 #!/bin/bash 2 3 : ${TRAVIS:="false"} 4 : ${FAST:="false"} 5 : ${test:=""} 6 : ${CACHE:="false"} 7 8 iter=10 9 10 if [ "$TRAVIS" != "false" ]; then 11 set -o xtrace 12 iter=1 13 fi 14 15 if [ "$FAST" != "false" ]; then 16 iter=1 17 fi 18 19 set -o errexit 20 set -o nounset 21 22 if [ $# -lt 2 ]; then 23 echo "Usage: $0 <mount cmd> <dir>" 24 exit 1 25 fi 26 27 cmd=$1 28 mnt=$2 29 if [ $# -gt 2 ]; then 30 t=$3 31 else 32 t= 33 fi 34 35 prefix=$mnt/test_dir 36 37 MOUNTED=0 38 39 if [[ "$cmd" == riofs* ]]; then 40 RIOFS="true" 41 else 42 RIOFS="false" 43 fi 44 45 $cmd >& mount.log & 46 PID=$! 47 48 function cleanup { 49 if [ $MOUNTED == 1 ]; then 50 popd >/dev/null 51 if [ "$TRAVIS" != "false" ]; then 52 rmdir $prefix 53 else 54 rmdir $prefix >& /dev/null || true # riofs doesn't support rmdir 55 fi 56 fi 57 58 if [ "$PID" != "" ]; then 59 kill $PID >& /dev/null || true 60 fusermount -u $mnt >& /dev/null || true 61 sleep 1 62 fi 63 } 64 65 function cleanup_err { 66 err=$? 67 cat mount.log 68 if [ $MOUNTED == 1 ]; then 69 popd >&/dev/null || true 70 rmdir $prefix >&/dev/null || true 71 fi 72 73 if [ "$PID" != "" ]; then 74 kill $PID >& /dev/null || true 75 fusermount -u $mnt >& /dev/null || true 76 fi 77 78 return $err 79 } 80 81 trap cleanup EXIT 82 trap cleanup_err ERR 83 84 if [ "$TRAVIS" == "false" -a "$cmd" != "cat" ]; then 85 for i in $(seq 1 10); do 86 if grep -q $mnt /proc/mounts; then 87 break 88 fi 89 sleep 1 90 done 91 if ! grep -q $mnt /proc/mounts; then 92 echo "$mnt not mounted by $cmd" 93 cat mount.log 94 exit 1 95 fi 96 MOUNTED=1 97 else 98 # in travis we mount things externally so we know we are mounted 99 MOUNTED=1 100 fi 101 102 mkdir -p "$prefix" 103 pushd "$prefix" >/dev/null 104 105 SUDO= 106 if [ $(id -u) != 0 ]; then 107 SUDO=sudo 108 fi 109 110 function drop_cache { 111 if [ "$TRAVIS" == "false" ]; then 112 (echo 3 | $SUDO tee /proc/sys/vm/drop_caches) > /dev/null 113 fi 114 } 115 116 export TIMEFORMAT=%R 117 118 function run_test { 119 test=$1 120 drop_cache 121 sleep 2 122 if [ "$CACHE" == "false" ]; then 123 # make sure riofs cache get cleared 124 if [ -d /tmp/cache ]; then 125 cache=$(ls -1 /tmp/cache) 126 rm -Rf /tmp/cache 2>/dev/null || true 127 mkdir -p /tmp/cache/$cache 128 fi 129 fi 130 echo -n "$test " 131 if [ $# -gt 1 ]; then 132 time $test $2 133 else 134 time $test 135 fi 136 } 137 138 function get_howmany { 139 if [ "$TRAVIS" != "false" ]; then 140 howmany=10 141 else 142 if [ $# == 0 ]; then 143 howmany=100 144 else 145 howmany=$1 146 fi 147 fi 148 } 149 150 function create_files { 151 get_howmany $@ 152 153 for i in $(seq 1 $howmany); do 154 echo $i > file$i 155 done 156 } 157 158 function ls_files { 159 get_howmany $@ 160 # people usually use ls in the terminal when color is on 161 numfiles=$(ls -1 --color=always | wc -l) 162 if [ "$numfiles" != "$howmany" ]; then 163 echo "$numfiles != $howmany" 164 false 165 fi 166 } 167 168 function rm_files { 169 get_howmany $@ 170 171 for i in $(seq 1 $howmany); do 172 rm file$i >&/dev/null || true 173 done 174 } 175 176 function find_files { 177 numfiles=$(find | wc -l) 178 179 if [ "$numfiles" != 820 ]; then 180 echo "$numfiles != 820" 181 rm_tree 182 exit 1 183 fi 184 } 185 186 function create_tree_parallel { 187 (for i in $(seq 1 9); do 188 mkdir $i 189 for j in $(seq 1 9); do 190 mkdir $i/$j 191 192 for k in $(seq 1 9); do 193 touch $i/$j/$k & true 194 done 195 done 196 done 197 wait) 198 } 199 200 function rm_tree { 201 for i in $(seq 1 9); do 202 if [ "$TRAVIS" != "false" ]; then 203 rm -Rf $i 204 else 205 rm -Rf $i >& /dev/null || true # riofs doesn't support rmdir 206 fi 207 done 208 } 209 210 function create_files_parallel { 211 get_howmany $@ 212 213 (for i in $(seq 1 $howmany); do 214 echo $i > file$i & true 215 done 216 wait) 217 } 218 219 function rm_files_parallel { 220 get_howmany $@ 221 222 (for i in $(seq 1 $howmany); do 223 rm file$i & true 224 done 225 wait) 226 } 227 228 function write_large_file { 229 count=1000 230 if [ "$FAST" == "true" ]; then 231 count=100 232 fi 233 dd if=/dev/zero of=largefile bs=1MB count=$count oflag=nocache status=none 234 } 235 236 function read_large_file { 237 dd if=largefile of=/dev/null bs=1MB iflag=nocache status=none 238 } 239 240 function read_first_byte { 241 dd if=largefile of=/dev/null bs=1 count=1 iflag=nocache status=none 242 } 243 244 if [ "$t" = "" -o "$t" = "create" ]; then 245 for i in $(seq 1 $iter); do 246 run_test create_files 247 run_test rm_files 248 done 249 fi 250 251 if [ "$t" = "" -o "$t" = "create_parallel" ]; then 252 for i in $(seq 1 $iter); do 253 run_test create_files_parallel 254 run_test rm_files_parallel 255 done 256 fi 257 258 function write_md5 { 259 seed=$(dd if=/dev/urandom bs=128 count=1 status=none | base64 -w 0) 260 random_cmd="openssl enc -aes-256-ctr -pass pass:$seed -nosalt" 261 count=1000 262 if [ "$FAST" == "true" ]; then 263 count=100 264 fi 265 MD5=$(dd if=/dev/zero bs=1MB count=$count status=none | $random_cmd | \ 266 tee >(md5sum) >(dd of=largefile bs=1MB oflag=nocache status=none) >/dev/null | cut -f 1 '-d ') 267 if [ "$RIOFS" == "true" ]; then 268 # riofs doesn't wait for flush, so we need to wait for object to show up 269 # XXX kind of broken due to eventual consistency but it's hte best we can do 270 while ! aws s3api --endpoint ${ENDPOINT} head-object --bucket ${BUCKET} --key test_dir/largefile >& /dev/null; do sleep 0.1; done 271 fi 272 } 273 274 function read_md5 { 275 READ_MD5=$(md5sum largefile | cut -f 1 '-d ') 276 if [ "$READ_MD5" != "$MD5" ]; then 277 echo "$READ_MD5 != $MD5" >&2 278 rm largefile 279 exit 1 280 fi 281 } 282 283 function rm { 284 if [ "$RIOFS" == "true" ]; then 285 while ! /bin/rm $@; do true; done 286 else 287 /bin/rm $@ 288 fi 289 } 290 291 if [ "$t" = "" -o "$t" = "io" ]; then 292 for i in $(seq 1 $iter); do 293 run_test write_md5 294 if [ "$CACHE" != "true" ]; then 295 run_test read_md5 296 run_test read_first_byte 297 fi 298 rm largefile 299 done 300 if [ "$CACHE" = "true" ]; then 301 write_md5 302 read_md5 303 for i in $(seq 1 $iter); do 304 run_test read_md5 305 run_test read_first_byte 306 done 307 rm largefile 308 fi 309 fi 310 311 if [ "$t" = "" -o "$t" = "ls" ]; then 312 create_files_parallel 1000 313 for i in $(seq 1 $iter); do 314 run_test ls_files 1000 315 done 316 rm_files 1000 317 fi 318 319 if [ "$t" = "ls_create" ]; then 320 create_files_parallel 1000 321 test=dummy 322 sleep 10 323 fi 324 325 if [ "$t" = "ls_ls" ]; then 326 run_test ls_files 1000 327 fi 328 329 if [ "$t" = "ls_rm" ]; then 330 rm_files 1000 331 test=dummy 332 fi 333 334 if [ "$t" = "" -o "$t" = "find" ]; then 335 create_tree_parallel 336 for i in $(seq 1 $iter); do 337 run_test find_files 338 done 339 rm_tree 340 fi 341 342 if [ "$t" = "find_create" ]; then 343 create_tree_parallel 344 test=dummy 345 sleep 10 346 fi 347 348 if [ "$t" = "find_find" ]; then 349 for i in $(seq 1 $iter); do 350 run_test find_files 351 done 352 fi 353 354 if [ "$t" = "issue231" ]; then 355 run_test write_md5 356 (for i in $(seq 1 20); do 357 run_test read_md5 & true 358 done; wait); 359 rm largefile 360 fi 361 362 363 if [ "$t" = "cleanup" ]; then 364 rm -Rf * 365 test=dummy 366 fi 367 368 # for https://github.com/kahing/goofys/issues/64 369 # quote: There are 5 concurrent transfers gong at a time. 370 # Data file size is often 100-400MB. 371 # Regarding the number of transfers, I think it's about 200 files. 372 # We read from the goofys mounted s3 bucket and write to a local spring webapp using curl. 373 if [ "$t" = "disable" -o "$t" = "issue64" ]; then 374 # setup the files 375 (for i in $(seq 0 9); do 376 dd if=/dev/zero of=file$i bs=1MB count=300 oflag=nocache status=none & true 377 done 378 wait) 379 if [ $? != 0 ]; then 380 exit $? 381 fi 382 383 # 200 files and 5 concurrent transfer means 40 times, do 50 times for good measure 384 (for i in $(seq 0 9); do 385 dd if=file$i of=/dev/null bs=1MB iflag=nocache status=none & 386 done 387 388 for i in $(seq 10 300); do 389 # wait for 1 to finish, then invoke more 390 wait -n 391 running=$(ps -ef | grep ' dd if=' | grep -v grep | sed 's/.*dd if=file\([0-9]\).*/\1/') 392 for i in $(seq 0 9); do 393 if echo $running | grep -v -q $i; then 394 dd if=file$i of=/dev/null bs=1MB iflag=nocache status=none & 395 break 396 fi 397 done 398 done 399 wait) 400 if [ $? != 0 ]; then 401 exit $? 402 fi 403 404 # cleanup 405 (for i in $(seq 0 9); do 406 rm -f file$i & true 407 done 408 wait) 409 fi 410 411 if [ "$test" = "" ]; then 412 echo "No test was run: $t" 413 exit 1 414 fi