github.com/gaukas/goofys100m@v0.24.0/bench/bench.sh (about) 1 #!/bin/bash 2 3 : ${TRAVIS:="false"} 4 : ${FAST:="false"} 5 : ${test:=""} 6 : ${CACHE:="false"} 7 : ${CLEANUP:="true"} 8 9 iter=10 10 11 if [ "$TRAVIS" != "false" ]; then 12 set -o xtrace 13 iter=1 14 fi 15 16 if [ "$FAST" != "false" ]; then 17 iter=1 18 fi 19 20 set -o errexit 21 set -o nounset 22 23 if [ $# -lt 2 ]; then 24 echo "Usage: $0 <mount cmd> <dir> [test name]" 25 exit 1 26 fi 27 28 cmd=$1 29 mnt=$2 30 if [ $# -gt 2 ]; then 31 t=$3 32 else 33 t= 34 fi 35 36 prefix=$mnt/test_dir 37 38 MOUNTED=0 39 40 if [[ "$cmd" == riofs* ]]; then 41 RIOFS="true" 42 else 43 RIOFS="false" 44 fi 45 46 if [[ "$cmd" == blobfuse* ]]; then 47 BLOBFUSE="true" 48 else 49 BLOBFUSE="false" 50 fi 51 52 $cmd >& mount.log & 53 PID=$! 54 55 function cleanup { 56 if [ $MOUNTED == 1 ]; then 57 popd >/dev/null 58 if [ "$CLEANUP" = "true" ]; then 59 if [ "$TRAVIS" != "false" ]; then 60 rmdir $prefix 61 else 62 rmdir $prefix >& /dev/null || true # riofs doesn't support rmdir 63 fi 64 fi 65 fi 66 67 if [ "$PID" != "" ]; then 68 kill $PID >& /dev/null || true 69 fusermount -u $mnt >& /dev/null || true 70 sleep 1 71 fi 72 } 73 74 function cleanup_err { 75 err=$? 76 cat mount.log 77 if [ $MOUNTED == 1 ]; then 78 popd >&/dev/null || true 79 rmdir $prefix >&/dev/null || true 80 fi 81 82 if [ "$PID" != "" ]; then 83 kill $PID >& /dev/null || true 84 fusermount -u $mnt >& /dev/null || true 85 fi 86 87 return $err 88 } 89 90 trap cleanup EXIT 91 trap cleanup_err ERR 92 93 function wait_for_mount { 94 for i in $(seq 1 10); do 95 if grep -q $mnt /proc/mounts; then 96 break 97 fi 98 sleep 1 99 done 100 if ! grep -q $mnt /proc/mounts; then 101 echo "$mnt not mounted by $cmd" 102 cat mount.log 103 exit 1 104 fi 105 } 106 107 if [ "$TRAVIS" == "false" -a "$cmd" != "cat" ]; then 108 wait_for_mount 109 MOUNTED=1 110 else 111 # in travis we mount things externally so we know we are mounted 112 MOUNTED=1 113 fi 114 115 mkdir -p "$prefix" 116 pushd "$prefix" >/dev/null 117 118 SUDO= 119 if [ $(id -u) != 0 ]; then 120 SUDO=sudo 121 fi 122 123 function drop_cache { 124 if [ "$TRAVIS" == "false" ]; then 125 (echo 3 | $SUDO tee /proc/sys/vm/drop_caches) > /dev/null 126 fi 127 } 128 129 export TIMEFORMAT=%R 130 131 function run_test { 132 test=$1 133 shift 134 drop_cache 135 sleep 2 136 if [ "$CACHE" == "false" ]; then 137 if [ -d /tmp/cache ]; then 138 rm -Rf /tmp/cache/* 139 fi 140 if [ "$BLOBFUSE" == "true" ]; then 141 popd >/dev/null 142 # re-mount blobfuse to cleanup cache 143 if [ "$PID" != "" ]; then 144 fusermount -u $mnt 145 sleep 1 146 fi 147 $cmd >& mount.log & 148 PID=$! 149 wait_for_mount 150 pushd "$prefix" >/dev/null 151 fi 152 fi 153 154 echo -n "$test " 155 if [ $# -gt 1 ]; then 156 time $test $@ 157 else 158 time $test 159 fi 160 } 161 162 function get_howmany { 163 if [ "$TRAVIS" != "false" ]; then 164 if [ $# == 2 ]; then 165 howmany=$2 166 else 167 howmany=10 168 fi 169 else 170 if [ $# == 0 ]; then 171 howmany=100 172 else 173 howmany=$1 174 fi 175 fi 176 } 177 178 function create_files { 179 get_howmany $@ 180 181 for i in $(seq 1 $howmany); do 182 echo $i > file$i 183 done 184 } 185 186 function ls_files { 187 get_howmany $@ 188 # people usually use ls in the terminal when color is on 189 numfiles=$(ls -1 --color=always | wc -l) 190 if [ "$numfiles" != "$howmany" ]; then 191 echo "$numfiles != $howmany" 192 false 193 fi 194 } 195 196 function rm_files { 197 get_howmany $@ 198 199 for i in $(seq 1 $howmany); do 200 rm file$i >&/dev/null || true 201 done 202 } 203 204 function find_files { 205 numfiles=$(find | wc -l) 206 207 if [ "$numfiles" != 820 ]; then 208 echo "$numfiles != 820" 209 rm_tree 210 exit 1 211 fi 212 } 213 214 function create_tree_parallel { 215 (for i in $(seq 1 9); do 216 mkdir $i 217 for j in $(seq 1 9); do 218 mkdir $i/$j 219 220 for k in $(seq 1 9); do 221 touch $i/$j/$k & true 222 done 223 done 224 done 225 wait) 226 } 227 228 function rm_tree { 229 for i in $(seq 1 9); do 230 rm -Rf $i 231 done 232 } 233 234 function create_files_parallel { 235 get_howmany $@ 236 237 (for i in $(seq 1 $howmany); do 238 echo $i > file$i & true 239 done 240 wait) 241 } 242 243 function rm_files_parallel { 244 get_howmany $@ 245 246 (for i in $(seq 1 $howmany); do 247 rm file$i & true 248 done 249 wait) 250 } 251 252 function write_large_file { 253 count=1000 254 if [ "$FAST" == "true" ]; then 255 count=100 256 fi 257 dd if=/dev/zero of=largefile bs=1MB count=$count oflag=nocache status=none 258 } 259 260 function read_large_file { 261 dd if=largefile of=/dev/null bs=1MB iflag=nocache status=none 262 } 263 264 function read_first_byte { 265 dd if=largefile of=/dev/null bs=1 count=1 iflag=nocache status=none 266 } 267 268 if [ "$t" = "" -o "$t" = "create" ]; then 269 for i in $(seq 1 $iter); do 270 run_test create_files 271 run_test rm_files 272 done 273 fi 274 275 if [ "$t" = "" -o "$t" = "create_parallel" ]; then 276 for i in $(seq 1 $iter); do 277 run_test create_files_parallel 278 run_test rm_files_parallel 279 done 280 fi 281 282 function write_md5 { 283 seed=$(dd if=/dev/urandom bs=128 count=1 status=none | base64 -w 0) 284 random_cmd="openssl enc -aes-256-ctr -pbkdf2 -pass pass:$seed -nosalt" 285 count=1000 286 if [ "$FAST" == "true" ]; then 287 count=100 288 fi 289 MD5=$(dd if=/dev/zero bs=1MB count=$count status=none | $random_cmd | \ 290 tee >(md5sum) >(dd of=largefile bs=1MB oflag=nocache status=none) >/dev/null | cut -f 1 '-d ') 291 if [ "$RIOFS" == "true" ]; then 292 # riofs doesn't wait for flush, so we need to wait for object to show up 293 # XXX kind of broken due to eventual consistency but it's hte best we can do 294 while ! aws s3api --endpoint ${ENDPOINT} head-object --bucket ${BUCKET} --key test_dir/largefile >& /dev/null; do sleep 0.1; done 295 fi 296 } 297 298 function read_md5 { 299 READ_MD5=$(md5sum largefile | cut -f 1 '-d ') 300 if [ "$READ_MD5" != "$MD5" ]; then 301 echo "$READ_MD5 != $MD5" >&2 302 rm largefile 303 exit 1 304 fi 305 } 306 307 function rm { 308 if [ "$RIOFS" == "true" ]; then 309 while ! /bin/rm $@; do true; done 310 else 311 /bin/rm $@ 312 fi 313 } 314 315 if [ "$t" = "" -o "$t" = "io" ]; then 316 for i in $(seq 1 $iter); do 317 run_test write_md5 318 if [ "$CACHE" != "true" ]; then 319 run_test read_md5 320 run_test read_first_byte 321 fi 322 rm largefile 323 done 324 if [ "$CACHE" = "true" ]; then 325 write_md5 326 read_md5 327 for i in $(seq 1 $iter); do 328 run_test read_md5 329 run_test read_first_byte 330 done 331 rm largefile 332 fi 333 fi 334 335 if [ "$t" = "" -o "$t" = "ls" ]; then 336 create_files_parallel 2000 2000 337 for i in $(seq 1 $iter); do 338 run_test ls_files 2000 2000 339 done 340 if [ "$CLEANUP" = "true" ]; then 341 rm_files 2000 2000 342 fi 343 fi 344 345 if [ "$t" = "ls_create" ]; then 346 create_files_parallel 1000 347 test=dummy 348 sleep 10 349 fi 350 351 if [ "$t" = "ls_ls" ]; then 352 run_test ls_files 1000 1000 353 fi 354 355 if [ "$t" = "ls_rm" ]; then 356 rm_files 1000 357 test=dummy 358 fi 359 360 if [ "$t" = "" -o "$t" = "find" ]; then 361 create_tree_parallel 362 for i in $(seq 1 $iter); do 363 run_test find_files 364 done 365 rm_tree 366 fi 367 368 if [ "$t" = "find_create" ]; then 369 create_tree_parallel 370 test=dummy 371 sleep 10 372 fi 373 374 if [ "$t" = "find_find" ]; then 375 for i in $(seq 1 $iter); do 376 run_test find_files 377 done 378 fi 379 380 if [ "$t" = "issue231" ]; then 381 run_test write_md5 382 (for i in $(seq 1 20); do 383 run_test read_md5 & true 384 done; wait); 385 rm largefile 386 fi 387 388 389 if [ "$t" = "cleanup" ]; then 390 rm -Rf * 391 test=dummy 392 fi 393 394 # for https://github.com/kahing/goofys/issues/64 395 # quote: There are 5 concurrent transfers gong at a time. 396 # Data file size is often 100-400MB. 397 # Regarding the number of transfers, I think it's about 200 files. 398 # We read from the goofys mounted s3 bucket and write to a local spring webapp using curl. 399 if [ "$t" = "disable" -o "$t" = "issue64" ]; then 400 # setup the files 401 (for i in $(seq 0 9); do 402 dd if=/dev/zero of=file$i bs=1MB count=300 oflag=nocache status=none & true 403 done 404 wait) 405 if [ $? != 0 ]; then 406 exit $? 407 fi 408 409 # 200 files and 5 concurrent transfer means 40 times, do 50 times for good measure 410 (for i in $(seq 0 9); do 411 dd if=file$i of=/dev/null bs=1MB iflag=nocache status=none & 412 done 413 414 for i in $(seq 10 300); do 415 # wait for 1 to finish, then invoke more 416 wait -n 417 running=$(ps -ef | grep ' dd if=' | grep -v grep | sed 's/.*dd if=file\([0-9]\).*/\1/') 418 for i in $(seq 0 9); do 419 if echo $running | grep -v -q $i; then 420 dd if=file$i of=/dev/null bs=1MB iflag=nocache status=none & 421 break 422 fi 423 done 424 done 425 wait) 426 if [ $? != 0 ]; then 427 exit $? 428 fi 429 430 # cleanup 431 (for i in $(seq 0 9); do 432 rm -f file$i & true 433 done 434 wait) 435 fi 436 437 if [ "$test" = "" ]; then 438 echo "No test was run: $t" 439 exit 1 440 fi