github.com/adrianjagielak/goofys@v0.24.1-0.20230810095418-94919a5d2254/bench/bench.sh (about) 1 #!/bin/bash 2 3 : ${TRAVIS:="false"} 4 : ${FAST:="false"} 5 : ${test:=""} 6 : ${CACHE:="false"} 7 : ${CLEANUP:="true"} 8 : ${MOUNT_LOG:="mount.log"} 9 10 iter=10 11 12 if [ "$TRAVIS" != "false" ]; then 13 set -o xtrace 14 iter=1 15 fi 16 17 if [ "$FAST" != "false" ]; then 18 iter=1 19 fi 20 21 set -o errexit 22 set -o nounset 23 24 if [ $# -lt 2 ]; then 25 echo "Usage: $0 <mount cmd> <dir> [test name]" 26 exit 1 27 fi 28 29 cmd=$1 30 mnt=$2 31 if [ $# -gt 2 ]; then 32 t=$3 33 else 34 t= 35 fi 36 37 prefix=$mnt/test_dir 38 39 MOUNTED=0 40 41 if [[ "$cmd" == riofs* ]]; then 42 RIOFS="true" 43 else 44 RIOFS="false" 45 fi 46 47 if [[ "$cmd" == blobfuse* ]]; then 48 BLOBFUSE="true" 49 else 50 BLOBFUSE="false" 51 fi 52 53 $cmd >& $MOUNT_LOG & 54 PID=$! 55 56 function cleanup { 57 if [ $MOUNTED == 1 ]; then 58 popd >/dev/null 59 if [ "$CLEANUP" = "true" ]; then 60 if [ "$TRAVIS" != "false" ]; then 61 rmdir $prefix 62 else 63 rmdir $prefix >& /dev/null || true # riofs doesn't support rmdir 64 fi 65 fi 66 fi 67 68 if [ "$PID" != "" ]; then 69 kill $PID >& /dev/null || true 70 fusermount -u $mnt >& /dev/null || true 71 sleep 1 72 fi 73 } 74 75 function cleanup_err { 76 err=$? 77 cat $MOUNT_LOG 78 if [ $MOUNTED == 1 ]; then 79 popd >&/dev/null || true 80 rmdir $prefix >&/dev/null || true 81 fi 82 83 if [ "$PID" != "" ]; then 84 kill $PID >& /dev/null || true 85 fusermount -u $mnt >& /dev/null || true 86 fi 87 88 return $err 89 } 90 91 trap cleanup EXIT 92 trap cleanup_err ERR 93 94 function wait_for_mount { 95 for i in $(seq 1 10); do 96 if grep -q $mnt /proc/mounts; then 97 break 98 fi 99 sleep 1 100 done 101 if ! grep -q $mnt /proc/mounts; then 102 echo "$mnt not mounted by $cmd" 103 cat $MOUNT_LOG 104 exit 1 105 fi 106 } 107 108 if [ "$TRAVIS" == "false" -a "$cmd" != "cat" ]; then 109 wait_for_mount 110 MOUNTED=1 111 else 112 # in travis we mount things externally so we know we are mounted 113 MOUNTED=1 114 fi 115 116 mkdir -p "$prefix" 117 pushd "$prefix" >/dev/null 118 119 SUDO= 120 if [ $(id -u) != 0 ]; then 121 SUDO=sudo 122 fi 123 124 function drop_cache { 125 if [ "$TRAVIS" == "false" ]; then 126 (echo 3 | $SUDO tee /proc/sys/vm/drop_caches) > /dev/null 127 fi 128 } 129 130 export TIMEFORMAT=%R 131 132 function run_test { 133 test=$1 134 shift 135 drop_cache 136 sleep 2 137 if [ "$CACHE" == "false" ]; then 138 if [ -d /tmp/cache ]; then 139 rm -Rf /tmp/cache/* 140 fi 141 if [ "$BLOBFUSE" == "true" ]; then 142 popd >/dev/null 143 # re-mount blobfuse to cleanup cache 144 if [ "$PID" != "" ]; then 145 fusermount -u $mnt 146 sleep 1 147 fi 148 $cmd >& $MOUNT_LOG & 149 PID=$! 150 wait_for_mount 151 pushd "$prefix" >/dev/null 152 fi 153 fi 154 155 echo -n "$test " 156 if [ $# -gt 1 ]; then 157 time $test $@ 158 else 159 time $test 160 fi 161 } 162 163 function get_howmany { 164 if [ "$TRAVIS" != "false" ]; then 165 if [ $# == 2 ]; then 166 howmany=$2 167 else 168 howmany=10 169 fi 170 else 171 if [ $# == 0 ]; then 172 howmany=100 173 else 174 howmany=$1 175 fi 176 fi 177 } 178 179 function create_files { 180 get_howmany $@ 181 182 for i in $(seq 1 $howmany); do 183 echo $i > file$i 184 done 185 } 186 187 function ls_files { 188 get_howmany $@ 189 # people usually use ls in the terminal when color is on 190 numfiles=$(ls -1 --color=always | wc -l) 191 if [ "$numfiles" != "$howmany" ]; then 192 echo "$numfiles != $howmany" 193 false 194 fi 195 } 196 197 function rm_files { 198 get_howmany $@ 199 200 for i in $(seq 1 $howmany); do 201 rm file$i >&/dev/null || true 202 done 203 } 204 205 function find_files { 206 numfiles=$(find | wc -l) 207 208 if [ "$numfiles" != 820 ]; then 209 echo "$numfiles != 820" 210 rm_tree 211 exit 1 212 fi 213 } 214 215 function create_tree_parallel { 216 (for i in $(seq 1 9); do 217 mkdir $i 218 for j in $(seq 1 9); do 219 mkdir $i/$j 220 221 for k in $(seq 1 9); do 222 touch $i/$j/$k & true 223 done 224 done 225 done 226 wait) 227 } 228 229 function rm_tree { 230 for i in $(seq 1 9); do 231 rm -Rf $i 232 done 233 } 234 235 function create_files_parallel { 236 get_howmany $@ 237 238 (for i in $(seq 1 $howmany); do 239 (echo $i > file$i && echo "created file$i") & true 240 done 241 wait) 242 } 243 244 function rm_files_parallel { 245 get_howmany $@ 246 247 (for i in $(seq 1 $howmany); do 248 rm file$i & true 249 done 250 wait) 251 } 252 253 function write_large_file { 254 count=1000 255 if [ "$FAST" == "true" ]; then 256 count=100 257 fi 258 dd if=/dev/zero of=largefile bs=1MB count=$count oflag=nocache status=none 259 } 260 261 function read_large_file { 262 dd if=largefile of=/dev/null bs=1MB iflag=nocache status=none 263 } 264 265 function read_first_byte { 266 dd if=largefile of=/dev/null bs=1 count=1 iflag=nocache status=none 267 } 268 269 if [ "$t" = "" -o "$t" = "create" ]; then 270 for i in $(seq 1 $iter); do 271 run_test create_files 272 run_test rm_files 273 done 274 fi 275 276 if [ "$t" = "" -o "$t" = "create_parallel" ]; then 277 for i in $(seq 1 $iter); do 278 run_test create_files_parallel 279 run_test rm_files_parallel 280 done 281 fi 282 283 function write_md5 { 284 seed=$(dd if=/dev/urandom bs=128 count=1 status=none | base64 -w 0) 285 random_cmd="openssl enc -aes-256-ctr -pbkdf2 -pass pass:$seed -nosalt" 286 count=1000 287 if [ "$FAST" == "true" ]; then 288 count=100 289 fi 290 MD5=$(dd if=/dev/zero bs=1MB count=$count status=none | $random_cmd | \ 291 tee >(md5sum) >(dd of=largefile bs=1MB oflag=nocache status=none) >/dev/null | cut -f 1 '-d ') 292 if [ "$RIOFS" == "true" ]; then 293 # riofs doesn't wait for flush, so we need to wait for object to show up 294 # XXX kind of broken due to eventual consistency but it's hte best we can do 295 while ! aws s3api --endpoint ${ENDPOINT} head-object --bucket ${BUCKET} --key test_dir/largefile >& /dev/null; do sleep 0.1; done 296 fi 297 } 298 299 function read_md5 { 300 READ_MD5=$(md5sum largefile | cut -f 1 '-d ') 301 if [ "$READ_MD5" != "$MD5" ]; then 302 echo "$READ_MD5 != $MD5" >&2 303 rm largefile 304 exit 1 305 fi 306 } 307 308 function rm { 309 if [ "$RIOFS" == "true" ]; then 310 while ! /bin/rm $@; do true; done 311 else 312 /bin/rm $@ 313 fi 314 } 315 316 if [ "$t" = "" -o "$t" = "io" ]; then 317 for i in $(seq 1 $iter); do 318 run_test write_md5 319 if [ "$CACHE" != "true" ]; then 320 run_test read_md5 321 run_test read_first_byte 322 fi 323 rm largefile 324 done 325 if [ "$CACHE" = "true" ]; then 326 write_md5 327 read_md5 328 for i in $(seq 1 $iter); do 329 run_test read_md5 330 run_test read_first_byte 331 done 332 rm largefile 333 fi 334 fi 335 336 if [ "$t" = "" -o "$t" = "ls" ]; then 337 create_files_parallel 2000 2000 338 for i in $(seq 1 $iter); do 339 run_test ls_files 2000 2000 340 done 341 if [ "$CLEANUP" = "true" ]; then 342 rm_files 2000 2000 343 fi 344 fi 345 346 if [ "$t" = "ls_create" ]; then 347 create_files_parallel 1000 348 test=dummy 349 sleep 10 350 fi 351 352 if [ "$t" = "ls_ls" ]; then 353 run_test ls_files 1000 1000 354 fi 355 356 if [ "$t" = "ls_rm" ]; then 357 rm_files 1000 358 test=dummy 359 fi 360 361 if [ "$t" = "" -o "$t" = "find" ]; then 362 create_tree_parallel 363 for i in $(seq 1 $iter); do 364 run_test find_files 365 done 366 rm_tree 367 fi 368 369 if [ "$t" = "find_create" ]; then 370 create_tree_parallel 371 test=dummy 372 sleep 10 373 fi 374 375 if [ "$t" = "find_find" ]; then 376 for i in $(seq 1 $iter); do 377 run_test find_files 378 done 379 fi 380 381 if [ "$t" = "issue231" ]; then 382 run_test write_md5 383 (for i in $(seq 1 20); do 384 run_test read_md5 & true 385 done; wait); 386 rm largefile 387 fi 388 389 390 if [ "$t" = "cleanup" ]; then 391 rm -Rf * 392 test=dummy 393 fi 394 395 # for https://github.com/kahing/goofys/issues/64 396 # quote: There are 5 concurrent transfers gong at a time. 397 # Data file size is often 100-400MB. 398 # Regarding the number of transfers, I think it's about 200 files. 399 # We read from the goofys mounted s3 bucket and write to a local spring webapp using curl. 400 if [ "$t" = "disable" -o "$t" = "issue64" ]; then 401 # setup the files 402 (for i in $(seq 0 9); do 403 dd if=/dev/zero of=file$i bs=1MB count=300 oflag=nocache status=none & true 404 done 405 wait) 406 if [ $? != 0 ]; then 407 exit $? 408 fi 409 410 # 200 files and 5 concurrent transfer means 40 times, do 50 times for good measure 411 (for i in $(seq 0 9); do 412 dd if=file$i of=/dev/null bs=1MB iflag=nocache status=none & 413 done 414 415 for i in $(seq 10 300); do 416 # wait for 1 to finish, then invoke more 417 wait -n 418 running=$(ps -ef | grep ' dd if=' | grep -v grep | sed 's/.*dd if=file\([0-9]\).*/\1/') 419 for i in $(seq 0 9); do 420 if echo $running | grep -v -q $i; then 421 dd if=file$i of=/dev/null bs=1MB iflag=nocache status=none & 422 break 423 fi 424 done 425 done 426 wait) 427 if [ $? != 0 ]; then 428 exit $? 429 fi 430 431 # cleanup 432 (for i in $(seq 0 9); do 433 rm -f file$i & true 434 done 435 wait) 436 fi 437 438 if [ "$test" = "" ]; then 439 echo "No test was run: $t" 440 exit 1 441 fi