github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/bench/tools/aisloader-composer/playbooks/scripts/aisloader_script.sh (about)

     1  #!/bin/bash
     2  hostname=$(hostname -s)
     3  
     4  outdir=/tmp/aisloader/
     5  sudo rm -rf $outdir
     6  sudo mkdir $outdir
     7  
     8  bucket=""
     9  bench_type=""
    10  each_size=""
    11  total_size=""
    12  duration=""
    13  epochs=0
    14  ais_proxies=""
    15  ais_port=""
    16  grafana_host=""
    17  workers=""
    18  filelist=""
    19  
    20  for arg in "$@"; do
    21      case "$arg" in
    22          --bench_type=*)
    23              bench_type="${arg#*=}"
    24              ;;
    25          --ais_proxies=*)
    26              ais_proxies="${arg#*=}"
    27              ;;
    28          --ais_port=*)
    29              ais_port="${arg#*=}"
    30              ;;
    31          --duration=*)
    32              duration="${arg#*=}"
    33              ;;
    34          --epochs=*)
    35              epochs="${arg#*=}"
    36              ;;
    37          --each_size=*)
    38              each_size="${arg#*=}"
    39              ;;
    40          --total_size=*)
    41              total_size="${arg#*=}"
    42              ;;
    43          --grafana_host=*)
    44              grafana_host="${arg#*=}"
    45              ;;
    46          --workers=*)
    47              workers="${arg#*=}"
    48              ;;
    49          --bucket=*)
    50              bucket="${arg#*=}"
    51              ;;
    52          --s3_endpoint=*)
    53              s3_endpoint="${arg#*=}"
    54              ;;
    55          --filelist=*)
    56              filelist="${arg#*=}"
    57              ;;
    58          *)
    59              echo "Invalid argument: $arg"
    60              ;;
    61      esac
    62  done
    63  
    64  if [[ "$bench_type" != *"get"* ]] && [[ "$bench_type" != *"put"* ]]; then
    65    echo "Error: Bench type must contain 'get' or 'put'"
    66    exit 1
    67  fi
    68  
    69  # Parse provider and bucket name from the bucket arg
    70  delimiter="://"
    71  # Check if the input string contains the delimiter
    72  if [[ "$bucket" == *"$delimiter"* ]]; then
    73      read -r provider bucket_name <<< "$(echo "$bucket" | awk -F "$delimiter" '{print $1 " " $2}')"
    74  else 
    75      provider="ais"
    76      bucket_name=$bucket
    77  fi
    78  
    79  echo "Running with provider $provider and bucket $bucket_name"
    80  
    81  filename="$bucket_name-$bench_type-"
    82  outfile="$outdir$filename$hostname.json"
    83  
    84  # Common aisloader args for all bench types
    85  bench_args=("-loaderid=$(hostname)" "-loaderidhashlen=8" "-bucket=$bucket" "-cleanup=false" "-json" "-stats-output=$outfile" "-statsdip=$grafana_host" "-numworkers=$workers")
    86  
    87  # Args specific to PUT or GET workloads
    88  if [[ "$bench_type" == *"put"* ]]; then
    89      bench_args+=("-totalputsize=$total_size")
    90      bench_args+=("-minsize=$each_size")
    91      bench_args+=("-maxsize=$each_size")
    92      bench_args+=("-pctput=100")
    93      bench_args+=("-skiplist")
    94  else
    95      if [ -n "$duration" ]; then
    96          bench_args+=("-duration=$duration")
    97      fi
    98      if [ "$epochs" -ne 0 ]; then
    99          bench_args+=("-epochs=$epochs")
   100      fi
   101      bench_args+=("-pctput=0")
   102      if [ -n "$filelist" ]; then
   103          bench_args+=("-filelist=$filelist")
   104      fi
   105  fi
   106  
   107  # Args specific to either cloud or AIS benchmarks
   108  if [ -n "$s3_endpoint" ]; then
   109      # Run the benchmark directly to the cloud bucket with the given name and s3endpoint
   110      bench_args+=("-s3endpoint=$s3_endpoint")
   111  else
   112      # Run the benchmark against the bucket in AIS
   113      # Split comma-separated string list of proxies into an array
   114      IFS=',' read -ra proxy_list <<< "$ais_proxies"
   115  
   116      bench_args+=("-ip=${proxy_list[0]}")
   117      bench_args+=("-port=$ais_port")
   118      bench_args+=("-randomproxy") 
   119  fi
   120  
   121  echo "\n Benchmark args: ${bench_args[@]} \n"
   122  # Run the aisloader binary
   123  aisloader "${bench_args[@]}"