github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ais/test/scripts/remais-blob-download.sh (about)

     1  #!/bin/bash
     2  
     3  ## Prerequisites: ###############################################################################
     4  # - aistore cluster
     5  # - remote aistore cluster (a.k.a. "remais")
     6  # - optionally, remais bucket (the bucket will be created if doesn't exist)
     7  # - ais (CLI)
     8  # - aisloader
     9  #
    10  # NOTE: max_num_downloads limit (hardcoded)
    11  #
    12  ## Example 1:
    13  ## first, make sure remote ais cluster is attached:
    14  #  $ ais show remote-cluster -H
    15  #  $ JcHy3JUrL  http://127.0.0.1:11080  remais    v9  1  11m22.312048996s
    16  #
    17  ## second, run:
    18  #  $ remais-blob-download.sh --bucket ais://@remais/abc --maxsize 10mb --totalsize 1G
    19  #
    20  ## Example 2:
    21  #  $ remais-blob-download.sh --bucket ais://@remais/abc --chunksize 500kb
    22  #################################################################################################
    23  
    24  if ! [ -x "$(command -v ais)" ]; then
    25    echo "Error: ais (CLI) not installed" >&2
    26    exit 1
    27  fi
    28  if ! [ -x "$(command -v aisloader)" ]; then
    29    echo "Error: aisloader (benchmark tool) not installed" >&2
    30    exit 1
    31  fi
    32  
    33  ## Command line options and respective defaults
    34  bucket="ais://@remais/abc"
    35  
    36  ## aisloader command line
    37  minsize="10MiB"
    38  maxsize="1GiB"
    39  totalsize="10GiB"
    40  
    41  ## permutating (1mb, 2mb, 3mb) unless explicitly given in the command line
    42  chunksize="1MB"
    43  
    44  ## put some limit to it
    45  max_num_downloads=30 ########## ---------------------------------------------------------------------
    46  
    47  ## destination for aisloader-generated content
    48  subdir="blob-$RANDOM"
    49  
    50  ## initial number of the chunk-reading workers
    51  numworkers_initial=3
    52  
    53  while (( "$#" )); do
    54    case "${1}" in
    55      --bucket) bucket=$2; shift; shift;;
    56      --minsize) minsize=$2; shift; shift;;
    57      --maxsize) maxsize=$2; shift; shift;;
    58      --totalsize) totalsize=$2; shift; shift;;
    59      --chunksize) chunksize=$2; shift; shift;;
    60      *) echo "fatal: unknown argument '${1}'"; exit 1;;
    61    esac
    62  done
    63  
    64  ## uncomment for verbose output
    65  ## set -x
    66  
    67  ## must be @remais
    68  [[ "$bucket" == *//@* ]] || { echo "Error: expecting remote ais bucket, got ${bucket}"; exit 1; }
    69  
    70  ## actual bucket inside remais:
    71  rbucket="ais://$(basename ${bucket})"
    72  
    73  ## remais must be attached
    74  rendpoint=$(ais show remote-cluster -H | awk '{print $2}')
    75  [[ ! -z "$rendpoint" ]] || { echo "Error: no remote ais clusters"; exit 1; }
    76  uuid=$(ais show remote-cluster -H | awk '{print $1}')
    77  
    78  echo "Note: remote ais bucket $bucket is, in fact, ais://@${uuid}/$(basename ${bucket})"
    79  echo
    80  
    81  ## check remote bucket; create if doesn't exist
    82  exists=true
    83  ais show bucket $bucket -c 1>/dev/null 2>&1 || exists=false
    84  
    85  cleanup() {
    86    rc=$?
    87    if [[ "$exists" == "true" ]]; then
    88      AIS_ENDPOINT=$rendpoint ais rmo $rbucket --prefix=$subdir/ 2> /dev/null
    89      ais rmo $bucket --prefix=$subdir/ 2> /dev/null
    90    else
    91      ais rmb $bucket -y 1> /dev/null 2>&1
    92    fi
    93    exit $rc
    94  }
    95  
    96  trap cleanup EXIT INT TERM
    97  
    98  chunk_size() {
    99    if [ $chunksize == "1MB" ]; then
   100      chunksize="2MB"
   101    elif [ $chunksize == "2MB" ]; then
   102      chunksize="3MB"
   103    else
   104      chunksize="1MB"
   105    fi
   106  }
   107  
   108  ## aisloader => remais, to generate (PUT) content in $bucket/$subdir
   109  echo "1. Run aisloader"
   110  AIS_ENDPOINT=$rendpoint aisloader -bucket=$rbucket -subdir=$subdir -cleanup=false -numworkers=2 -quiet -pctput=100 -minsize=$minsize -maxsize=$maxsize -totalputsize=$totalsize
   111  
   112  ais ls $bucket --all --limit 4
   113  echo "..."
   114  files=$(ais ls $bucket --prefix=$subdir/ --name-only -H --no-footers --all | awk '{print $1}')
   115  
   116  count=0 ## up to max_num_downloads
   117  
   118  ## first, run as xaction
   119  echo
   120  echo "2. Run blob-download jobs"
   121  numworkers=$numworkers_initial
   122  for f in $files; do
   123    xid=$(ais blob-download $bucket/$f --chunk-size $chunksize --num-workers $numworkers --nv || exit $?)
   124    ais wait $xid >/dev/null || exit $?
   125    count=`expr $count + 1`
   126    if [ $count -ge $max_num_downloads ]; then
   127       break
   128    fi
   129    numworkers=`expr $numworkers + 1`
   130    if [ $numworkers -ge 16 ]; then
   131       numworkers=$numworkers_initial
   132    fi
   133    chunk_size
   134  done
   135  
   136  ## show some tails
   137  echo "..."
   138  ais show job blob-download --all | tail
   139  echo "..."
   140  ais ls $bucket --cached | tail
   141  
   142  ## second, run the same via GET
   143  echo
   144  echo "3. Run GET via blob-downloader - evict first..."
   145  ais evict $bucket --keep-md || exit $?
   146  count=0
   147  numworkers=$numworkers_initial
   148  for f in $files; do
   149    ais get $bucket/$f /dev/null --blob-download --num-workers=$numworkers >/dev/null || exit $?
   150    count=`expr $count + 1`
   151    if [ $count -ge $max_num_downloads ]; then
   152       break
   153    fi
   154    numworkers=`expr $numworkers + 1`
   155    if [ $numworkers -ge 16 ]; then
   156       numworkers=$numworkers_initial
   157    fi
   158    chunk_size
   159  done
   160  
   161  ## ditto
   162  echo "..."
   163  ais show job blob-download --all | tail
   164  echo "..."
   165  ais ls $bucket --cached | tail