github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ais/test/scripts/dsort-ex1.sh (about)

     1  #!/bin/bash
     2  
     3  ## Example usage:
     4  ## ./ais/test/scripts/dsort-ex1.sh --srcbck ais://aaaaa --dstbck ais://qqqqq
     5  
     6  if ! [ -x "$(command -v ais)" ]; then
     7    echo "Error: ais (CLI) not installed" >&2
     8    exit 1
     9  fi
    10  
    11  SCRIPT_PATH="$(cd "$(dirname "$0")"; pwd -P)"
    12  
    13  ## Command line options (and their respective defaults)
    14  srcbck="ais://src"
    15  dstbck="ais://dst"
    16  spec="dsort-spec1.json"
    17  
    18  while (( "$#" )); do
    19    case "${1}" in
    20      --srcbck) srcbck=$2; shift; shift;;
    21      --dstbck) dstbck=$2; shift; shift;;
    22      --spec) spec=$2; shift; shift;;
    23      --nocleanup) nocleanup="true"; shift;;
    24      *) echo "fatal: unknown argument '${1}'"; exit 1;;
    25    esac
    26  done
    27  
    28  ## establish existence
    29  ais show bucket $srcbck -c 1>/dev/null 2>&1
    30  srcexists=$?
    31  ais show bucket $dstbck -c 1>/dev/null 2>&1
    32  dstexists=$?
    33  
    34  ## generate 10 tar shards, each containing 5 files (50 files total)
    35  ## note that dstbck, if doesn't exist, will be created on the fly
    36  [[ $srcexists == 0 ]] || ais create $srcbck || exit 1
    37  ais archive gen-shards "$srcbck/shard-{0..9}.tar" || \
    38  exit 1
    39  
    40  ## run dsort and wait for the job to finish
    41  ## (see 'dsort-ex1-spec.json' in this directory)
    42  echo "Using spec '$spec'"
    43  ais wait $(ais start dsort ${srcbck} ${dstbck} -f ${SCRIPT_PATH}/${spec})
    44  
    45  ## list new shards to confirm 5 new shards, each containing 10 original files
    46  ## (the same 50 total - see above)
    47  num=$(ais ls $dstbck --summary --H | awk '{print $3}')
    48  [[ $num == 5 ]] || { echo "FAIL: $num != 5"; exit 1; }
    49  
    50  echo "Successfully resharded $srcbck => $dstbck:"
    51  ais ls $dstbck
    52  
    53  ## cleanup: rmb buckets created during this run
    54  if [[ ${nocleanup} != "true" && $srcexists != 0 ]]; then
    55    echo "Deleting source: $srcbck"
    56    ais rmb $srcbck -y 2>/dev/null 1>&2
    57  fi
    58  if [[ ${nocleanup} != "true" && $dstexists != 0 ]]; then
    59    echo "Deleting destination: $dstbck"
    60    ais rmb $dstbck -y 2>/dev/null 1>&2
    61  fi