github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ais/test/scripts/cp-sync-remais-out-of-band.sh (about)

     1  #!/bin/bash
     2  
     3  ## Prerequisites: #################################################################################
     4  # - cloud bucket
     5  # - aistore cluster
     6  # - remote aistore cluster (referred to by its "remais" alias)
     7  #
     8  ## Usage:
     9  ## cp-sync-remais-out-of-band.sh --bucket BUCKET       ### NOTE: requires cloud bucket
    10  ##
    11  ## Example:
    12  ## cp-sync-remais-out-of-band.sh --bucket s3://abc      ########################
    13  
    14  if ! [ -x "$(command -v ais)" ]; then
    15    echo "Error: ais (CLI) not installed" >&2
    16    exit 1
    17  fi
    18  
    19  ## default cloud bucket
    20  src="s3://abc"
    21  
    22  ## constants
    23  host="--host=s3.amazonaws.com"
    24  
    25  while (( "$#" )); do
    26    case "${1}" in
    27      --bucket) src=$2; shift; shift;;
    28      *) echo "fatal: unknown argument '${1}'"; exit 1;;
    29    esac
    30  done
    31  
    32  ## uncomment for verbose output
    33  ## set -x
    34  
    35  ## establish existence
    36  ais show bucket $src -c 1>/dev/null || exit $?
    37  
    38  ## remais must be attached
    39  rendpoint=$(ais show remote-cluster -H | awk '{print $2}')
    40  [[ ! -z "$rendpoint" ]] || { echo "Error: need remote ais cluster to make out-of-band changes"; exit 1; }
    41  
    42  ## remais must see the source as well
    43  AIS_ENDPOINT=$rendpoint ais show bucket $src -c 1>/dev/null || exit $?
    44  
    45  ## temp vars
    46  dst="ais://dst-$RANDOM" ## destination bucket (created on the fly)
    47  prf="prefix-$RANDOM"    ## source prefix
    48  
    49  ## remember existing state
    50  
    51  ## cleanup
    52  cleanup() {
    53    rc=$?
    54    ais rmo $src --prefix $prf --wait 1>/dev/null
    55    ais rmb $dst --yes 1>/dev/null
    56    exit $rc
    57  }
    58  
    59  trap cleanup EXIT INT TERM
    60  
    61  echo " 1. generate and write 500 random shards => $src"
    62  ais archive gen-shards "$src/$prf/shard-{001..500}.tar" 1>/dev/null 2>&1
    63  cnt_src=$(ais ls $src --prefix $prf | grep Listed)
    64  
    65  echo " 2. copy $src => $dst"
    66  ais cp $src $dst --sync --wait 1>/dev/null 2>&1
    67  cnt_dst=$(ais ls $dst --prefix $prf | grep Listed)
    68  [[ $cnt_src == $cnt_dst ]] || { echo "FAIL: '$cnt_src' != '$cnt_dst'"; exit 1; }
    69  
    70  echo " 3. remove 10 shards from the source"
    71  ais rmo "$src/$prf/shard-{51..60}.tar" --wait 1>/dev/null 2>&1
    72  cnt_src=$(ais ls $src --prefix $prf | grep Listed)
    73  
    74  echo " 4. copy $src => $dst w/ synchronization ('--sync' option)"
    75  ais cp $src $dst --sync --wait 1>/dev/null 2>&1
    76  cnt_dst=$(ais ls $dst --prefix $prf | grep Listed)
    77  [[ $cnt_src == $cnt_dst ]] || { echo "FAIL: '$cnt_src' != '$cnt_dst'"; exit 1; }
    78  
    79  echo " 5. remove another 10 shards"
    80  ais rmo "$src/$prf/shard-{151..160}.tar" --wait 1>/dev/null 2>&1
    81  cnt_src=$(ais ls $src --prefix $prf | grep Listed)
    82  
    83  echo " 6. copy multiple objects using bash-expansion defined range and '--sync'"
    84  ais cp "$src/$prf/shard-{001..500}.tar" $dst --sync --wait 1>/dev/null 2>&1
    85  cnt_dst=$(ais ls $dst --prefix $prf | grep Listed)
    86  [[ $cnt_src == $cnt_dst ]] || { echo "FAIL: '$cnt_src' != '$cnt_dst'"; exit 1; }
    87  
    88  echo " #"
    89  echo " # out of band DELETE using remote AIS (remais)"
    90  echo " #"
    91  
    92  echo " 7. use remote AIS cluster (\"remais\") to out-of-band remove 10 shards from the source"
    93  AIS_ENDPOINT=$rendpoint ais rmo "$src/$prf/shard-{251..260}.tar" --wait 1>/dev/null 2>&1
    94  cnt_src=$(ais ls $src --prefix $prf | grep Listed)
    95  
    96  echo " 8. copy $src => $dst w/ --sync"
    97  ais cp $src $dst --sync --wait 1>/dev/null 2>&1
    98  cnt_dst=$(ais ls $dst --prefix $prf | grep Listed)
    99  [[ $cnt_src == $cnt_dst ]] || { echo "FAIL: '$cnt_src' != '$cnt_dst'"; exit 1; }
   100  
   101  echo " 9. when copying, we always synchronize content of the in-cluster source as well"
   102  cnt_src_cached=$(ais ls $src --prefix $prf --cached | grep Listed)
   103  [[ $cnt_src_cached == $cnt_src ]] || { echo "FAIL: '$cnt_src_cached' != '$cnt_src'"; exit 1; }
   104  
   105  echo "10. use remais to out-of-band remove 10 more shards from $src source"
   106  AIS_ENDPOINT=$rendpoint ais rmo "$src/$prf/shard-{351..360}.tar" --wait 1>/dev/null 2>&1
   107  cnt_src=$(ais ls $src --prefix $prf | grep Listed)
   108  
   109  echo "11. copy a range of shards from $src to $dst, and compare"
   110  ais cp "$src/$prf/shard-{001..500}.tar" $dst --sync --wait 1>/dev/null 2>&1
   111  cnt_dst=$(ais ls $dst --prefix $prf | grep Listed)
   112  [[ $cnt_src == $cnt_dst ]] || { echo "FAIL: '$cnt_src' != '$cnt_dst'"; exit 1; }
   113  
   114  echo "12. and again: when copying, we always synchronize content of the in-cluster source as well"
   115  cnt_src_cached=$(ais ls $src --prefix $prf --cached | grep Listed)
   116  [[ $cnt_src_cached == $cnt_src ]] || { echo "FAIL: '$cnt_src_cached' != '$cnt_src'"; exit 1; }
   117  
   118  echo " #"
   119  echo " # out of band ADD using remote AIS (remais)"
   120  echo " #"
   121  
   122  echo "13. use remais to out-of-band add (i.e., PUT) 17 new shards"
   123  AIS_ENDPOINT=$rendpoint ais archive gen-shards "$src/$prf/shard-{501..517}.tar" 1>/dev/null 2>&1
   124  cnt_src=$(ais ls $src --prefix $prf | grep Listed)
   125  
   126  echo "14. copy a range of shards from $src to $dst, and check whether the destination has new shards"
   127  ais cp "$src/$prf/shard-{001..600}.tar" $dst --sync --wait 1>/dev/null 2>&1
   128  
   129  echo "15. compare the contents but NOTE: as of v3.22, this part requires multi-object copy (using '--list' or '--template')"
   130  cnt_dst=$(ais ls $dst --prefix $prf | grep Listed)
   131  [[ $cnt_src == $cnt_dst ]] || { echo "FAIL: '$cnt_src' != '$cnt_dst'"; exit 1; }