github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ais/test/scripts/s3-cp-latest-prefix.sh (about)

     1  #!/bin/bash
     2  
     3  ## Prerequisites: #################################################################################
     4  # - s3 bucket
     5  # - s3cmd, $PATH-executable and configured to access the bucket out-of-band
     6  # - aistore cluster, also configured to access the same bucket
     7  #
     8  ## Usage:
     9  ## s3-cp-latest-prefix.sh --bucket BUCKET
    10  #
    11  ## Example:
    12  ## s3-cp-latest-prefix.sh --bucket s3://abc
    13  
    14  lorem='Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.'
    15  
    16  duis='Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Et harum quidem..'
    17  
    18  ## Command line options (and their respective defaults)
    19  src="s3://abc"
    20  
    21  ## constants
    22  sum1="xxhash[ad97df912d23103f]"
    23  sum2="xxhash[ecb5ed42299ea74d]"
    24  
    25  host="--host=s3.amazonaws.com"
    26  
    27  while (( "$#" )); do
    28    case "${1}" in
    29      --bucket) src=$2; shift; shift;;
    30      *) echo "fatal: unknown argument '${1}'"; exit 1;;
    31    esac
    32  done
    33  
    34  if ! [ -x "$(command -v s3cmd)" ]; then
    35    echo "Error: s3cmd not installed" >&2
    36    exit 1
    37  fi
    38  if ! [ -x "$(command -v ais)" ]; then
    39    echo "Error: ais (CLI) not installed" >&2
    40    exit 1
    41  fi
    42  
    43  ## uncomment for verbose output
    44  set -x ## DEBUG
    45  
    46  ## establish existence
    47  ais show bucket $src -c 1>/dev/null || exit $?
    48  
    49  ## temp destination bucket will be created on the fly
    50  dst="ais://dst-$RANDOM"
    51  
    52  ## remember existing bucket's versioning; disable if need be
    53  validate=$(ais bucket props show ${src} versioning.validate_warm_get -H | awk '{print $2}')
    54  [[ "$validate" == "false"  ]] || ais bucket props set $src versioning.validate_warm_get=false
    55  sync=$(ais bucket props show ${src} versioning.synchronize -H | awk '{print $2}')
    56  [[ "$sync" == "false"  ]] || ais bucket props set $src versioning.synchronize=false
    57  
    58  cleanup() {
    59    rc=$?
    60    ais object rm "$src/lorem-duis" 1>/dev/null 2>&1
    61    [[ "$validate" == "true"  ]] || ais bucket props set $src versioning.validate_warm_get=false 1>/dev/null 2>&1
    62    [[ "$sync" == "true"  ]] || ais bucket props set $src versioning.synchronize=false 1>/dev/null 2>&1
    63    ais rmb $dst --yes 1>/dev/null 2>&1
    64    exit $rc
    65  }
    66  
    67  trap cleanup EXIT INT TERM
    68  
    69  echo -e
    70  ais show performance counters --regex "(GET-COLD$|VERSION-CHANGE$|DELETE)"
    71  echo -e
    72  
    73  echo "1. out-of-band PUT: 1st version"
    74  echo $lorem | s3cmd put - "$src/lorem-duis" $host 1>/dev/null || exit $?
    75  
    76  echo "2. copy, and check"
    77  ais cp "$src/lorem-duis" $dst --all --wait  || exit $?
    78  checksum=$(ais ls "$dst/lorem-duis" --cached -H -props checksum | awk '{print $2}')
    79  [[ "$checksum" == "$sum1"  ]] || { echo "FAIL: $checksum != $sum1"; exit 1; }
    80  
    81  echo "3. out-of-band PUT: 2nd version (overwrite)"
    82  echo $duis | s3cmd put - "$src/lorem-duis" $host 1>/dev/null || exit $?
    83  
    84  echo "4. copy and check (expecting the first version's checksum)"
    85  ais cp "$src/lorem-duis" $dst --wait
    86  checksum=$(ais ls "$dst/lorem-duis" --cached -H -props checksum | awk '{print $2}')
    87  [[ "$checksum" != "$sum2"  ]] || { echo "FAIL: $checksum == $sum2"; exit 1; }
    88  
    89  echo "5. query cold-get count (statistics)"
    90  cnt1=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
    91  
    92  echo "6. copy latest: detect version change and update in-cluster copy"
    93  ais cp "$src/lorem-duis" $dst --latest --wait
    94  checksum=$(ais ls "$dst/lorem-duis" --cached -H -props checksum | awk '{print $2}')
    95  [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
    96  
    97  echo "7. cold-get counter must increment"
    98  cnt2=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
    99  [[ $cnt2 == $(($cnt1+1)) ]] || { echo "FAIL: $cnt2 != $(($cnt1+1))"; exit 1; }
   100  
   101  echo "8. remember 'remote-deleted' counter"
   102  cnt3=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
   103  
   104  echo "9. out-of-band DELETE"
   105  s3cmd del "$src/lorem-duis" $host 1>/dev/null || exit $?
   106  
   107  echo "10. copy '--latest' (expecting no changes)"
   108  ais cp "$src/lorem-duis" $dst --latest --wait || exit $?
   109  checksum=$(ais ls "$src/lorem-duis" --cached -H -props checksum | awk '{print $2}')
   110  [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
   111  
   112  cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
   113  [[ $cnt4 == $cnt3 ]] || { echo "FAIL: $cnt4 != $cnt3"; exit 1; }
   114  
   115  echo "11. run 'cp --sync' one last time, and make sure the object \"disappears\""
   116  ais cp "$src/lorem-duis" --sync --wait
   117  
   118  ### [[ $? == 0 ]] || { echo "FAIL: expecting 'cp --wait' to return Ok, got $?"; exit 1; }
   119  
   120  echo "12. 'remote-deleted' counter must increment"
   121  cnt5=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
   122  [[ $cnt5 == $(($cnt3+1)) ]] || { echo "FAIL: $cnt5 != $(($cnt3+1))"; exit 1; }
   123  
   124  ais ls "$src/lorem-duis" --cached --silent -H 2>/dev/null
   125  [[ $? != 0 ]] || { echo "FAIL: expecting 'show object' error, got $?"; exit 1; }
   126  
   127  echo -e
   128  ais show performance counters --regex "(GET-COLD$|VERSION-CHANGE$|DELETE)"