github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ais/test/scripts/s3-prefetch-latest-range.sh (about)

     1  #!/bin/bash
     2  
     3  ## Prerequisites: #################################################################################
     4  # - s3 bucket
     5  # - s3cmd, $PATH-executable and configured to access the bucket out-of-band
     6  # - aistore cluster, also configured to access the same bucket
     7  #
     8  ## Example:
     9  ## s3-prefetch-latest-range.sh --bucket s3://abc
    10  
    11  lorem='Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.'
    12  
    13  duis='Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Et harum quidem..'
    14  
    15  ## Command line options (and their respective defaults)
    16  bucket="s3://abc"
    17  
    18  ## constants
    19  sum1="xxhash[ad97df912d23103f]"
    20  sum2="xxhash[ecb5ed42299ea74d]"
    21  
    22  host="--host=s3.amazonaws.com"
    23  
    24  while (( "$#" )); do
    25    case "${1}" in
    26      --bucket) bucket=$2; shift; shift;;
    27      *) echo "fatal: unknown argument '${1}'"; exit 1;;
    28    esac
    29  done
    30  
    31  if ! [ -x "$(command -v s3cmd)" ]; then
    32    echo "Error: s3cmd not installed" >&2
    33    exit 1
    34  fi
    35  if ! [ -x "$(command -v ais)" ]; then
    36    echo "Error: ais (CLI) not installed" >&2
    37    exit 1
    38  fi
    39  
    40  ## uncomment for verbose output
    41  ## set -x
    42  
    43  ## establish existence
    44  ais show bucket $bucket -c 1>/dev/null || exit $?
    45  
    46  ## remember existing bucket's versioning; disable if need be
    47  sync=$(ais bucket props show ${bucket} versioning.synchronize -H | awk '{print $2}')
    48  [[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.synchronize=false
    49  
    50  cleanup() {
    51    rc=$?
    52    ais object rm "$bucket/shard-001" 1>/dev/null 2>&1
    53    [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.synchronize=false 1>/dev/null 2>&1
    54    exit $rc
    55  }
    56  
    57  trap cleanup EXIT INT TERM
    58  
    59  echo -e
    60  ais show performance counters --regex "(GET-COLD$|VERSION-CHANGE$|DELETE)"
    61  echo -e
    62  
    63  echo "1. out-of-band PUT: 1st version"
    64  echo $lorem | s3cmd put - "$bucket/shard-001" $host 1>/dev/null || exit $?
    65  
    66  echo "2. prefetch, and check"
    67  ais prefetch "$bucket/shard-{001..009}" --wait
    68  checksum=$(ais ls "$bucket/shard-001" --cached -H -props checksum | awk '{print $2}')
    69  [[ "$checksum" == "$sum1"  ]] || { echo "FAIL: $checksum != $sum1"; exit 1; }
    70  
    71  echo "3. out-of-band PUT: 2nd version (overwrite)"
    72  echo $duis | s3cmd put - "$bucket/shard-001" $host 1>/dev/null || exit $?
    73  
    74  echo "4. prefetch and check (expecting the first version's checksum)"
    75  ais prefetch "$bucket/shard-{001..009}" --wait
    76  checksum=$(ais ls "$bucket/shard-001" --cached -H -props checksum | awk '{print $2}')
    77  [[ "$checksum" != "$sum2"  ]] || { echo "FAIL: $checksum == $sum2"; exit 1; }
    78  
    79  echo "5. query cold-get count (statistics)"
    80  cnt1=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
    81  
    82  echo "6. prefetch latest: detect version change and update in-cluster copy"
    83  ais prefetch "$bucket/shard-{001..009}" --latest --wait
    84  checksum=$(ais ls "$bucket/shard-001" --cached -H -props checksum | awk '{print $2}')
    85  [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
    86  
    87  echo "7. cold-get counter must increment"
    88  cnt2=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
    89  [[ $cnt2 == $(($cnt1+1)) ]] || { echo "FAIL: $cnt2 != $(($cnt1+1))"; exit 1; }
    90  
    91  echo "8. warm GET must remain \"warm\" and cold-get-count must not increment"
    92  ais get "$bucket/shard-001" /dev/null 1>/dev/null
    93  checksum=$(ais ls "$bucket/shard-001" --cached -H -props checksum | awk '{print $2}')
    94  [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
    95  
    96  cnt3=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
    97  [[ $cnt3 == $cnt2 ]] || { echo "FAIL: $cnt3 != $cnt2"; exit 1; }
    98  
    99  echo "9. out-of-band DELETE"
   100  s3cmd del "$bucket/shard-001" $host 1>/dev/null || exit $?
   101  
   102  echo "10. prefetch without '--latest': expecting no changes"
   103  ais prefetch "$bucket/shard-{001..009}" --wait
   104  checksum=$(ais ls "$bucket/shard-001" --cached -H -props checksum | awk '{print $2}')
   105  [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
   106  
   107  echo "11. remember 'remote-deleted' counter _and_ enable version synchronization"
   108  cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
   109  ais bucket props set $bucket versioning.synchronize=true
   110  
   111  echo "12. run 'prefetch --latest' one last time; make sure the object \"disappears\""
   112  ais prefetch "$bucket/shard-{001..009}" --latest --wait 2>/dev/null
   113  [[ $? == 0 ]] || { echo "FAIL: expecting 'prefetch --wait' to return Ok, got $?"; exit 1; }
   114  
   115  echo "13. 'remote-deleted' counter must increment (because 'versioning.synchronize=true')"
   116  cnt5=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
   117  [[ $cnt5 == $(($cnt4+1)) ]] || { echo "FAIL: $cnt5 != $(($cnt4+1))"; exit 1; }
   118  
   119  ais ls "$bucket/shard-001" --cached --silent -H 2>/dev/null
   120  [[ $? != 0 ]] || { echo "FAIL: expecting 'show object' error, got $?"; exit 1; }
   121  
   122  echo -e
   123  ais show performance counters --regex "(GET-COLD$|VERSION-CHANGE$|DELETE)"