github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ais/test/scripts/remais-get-validate.sh (about)

     1  #!/bin/bash
     2  
     3  ## Prerequisites: #################################################################################
     4  # - aistore cluster
     5  # - remote aistore cluster (a.k.a. "remais")
     6  # - optionally, remais bucket (the bucket will be created if doesn't exist)
     7  # - ais (CLI)
     8  #
     9  ## Example:
    10  ## first, make sure remote ais cluster is attached:
    11  #
    12  #  $ ais show remote-cluster -H
    13  #  $ JcHy3JUrL  http://127.0.0.1:11080  remais    v9  1  11m22.312048996s
    14  #
    15  ## second, run:
    16  #  $./ais/test/scripts/remais-get-validate.sh --bucket ais://@remais/abc     #####################
    17  
    18  if ! [ -x "$(command -v ais)" ]; then
    19    echo "Error: ais (CLI) not installed" >&2
    20    exit 1
    21  fi
    22  
    23  lorem='Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.'
    24  
    25  duis='Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Et harum quidem..'
    26  
    27  ## Command line options (and their respective defaults)
    28  bucket="ais://@remais/abc"
    29  
    30  ## constants
    31  sum1="xxhash[ad97df912d23103f]"
    32  sum2="xxhash[ecb5ed42299ea74d]"
    33  
    34  while (( "$#" )); do
    35    case "${1}" in
    36      --bucket) bucket=$2; shift; shift;;
    37      *) echo "fatal: unknown argument '${1}'"; exit 1;;
    38    esac
    39  done
    40  
    41  ## uncomment for verbose output
    42  ## set -x
    43  
    44  ## must be @remais
    45  [[ "$bucket" == *//@* ]] || { echo "Error: expecting remote ais bucket, got ${bucket}"; exit 1; }
    46  
    47  ## actual bucket inside remais:
    48  rbucket="ais://$(basename ${bucket})"
    49  
    50  ## remais must be attached
    51  rendpoint=$(ais show remote-cluster -H | awk '{print $2}')
    52  [[ ! -z "$rendpoint" ]] || { echo "Error: no remote ais clusters"; exit 1; }
    53  
    54  ## check remote bucket; create if doesn't exist
    55  exists=true
    56  AIS_ENDPOINT=$rendpoint ais show bucket $rbucket -c 1>/dev/null 2>&1 || exists=false
    57  [[ "$exists" == "true" ]] || AIS_ENDPOINT=$rendpoint ais create $rbucket || exit 1
    58  
    59  ## add it to _this_ cluster's BMD
    60  ais show bucket $bucket --add 1>/dev/null || exit 1
    61  
    62  ## remember existing bucket 'validate_warm_get' setting; disable if need be
    63  validate=$(ais bucket props show ${bucket} versioning.validate_warm_get -H | awk '{print $2}')
    64  [[ "$validate" == "false"  ]] || ais bucket props set $bucket versioning.validate_warm_get=false
    65  sync=$(ais bucket props show ${bucket} versioning.synchronize -H | awk '{print $2}')
    66  [[ "$sync" == "false"  ]] || ais bucket props set $bucket versioning.synchronize=false
    67  
    68  cleanup() {
    69    rc=$?
    70    ais object rm "$bucket/lorem-duis" 1>/dev/null 2>&1
    71    [[ "$validate" == "true"  ]] || ais bucket props set $bucket versioning.validate_warm_get=false 1>/dev/null 2>&1
    72    [[ "$sync" == "true"  ]] || ais bucket props set $bucket versioning.synchronize=false 1>/dev/null 2>&1
    73    [[ "$exists" == "true" ]] || ais rmb $bucket -y 1>/dev/null 2>&1
    74    exit $rc
    75  }
    76  
    77  trap cleanup EXIT INT TERM
    78  
    79  echo -e
    80  ais show performance counters --regex "(GET-COLD$|VERSION-CHANGE$|DELETE)"
    81  echo -e
    82  
    83  echo "1. out-of-band PUT: 1st version"
    84  echo $lorem | AIS_ENDPOINT=$rendpoint ais put - "$rbucket/lorem-duis" 1>/dev/null || exit $?
    85  
    86  echo "2. cold GET, and check"
    87  ais get "$bucket/lorem-duis" /dev/null 1>/dev/null
    88  checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
    89  [[ "$checksum" == "$sum1"  ]] || { echo "FAIL: $checksum != $sum1"; exit 1; }
    90  
    91  echo "3. out-of-band PUT: 2nd version (overwrite)"
    92  echo $duis | AIS_ENDPOINT=$rendpoint ais put - "$rbucket/lorem-duis" 1>/dev/null || exit $?
    93  
    94  ######### --latest
    95  
    96  echo "3.1 'get --latest' without changing bucket props"
    97  
    98  ais get "$bucket/lorem-duis" /dev/null --latest 1>/dev/null
    99  checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
   100  [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
   101  
   102  echo "3.2 restore the state prior to step 3.1"
   103  
   104  echo $lorem | ais put - "$bucket/lorem-duis" 1>/dev/null || exit $?
   105  echo $duis  | AIS_ENDPOINT=$rendpoint ais put - "$rbucket/lorem-duis" $host 1>/dev/null || exit $?
   106  
   107  ######### end of --latest
   108  
   109  echo "4. warm GET and check (expecting the first version's checksum)"
   110  ais get "$bucket/lorem-duis" /dev/null 1>/dev/null
   111  checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
   112  [[ "$checksum" != "$sum2"  ]] || { echo "FAIL: $checksum == $sum2"; exit 1; }
   113  
   114  echo "5. update bucket props: set validate-warm-get = true"
   115  ais bucket props set $bucket versioning.validate_warm_get=true
   116  
   117  echo "6. query cold-get count (statistics)"
   118  cnt1=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
   119  
   120  echo "7. warm GET: detect version change and trigger cold GET"
   121  ais get "$bucket/lorem-duis" /dev/null 1>/dev/null
   122  checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
   123  [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
   124  
   125  echo "8. cold-get counter must increment"
   126  cnt2=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
   127  [[ $cnt2 == $(($cnt1+1)) ]] || { echo "FAIL: $cnt2 != $(($cnt1+1))"; exit 1; }
   128  
   129  echo "9. 2nd warm GET must remain \"warm\" and cold-get-count must not increment"
   130  ais get "$bucket/lorem-duis" /dev/null 1>/dev/null
   131  checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
   132  [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
   133  
   134  cnt3=$(ais show performance counters --regex GET-COLD -H | awk '{sum+=$2;}END{print sum;}')
   135  [[ $cnt3 == $cnt2 ]] || { echo "FAIL: $cnt3 != $cnt2"; exit 1; }
   136  
   137  echo "10. out-of-band DELETE"
   138  AIS_ENDPOINT=$rendpoint ais object rm "$rbucket/lorem-duis" 1>/dev/null || exit $?
   139  
   140  echo "11. update bucket props: disable validate-warm-get; subsequent warm GET must still succeed"
   141  ais bucket props set $bucket versioning.validate_warm_get=false
   142  
   143  ais get "$bucket/lorem-duis" /dev/null --silent 1>/dev/null 2>&1
   144  [[ $? == 0 ]] || { echo "FAIL: expecting warm GET to succeed, got $?"; exit 1; }
   145  
   146  echo "12. remember 'remote-deleted' counter and enable version synchronization"
   147  ais bucket props set $bucket versioning.synchronize=true
   148  
   149  cnt4=$(ais show performance counters --regex DELETED -H | awk '{sum+=$2;}END{print sum;}')
   150  
   151  echo "13. this time, warm GET must trigger deletion, 'remote-deleted' must increment"
   152  
   153  ais get "$bucket/lorem-duis" /dev/null --silent 1>/dev/null 2>&1
   154  [[ $? != 0 ]] || { echo "FAIL: expecting GET error, got $?"; exit 1; }
   155  ais ls "$bucket/lorem-duis" --cached --silent -H 2>/dev/null
   156  [[ $? != 0 ]] || { echo "FAIL: expecting 'show object' error, got $?"; exit 1; }
   157  
   158  cnt5=$(ais show performance counters --regex DELETED -H | awk '{sum+=$2;}END{print sum;}')
   159  [[ $cnt5 == $(($cnt4+1)) ]] || { echo "FAIL: $cnt5 != $(($cnt4+1))"; exit 1; }
   160  
   161  echo -e
   162  ais show performance counters --regex "(GET-COLD$|VERSION-CHANGE$|DELETE)"