github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ais/test/scripts/cp-rmnode-rebalance.sh (about)

     1  #!/bin/bash
     2  
     3  if ! [ -x "$(command -v ais)" ]; then
     4    echo "Error: ais (CLI) not installed" >&2
     5    exit 1
     6  fi
     7  
     8  ## start from scratch and generate 999 tgz shards
     9  ais bucket rm ais://src ais://dst -y 2>/dev/null
    10  ais create ais://src
    11  ais archive gen-shards 'ais://src/shard-{001..999}.tgz'
    12  
    13  ## list-objects to confirm
    14  num=$(ais ls ais://src --no-headers | wc -l)
    15  [[ $num == 999 ]] || { echo "FAIL: $num != 999"; exit 1; }
    16  
    17  cleanup() {
    18    ais cluster add-remove-nodes stop-maintenance $node
    19  }
    20  
    21  trap cleanup EXIT INT TERM
    22  
    23  ## run forever or Ctrl-C
    24  while true
    25  do
    26    ## 1. start copying with an empty template (any prefix, any range)
    27    ais cp ais://src ais://dst --template ""
    28  
    29    ## randomize event timing: traffic vs cluster losing a node
    30    sleep $((RANDOM % 5))
    31  
    32    ## 2. gracefully remove a random node
    33    node=$(ais advanced random-node)
    34    ais cluster add-remove-nodes start-maintenance $node -y ## triggers global rebalance
    35    sleep $((2 + RANDOM % 2))
    36  
    37    ## 3. wait for the copy
    38    ais wait copy-objects
    39  
    40    ## 4. check the numbers
    41    res=$(ais ls ais://dst --no-headers | wc -l)
    42    [[ $num == $res ]] || { echo "FAIL: destination $num != $res"; exit 1; }
    43    ## the same using '--summary' option:
    44    res=$(ais ls ais://src --summary --no-headers | awk '{print $3}')
    45    [[ $num == $res ]] || { echo "FAIL: source $num != $res"; exit 1; }
    46  
    47    ## 5. activate the (previously lost) node and join it back
    48    ## (rebalance from step 2 may still be running at this point - it'll be aborted and restarted)
    49    ais cluster add-remove-nodes stop-maintenance $node
    50  
    51    ## 6. cleanup and repeat
    52    ais wait rebalance
    53  
    54    ## and double-check again
    55    res=$(ais ls ais://dst --summary --no-headers | awk '{print $3}')
    56    [[ $num == $res ]] || { echo "FAIL: destination $num != $res"; exit 1; }
    57    res=$(ais ls ais://src --summary --no-headers | awk '{print $3}')
    58    [[ $num == $res ]] || { echo "FAIL: source $num != $res"; exit 1; }
    59  
    60    ais bucket rm ais://dst -y
    61  done