github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ais/test/scripts/cp-rmnode-ec.sh (about)

     1  #!/bin/bash
     2  
     3  if ! [ -x "$(command -v ais)" ]; then
     4    echo "Error: ais (CLI) not installed" >&2
     5    exit 1
     6  fi
     7  
     8  ais bucket rm ais://src-ec ais://dst -y 2>/dev/null
     9  
    10  ## create erasure-coded bucket
    11  ## NOTE: must have enough nodes in the cluster (i.e., 5 or more nodes in this case)
    12  ais create ais://src-ec --props "ec.enabled=true ec.data_slices=3 ec.parity_slices=1 ec.objsize_limit=0" || \
    13  exit 1
    14  
    15  ## generate a large number (say, 999) tar shards
    16  ais archive gen-shards 'ais://src-ec/shard-{001..999}.tar'
    17  
    18  ## list-objects to confirm
    19  num=$(ais ls ais://src-ec --no-headers | wc -l)
    20  [[ $num == 999 ]] || { echo "FAIL: $num != 999"; exit 1; }
    21  
    22  cleanup() {
    23    ais cluster add-remove-nodes stop-maintenance $node
    24  }
    25  
    26  trap cleanup EXIT INT TERM
    27  
    28  ## run forever or Ctrl-C
    29  while true
    30  do
    31    ## 1. start copying
    32    ais cp ais://src-ec ais://dst --template ""
    33  
    34    ## randomize event timing: traffic vs cluster losing a node
    35    sleep $((RANDOM % 5))
    36  
    37    ## 2. remove a random node immediately (no rebalance!)
    38    node=$(ais advanced random-node)
    39    ais cluster add-remove-nodes start-maintenance $node --no-rebalance -y
    40  
    41    ## 3. wait for the copying job to finish
    42    ais wait copy-objects
    43  
    44    ## 4. activate and join back
    45    ais cluster add-remove-nodes stop-maintenance $node
    46    ais wait rebalance
    47  
    48    ## 5. check numbers
    49    res=$(ais ls ais://src-ec --no-headers | wc -l)
    50    [[ $num == $res ]] || { echo "FAIL: source $num != $res"; exit 1; }
    51    ## alternatively, using summary:
    52    res=$(ais ls ais://dst --no-headers --summary | awk '{print $3}')
    53    [[ $num == $res ]] || { echo "FAIL: destination $num != $res"; exit 1; }
    54  
    55    ## 6. cleanup and repeat
    56    ais bucket rm ais://dst -y
    57  done