github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ais/test/scripts/cp-rmdisk.sh (about)

     1  #!/bin/bash
     2  
     3  if ! [ -x "$(command -v ais)" ]; then
     4    echo "Error: ais (CLI) not installed" >&2
     5    exit 1
     6  fi
     7  
     8  ## start from scratch and generate 999 tgz shards
     9  ais bucket rm ais://src-mirror ais://dst -y 2>/dev/null
    10  
    11  ## NOTE: 3-way mirror implies that each target has at least 3 disks
    12  ais create ais://src-mirror --props="mirror.enabled=true mirror.copies=3" || \
    13  exit 1
    14  
    15  ais archive gen-shards 'ais://src-mirror/shard-{001..999}.tgz'
    16  
    17  ## list-objects to confirm
    18  num=$(ais ls ais://src-mirror --no-headers | wc -l)
    19  [[ $num == 999 ]] || { echo "FAIL: $num != 999"; exit 1; }
    20  
    21  cleanup() {
    22    ais storage mountpath attach $node $mountpath
    23  }
    24  
    25  trap cleanup EXIT INT TERM
    26  
    27  ## run forever or Ctrl-C
    28  while true
    29  do
    30    ## 1. start copying with an empty template (any prefix, any range)
    31    ais cp ais://src-mirror ais://dst --template ""
    32  
    33    ## randomize event timing: traffic vs node losing a disk
    34    sleep $((RANDOM % 5))
    35  
    36    ## 2. remove a random node's (random) mountpath (aka disk)
    37    node=$(ais advanced random-node)
    38    mountpath=$(ais advanced random-mountpath $node)
    39    ais storage mountpath detach $node $mountpath
    40  
    41    ## 3. wait for the copying job
    42    ais wait copy-objects
    43  
    44    ## 4. check the numbers
    45    res=$(ais ls ais://dst --no-headers | wc -l)
    46    [[ $num == $res ]] || { echo "FAIL: destination $num != $res"; exit 1; }
    47    ## the same using '--summary' option:
    48    res=$(ais ls ais://src-mirror --summary --no-headers | awk '{print $3}')
    49    [[ $num == $res ]] || { echo "FAIL: source $num != $res"; exit 1; }
    50  
    51    ## 5. attach the (previously lost) disk
    52    ais storage mountpath attach $node $mountpath
    53  
    54    ## 6. wait for the node to resilver itself
    55    ais wait resilver
    56  
    57    ## and double-check again
    58    res=$(ais ls ais://dst --summary --no-headers | awk '{print $3}')
    59    [[ $num == $res ]] || { echo "FAIL: destination $num != $res"; exit 1; }
    60    res=$(ais ls ais://src-mirror --summary --no-headers | awk '{print $3}')
    61    [[ $num == $res ]] || { echo "FAIL: source $num != $res"; exit 1; }
    62  
    63    ais bucket rm ais://dst -y
    64  done