github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/_utils/run_dm_ctl_with_retry (about)

     1  #!/bin/bash
     2  # tools to run dmctl from command line
     3  # parameter 1: work directory
     4  # parameter 2: master-addr port
     5  # parameter 3: command
     6  # parameter 4...: check output content and count
     7  
     8  workdir=$1
     9  master_addr=$2
    10  cmd=$3
    11  
    12  shift 3
    13  
    14  PWD=$(pwd)
    15  binary=$PWD/bin/dmctl.test
    16  ts=$(date +"%s")
    17  dmctl_log=$workdir/dmctl.$ts.log
    18  pid=$$
    19  
    20  all_matched=true
    21  echo "dmctl test cmd: \"$cmd\""
    22  for ((k = 0; k < 10; k++)); do
    23  	echo "$cmd" | $binary -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.dmctl.$ts.$pid.out" DEVEL --master-addr=$master_addr >$dmctl_log 2>&1
    24  	all_matched=true
    25  	for ((i = 1; i < $#; i += 2)); do
    26  		j=$((i + 1))
    27  		value=${!i}
    28  		expected=${!j}
    29  		got=$(sed "s/$value/$value\n/g" $dmctl_log | grep -c "$value")
    30  		echo "got=$got expected=$expected"
    31  		if [ "$expected" == "1+" ]; then
    32  			if [ "$got" == "0" ]; then
    33  				echo "command: $cmd $value count: $got 0, expect 1+, failed the $k-th time, will retry again"
    34  				all_matched=false
    35  				break
    36  			fi
    37  		elif [ "$got" != "$expected" ]; then
    38  			echo "command: $cmd $value count: $got != expected: $expected, failed the $k-th time, will retry again"
    39  			all_matched=false
    40  			break
    41  		fi
    42  	done
    43  
    44  	if $all_matched; then
    45  		exit 0
    46  	fi
    47  
    48  	sleep 2
    49  done
    50  
    51  cat $dmctl_log
    52  exit 1