github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/tests/changefeed_error/run.sh (about) 1 #!/bin/bash 2 3 set -e 4 5 CUR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 6 source $CUR/../_utils/test_prepare 7 WORK_DIR=$OUT_DIR/$TEST_NAME 8 CDC_BINARY=cdc.test 9 SINK_TYPE=$1 10 MAX_RETRIES=10 11 12 function check_changefeed_mark_failed() { 13 endpoints=$1 14 changefeedid=$2 15 error_msg=$3 16 info=$(cdc cli changefeed query --pd=$endpoints -c $changefeedid -s) 17 echo "$info" 18 state=$(echo $info|jq -r '.state') 19 if [[ ! "$state" == "failed" ]]; then 20 echo "changefeed state $state does not equal to failed" 21 exit 1 22 fi 23 message=$(echo $info|jq -r '.error.message') 24 if [[ ! "$message" =~ "$error_msg" ]]; then 25 echo "error message '$message' is not as expected '$error_msg'" 26 exit 1 27 fi 28 } 29 30 function check_changefeed_mark_failed_regex() { 31 endpoints=$1 32 changefeedid=$2 33 error_msg=$3 34 info=$(cdc cli changefeed query --pd=$endpoints -c $changefeedid -s) 35 echo "$info" 36 state=$(echo $info|jq -r '.state') 37 if [[ ! "$state" == "failed" ]]; then 38 echo "changefeed state $state does not equal to failed" 39 exit 1 40 fi 41 message=$(echo $info|jq -r '.error.message') 42 if [[ ! "$message" =~ $error_msg ]]; then 43 echo "error message '$message' does not match '$error_msg'" 44 exit 1 45 fi 46 } 47 48 function check_changefeed_mark_stopped_regex() { 49 endpoints=$1 50 changefeedid=$2 51 error_msg=$3 52 info=$(cdc cli changefeed query --pd=$endpoints -c $changefeedid -s) 53 echo "$info" 54 state=$(echo $info|jq -r '.state') 55 if [[ ! "$state" == "stopped" ]]; then 56 echo "changefeed state $state does not equal to stopped" 57 exit 1 58 fi 59 message=$(echo $info|jq -r '.error.message') 60 if [[ ! "$message" =~ $error_msg ]]; then 61 echo "error message '$message' does not match '$error_msg'" 62 exit 1 63 fi 64 } 65 66 function check_changefeed_mark_stopped() { 67 endpoints=$1 68 changefeedid=$2 69 error_msg=$3 70 info=$(cdc cli changefeed query --pd=$endpoints -c $changefeedid -s) 71 echo "$info" 72 state=$(echo $info|jq -r '.state') 73 if [[ ! "$state" == "stopped" ]]; then 74 echo "changefeed state $state does not equal to stopped" 75 exit 1 76 fi 77 message=$(echo $info|jq -r '.error.message') 78 if [[ ! "$message" =~ "$error_msg" ]]; then 79 echo "error message '$message' is not as expected '$error_msg'" 80 exit 1 81 fi 82 } 83 84 function check_no_changefeed() { 85 pd=$1 86 count=$(cdc cli changefeed list --pd=$pd 2>&1|jq '.|length') 87 if [[ ! "$count" -eq "0" ]]; then 88 exit 1 89 fi 90 } 91 92 function check_no_capture() { 93 pd=$1 94 count=$(cdc cli capture list --pd=$pd 2>&1|jq '.|length') 95 if [[ ! "$count" -eq "0" ]]; then 96 exit 1 97 fi 98 } 99 100 export -f check_changefeed_mark_failed 101 export -f check_changefeed_mark_failed_regex 102 export -f check_changefeed_mark_stopped_regex 103 export -f check_changefeed_mark_stopped 104 export -f check_no_changefeed 105 export -f check_no_capture 106 107 function run() { 108 rm -rf $WORK_DIR && mkdir -p $WORK_DIR 109 110 start_tidb_cluster --workdir $WORK_DIR 111 112 cd $WORK_DIR 113 114 start_ts=$(run_cdc_cli tso query --pd=http://$UP_PD_HOST_1:$UP_PD_PORT_1) 115 run_sql "CREATE DATABASE changefeed_error;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} 116 go-ycsb load mysql -P $CUR/conf/workload -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=changefeed_error 117 export GO_FAILPOINTS='github.com/pingcap/ticdc/cdc/NewChangefeedNoRetryError=1*return(true)' 118 run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY 119 capture_pid=$(ps -C $CDC_BINARY -o pid= | awk '{print $1}') 120 121 TOPIC_NAME="ticdc-sink-retry-test-$RANDOM" 122 case $SINK_TYPE in 123 kafka) SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&kafka-version=${KAFKA_VERSION}";; 124 *) SINK_URI="mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1";; 125 esac 126 changefeedid=$(cdc cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" 2>&1|tail -n2|head -n1|awk '{print $2}') 127 if [ "$SINK_TYPE" == "kafka" ]; then 128 run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&version=${KAFKA_VERSION}" 129 fi 130 131 ensure $MAX_RETRIES check_changefeed_mark_failed_regex http://${UP_PD_HOST_1}:${UP_PD_PORT_1} ${changefeedid} ".*CDC:ErrStartTsBeforeGC.*" 132 changefeed_info=$(ETCDCTL_API=3 etcdctl --endpoints=${UP_PD_HOST_1}:${UP_PD_PORT_1} get /tidb/cdc/changefeed/info/${changefeedid}|tail -n 1) 133 new_info=$(echo $changefeed_info|sed 's/"state":"failed"/"state":"normal"/g') 134 ETCDCTL_API=3 etcdctl --endpoints=${UP_PD_HOST_1}:${UP_PD_PORT_1} put /tidb/cdc/changefeed/info/${changefeedid} "$new_info" 135 136 check_table_exists "changefeed_error.USERTABLE" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 137 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml 138 139 go-ycsb load mysql -P $CUR/conf/workload -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=changefeed_error 140 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml 141 142 export GO_FAILPOINTS='github.com/pingcap/ticdc/cdc/NewChangefeedRetryError=return(true)' 143 kill $capture_pid 144 ensure $MAX_RETRIES check_no_capture http://${UP_PD_HOST_1}:${UP_PD_PORT_1} 145 run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY 146 ensure $MAX_RETRIES check_changefeed_mark_failed http://${UP_PD_HOST_1}:${UP_PD_PORT_1} ${changefeedid} "failpoint injected retriable error" 147 148 cdc cli changefeed remove -c $changefeedid 149 ensure $MAX_RETRIES check_no_changefeed ${UP_PD_HOST_1}:${UP_PD_PORT_1} 150 151 export GO_FAILPOINTS='' 152 cleanup_process $CDC_BINARY 153 154 # changefeed Async DDL error case 155 export GO_FAILPOINTS='github.com/pingcap/ticdc/cdc/InjectChangefeedAsyncDDLError=return(true)' 156 run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY 157 changefeedid_0=$(cdc cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" 2>&1|tail -n2|head -n1|awk '{print $2}') 158 159 run_sql "CREATE table changefeed_error.AsyncDDLERROR(id int primary key, val int);" 160 ensure $MAX_RETRIES check_changefeed_mark_stopped http://${UP_PD_HOST_1}:${UP_PD_PORT_1} ${changefeedid_0} "[CDC:ErrExecDDLFailed]InjectChangefeedAsyncDDLError" 161 162 163 cdc cli changefeed remove -c $changefeedid_0 164 cleanup_process $CDC_BINARY 165 166 # owner DDL error case 167 export GO_FAILPOINTS='github.com/pingcap/ticdc/cdc/InjectChangefeedDDLError=return(true)' 168 run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY 169 changefeedid_1=$(cdc cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" 2>&1|tail -n2|head -n1|awk '{print $2}') 170 171 run_sql "CREATE table changefeed_error.DDLERROR(id int primary key, val int);" 172 ensure $MAX_RETRIES check_changefeed_mark_stopped http://${UP_PD_HOST_1}:${UP_PD_PORT_1} ${changefeedid_1} "[CDC:ErrExecDDLFailed]exec DDL failed" 173 174 cdc cli changefeed remove -c $changefeedid_1 175 cleanup_process $CDC_BINARY 176 177 # updating GC safepoint failure case 178 export GO_FAILPOINTS='github.com/pingcap/ticdc/cdc/InjectActualGCSafePoint=return(9223372036854775807)' 179 run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY 180 181 changefeedid_2=$(cdc cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" 2>&1|tail -n2|head -n1|awk '{print $2}') 182 ensure $MAX_RETRIES check_changefeed_mark_stopped_regex http://${UP_PD_HOST_1}:${UP_PD_PORT_1} ${changefeedid_2} "service safepoint lost" 183 184 cdc cli changefeed remove -c $changefeedid_2 185 export GO_FAILPOINTS='' 186 cleanup_process $CDC_BINARY 187 } 188 189 trap stop_tidb_cluster EXIT 190 run $* 191 check_logs $WORK_DIR 192 echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"