github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/tests/integration_tests/ddl_puller_lag/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $CUR/../_utils/test_prepare
     7  WORK_DIR=$OUT_DIR/$TEST_NAME
     8  CDC_BINARY=cdc.test
     9  SINK_TYPE=$1
    10  
    11  function prepare() {
    12  	rm -rf $WORK_DIR && mkdir -p $WORK_DIR
    13  
    14  	start_tidb_cluster --workdir $WORK_DIR
    15  
    16  	cd $WORK_DIR
    17  
    18  	# record tso before we create tables to skip the system table DDLs
    19  	start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1})
    20  
    21  	run_sql "CREATE table test.ddl_puller_lag1(id int primary key, val int);"
    22  	run_sql "CREATE table test.ddl_puller_lag2(id int primary key, val int);"
    23  
    24  	run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --failpoint 'github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)'
    25  
    26  	TOPIC_NAME="ticdc-ddl-puller-lag-test-$RANDOM"
    27  	case $SINK_TYPE in
    28  	kafka) SINK_URI="kafka+ssl://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&kafka-client-id=ddl_puller_lag&kafka-version=${KAFKA_VERSION}&max-message-bytes=10485760" ;;
    29  	storage) SINK_URI="file://$WORK_DIR/storage_test/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true" ;;
    30  	pulsar)
    31  		run_pulsar_cluster $WORK_DIR normal
    32  		SINK_URI="pulsar://127.0.0.1:6650/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true"
    33  		;;
    34  	*) SINK_URI="mysql+ssl://normal:123456@127.0.0.1:3306/" ;;
    35  	esac
    36  	run_cdc_cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI"
    37  	case $SINK_TYPE in
    38  	kafka) run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&version=${KAFKA_VERSION}&max-message-bytes=10485760" ;;
    39  	storage) run_storage_consumer $WORK_DIR $SINK_URI "" "" ;;
    40  	pulsar) run_pulsar_consumer --upstream-uri $SINK_URI ;;
    41  	esac
    42  }
    43  
    44  function sql_check() {
    45  	# run check in sequence and short circuit principle, if error hanppens,
    46  	# the following statement will be not executed
    47  
    48  	# check table ddl_puller_lag1.
    49  	run_sql "SELECT id, val FROM test.ddl_puller_lag1;" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} &&
    50  		check_contains "id: 1" &&
    51  		check_contains "val: 1" &&
    52  		check_contains "id: 2" &&
    53  		check_contains "val: 22" &&
    54  		check_not_contains "id: 3" &&
    55  
    56  		# check table ddl_puller_lag2.
    57  		run_sql "SELECT id, val FROM test.ddl_puller_lag2;" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} &&
    58  		check_contains "id: 1" &&
    59  		check_contains "val: 1" &&
    60  		check_contains "id: 2" &&
    61  		check_contains "val: 22" &&
    62  		check_not_contains "id: 3"
    63  }
    64  
    65  function sql_test() {
    66  	# test insert/update/delete for two table in the same way.
    67  	run_sql "INSERT INTO test.ddl_puller_lag1(id, val) VALUES (1, 1);"
    68  	run_sql "INSERT INTO test.ddl_puller_lag1(id, val) VALUES (2, 2);"
    69  	run_sql "INSERT INTO test.ddl_puller_lag1(id, val) VALUES (3, 3);"
    70  
    71  	# update id = 2 and delete id = 3
    72  	run_sql "UPDATE test.ddl_puller_lag1 set val = 22 where id = 2;"
    73  	run_sql "DELETE from test.ddl_puller_lag1 where id = 3;"
    74  
    75  	# same dml for table ddl_puller_lag2
    76  	run_sql "INSERT INTO test.ddl_puller_lag2(id, val) VALUES (1, 1);"
    77  	run_sql "INSERT INTO test.ddl_puller_lag2(id, val) VALUES (2, 2);"
    78  	run_sql "INSERT INTO test.ddl_puller_lag2(id, val) VALUES (3, 3);"
    79  
    80  	run_sql "UPDATE test.ddl_puller_lag2 set val = 22 where id = 2;"
    81  	run_sql "DELETE from test.ddl_puller_lag2 where id = 3;"
    82  
    83  	i=0
    84  	check_time=50
    85  	set +e
    86  	while [ $i -lt $check_time ]; do
    87  		sql_check
    88  		ret=$?
    89  		if [ "$ret" == 0 ]; then
    90  			echo "check data successfully"
    91  			break
    92  		fi
    93  		((i++))
    94  		echo "check data failed $i-th time, retry later"
    95  		sleep 2
    96  	done
    97  	set -e
    98  
    99  	if [ $i -ge $check_time ]; then
   100  		echo "check data failed at last"
   101  		exit 1
   102  	fi
   103  
   104  	cleanup_process $CDC_BINARY
   105  }
   106  
   107  trap stop_tidb_cluster EXIT
   108  prepare $*
   109  sleep 180
   110  sql_test $*
   111  check_logs $WORK_DIR
   112  echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"