github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/tests/integration_tests/force_replicate_table/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $CUR/../_utils/test_prepare
     7  WORK_DIR=$OUT_DIR/$TEST_NAME
     8  CDC_BINARY=cdc.test
     9  SINK_TYPE=$1
    10  
    11  function check_data_subset() {
    12  	tbl=$1
    13  	up_host=$2
    14  	up_port=$3
    15  	down_host=$4
    16  	down_port=$5
    17  	for i in $(seq 0 100); do
    18  		stmt="select * from $tbl order by id limit $i,1\G"
    19  		query=$(mysql -h${up_host} -P${up_port} -uroot -e "${stmt}")
    20  		clean_query="${query//\*/}"
    21  		if [ -n "$clean_query" ]; then
    22  			data_id=$(echo $clean_query | awk '{print $(NF-2)}')
    23  			data_a=$(echo $clean_query | awk '{print $(NF)}')
    24  			condition=""
    25  			if [[ "$data_id" -eq "NULL" ]]; then
    26  				condition="id is NULL"
    27  			else
    28  				condition="id=$data_id"
    29  			fi
    30  			condition="${condition} AND "
    31  			if [[ "$data_a" -eq "NULL" ]]; then
    32  				condition="${condition} a is NULL"
    33  			else
    34  				condition="${condition} a=$data_a"
    35  			fi
    36  			stmt2="select * from $tbl where $condition"
    37  			query2=$(mysql -h${down_host} -P${down_port} -uroot -e "${stmt2}")
    38  			if [ -z "$query2" ]; then
    39  				echo "id=$data_id,a=$data_a doesn't exist in downstream table $tbl"
    40  				exit 1
    41  			fi
    42  		fi
    43  	done
    44  }
    45  
    46  export -f check_data_subset
    47  
    48  function run() {
    49  	rm -rf $WORK_DIR && mkdir -p $WORK_DIR
    50  
    51  	start_tidb_cluster --workdir $WORK_DIR --tidb-config $CUR/conf/tidb_config.toml
    52  
    53  	cd $WORK_DIR
    54  
    55  	# record tso before we create tables to skip the system table DDLs
    56  	start_ts=$(cdc cli tso query --pd=http://$UP_PD_HOST_1:$UP_PD_PORT_1)
    57  
    58  	run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY
    59  
    60  	TOPIC_NAME="ticdc-force_replicate_table-test-$RANDOM"
    61  	case $SINK_TYPE in
    62  	kafka) SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&kafka-version=${KAFKA_VERSION}&max-message-bytes=10485760" ;;
    63  	storage) SINK_URI="file://$WORK_DIR/storage_test/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true" ;;
    64  	pulsar)
    65  		run_pulsar_cluster $WORK_DIR normal
    66  		SINK_URI="pulsar://127.0.0.1:6650/$TOPIC_NAME?protocol=canal-json&enable-tidb-extension=true"
    67  		;;
    68  	*) SINK_URI="mysql://normal:123456@127.0.0.1:3306/?safe-mode=true" ;;
    69  	esac
    70  
    71  	cdc cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" --config $CUR/conf/changefeed.toml
    72  
    73  	case $SINK_TYPE in
    74  	kafka) run_kafka_consumer $WORK_DIR $SINK_URI $CUR/conf/changefeed.toml ;;
    75  	storage) run_storage_consumer $WORK_DIR $SINK_URI $CUR/conf/changefeed.toml "" ;;
    76  	pulsar) run_pulsar_consumer --upstream-uri $SINK_URI --config $CUR/conf/changefeed.toml ;;
    77  	esac
    78  
    79  	run_sql_file $CUR/data/test.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    80  	for i in $(seq 0 6); do
    81  		table="force_replicate_table.t$i"
    82  		check_table_exists $table ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
    83  	done
    84  	# data could be duplicated due to https://github.com/pingcap/tiflow/issues/964,
    85  	# so we just check downstream contains all data in upstream.
    86  	for i in $(seq 0 6); do
    87  		ensure 10 check_data_subset "force_replicate_table.t$i" \
    88  			${UP_TIDB_HOST} ${UP_TIDB_PORT} ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
    89  	done
    90  	cleanup_process $CDC_BINARY
    91  }
    92  
    93  trap stop_tidb_cluster EXIT
    94  run $*
    95  check_logs $WORK_DIR
    96  echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"