github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/tests/force_replicate_table/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -e
     4  
     5  CUR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
     6  source $CUR/../_utils/test_prepare
     7  WORK_DIR=$OUT_DIR/$TEST_NAME
     8  CDC_BINARY=cdc.test
     9  SINK_TYPE=$1
    10  
    11  function check_data_subset() {
    12      tbl=$1
    13      up_host=$2
    14      up_port=$3
    15      down_host=$4
    16      down_port=$5
    17      for i in $(seq 0 100); do
    18          stmt="select * from $tbl order by id limit $i,1\G"
    19          query=$(mysql -h${up_host} -P${up_port} -uroot -e "${stmt}")
    20          clean_query="${query//\*/}"
    21          if [ -n "$clean_query" ]; then
    22              data_id=$(echo $clean_query|awk '{print $(NF-2)}')
    23              data_a=$(echo $clean_query|awk '{print $(NF)}')
    24              condition=""
    25              if [[ "$data_id" -eq "NULL" ]]; then
    26                  condition="id is NULL"
    27              else
    28                  condition="id=$data_id"
    29              fi
    30              condition="${condition} AND "
    31              if [[ "$data_a" -eq "NULL" ]]; then
    32                  condition="${condition} a is NULL"
    33              else
    34                  condition="${condition} a=$data_a"
    35              fi
    36              stmt2="select * from $tbl where $condition"
    37              query2=$(mysql -h${down_host} -P${down_port} -uroot -e "${stmt2}")
    38              if [ -z "$query2" ]; then
    39                  echo "id=$data_id,a=$data_a doesn't exist in downstream table $tbl"
    40                  exit 1
    41              fi
    42          fi
    43      done
    44  }
    45  
    46  export -f check_data_subset
    47  
    48  function run() {
    49      rm -rf $WORK_DIR && mkdir -p $WORK_DIR
    50  
    51      start_tidb_cluster --workdir $WORK_DIR --tidb-config $CUR/conf/tidb_config.toml
    52  
    53      cd $WORK_DIR
    54  
    55      # record tso before we create tables to skip the system table DDLs
    56      start_ts=$(cdc cli tso query --pd=http://$UP_PD_HOST_1:$UP_PD_PORT_1)
    57  
    58      run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY
    59  
    60      TOPIC_NAME="ticdc-force_replicate_table-test-$RANDOM"
    61      case $SINK_TYPE in
    62          kafka) SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&kafka-version=${KAFKA_VERSION}";;
    63          *) SINK_URI="mysql://normal:123456@127.0.0.1:3306/?safe-mode=true";;
    64      esac
    65      cdc cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" --config $CUR/conf/changefeed.toml
    66      if [ "$SINK_TYPE" == "kafka" ]; then
    67        run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&version=${KAFKA_VERSION}"
    68      fi
    69  
    70      run_sql_file $CUR/data/test.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    71      for i in $(seq 0 6); do
    72          table="force_replicate_table.t$i"
    73          check_table_exists $table ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
    74      done
    75      # data could be duplicated due to https://github.com/pingcap/ticdc/issues/964,
    76      # so we just check downstream contains all data in upstream.
    77      for i in $(seq 0 6); do
    78          ensure 5 check_data_subset "force_replicate_table.t$i" \
    79              ${UP_TIDB_HOST} ${UP_TIDB_PORT} ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
    80      done
    81      cleanup_process $CDC_BINARY
    82  }
    83  
    84  trap stop_tidb_cluster EXIT
    85  run $*
    86  check_logs $WORK_DIR
    87  echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"