github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/tests/kafka_messages/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -e
     4  
     5  CUR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
     6  source $CUR/../_utils/test_prepare
     7  WORK_DIR=$OUT_DIR/$TEST_NAME
     8  CDC_BINARY=cdc.test
     9  SINK_TYPE=$1
    10  
    11  CDC_COUNT=3
    12  DB_COUNT=4
    13  
    14  function run_length_limit() {
    15      # test kafka sink only in this case
    16      if [ "$SINK_TYPE" == "mysql" ]; then
    17        return
    18      fi
    19  
    20      rm -rf $WORK_DIR && mkdir -p $WORK_DIR
    21  
    22      start_tidb_cluster --workdir $WORK_DIR
    23  
    24      cd $WORK_DIR
    25  
    26      run_sql "DROP DATABASE if exists kafka_message;" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    27  
    28      start_ts=$(run_cdc_cli tso query --pd=http://$UP_PD_HOST_1:$UP_PD_PORT_1)
    29      run_sql "CREATE DATABASE kafka_message;" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    30      go-ycsb load mysql -P $CUR/conf/workload -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=kafka_message
    31      run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --loglevel "info"
    32  
    33      TOPIC_NAME="ticdc-kafka-message-test-$RANDOM"
    34      SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&max-message-bytes=4096&kafka-version=${KAFKA_VERSION}"
    35      run_cdc_cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI"
    36      if [ "$SINK_TYPE" == "kafka" ]; then
    37        run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&max-message-bytes=4096&version=${KAFKA_VERSION}"
    38      fi
    39  
    40      # Add a check table to reduce check time, or if we check data with sync diff
    41      # directly, there maybe a lot of diff data at first because of the incremental scan
    42      run_sql "CREATE table kafka_message.check1(id int primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    43      check_table_exists "kafka_message.USERTABLE" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
    44      check_table_exists "kafka_message.check1" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
    45      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
    46  
    47      run_sql "truncate table kafka_message.USERTABLE" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    48      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
    49      run_sql "CREATE table kafka_message.check2(id int primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    50      check_table_exists "kafka_message.check2" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
    51      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
    52  
    53      go-ycsb load mysql -P $CUR/conf/workload -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=kafka_message
    54      run_sql "CREATE table kafka_message.check3(id int primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    55      check_table_exists "kafka_message.check3" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
    56      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
    57  
    58      run_sql "create table kafka_message.USERTABLE2 like kafka_message.USERTABLE" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    59      run_sql "insert into kafka_message.USERTABLE2 select * from kafka_message.USERTABLE" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    60      run_sql "create table kafka_message.check4(id int primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    61      check_table_exists "kafka_message.USERTABLE2" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
    62      check_table_exists "kafka_message.check4" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
    63  
    64      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
    65  
    66      cleanup_process $CDC_BINARY
    67      stop_tidb_cluster
    68  }
    69  
    70  function run_batch_size_limit() {
    71        # test kafka sink only in this case
    72      if [ "$SINK_TYPE" == "mysql" ]; then
    73        return
    74      fi
    75  
    76      rm -rf $WORK_DIR && mkdir -p $WORK_DIR
    77  
    78      start_tidb_cluster --workdir $WORK_DIR
    79  
    80      cd $WORK_DIR
    81  
    82      run_sql "DROP DATABASE if exists kafka_message;" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    83  
    84      start_ts=$(run_cdc_cli tso query --pd=http://$UP_PD_HOST_1:$UP_PD_PORT_1)
    85      run_sql "CREATE DATABASE kafka_message;" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    86      go-ycsb load mysql -P $CUR/conf/workload -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=kafka_message
    87      run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY --loglevel "info"
    88  
    89      TOPIC_NAME="ticdc-kafka-message-test-$RANDOM"
    90      SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&max-batch-size=3&kafka-version=${KAFKA_VERSION}"
    91      run_cdc_cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI"
    92      if [ "$SINK_TYPE" == "kafka" ]; then
    93        run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&max-batch-size=3&version=${KAFKA_VERSION}"
    94      fi
    95  
    96      # Add a check table to reduce check time, or if we check data with sync diff
    97      # directly, there maybe a lot of diff data at first because of the incremental scan
    98      run_sql "CREATE table kafka_message.check1(id int primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    99      check_table_exists "kafka_message.USERTABLE" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
   100      check_table_exists "kafka_message.check1" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
   101      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
   102  
   103      run_sql "truncate table kafka_message.USERTABLE" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
   104      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
   105      run_sql "CREATE table kafka_message.check2(id int primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
   106      check_table_exists "kafka_message.check2" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
   107      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
   108  
   109      go-ycsb load mysql -P $CUR/conf/workload -p mysql.host=${UP_TIDB_HOST} -p mysql.port=${UP_TIDB_PORT} -p mysql.user=root -p mysql.db=kafka_message
   110      run_sql "CREATE table kafka_message.check3(id int primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
   111      check_table_exists "kafka_message.check3" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
   112      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
   113  
   114      run_sql "create table kafka_message.USERTABLE2 like kafka_message.USERTABLE" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
   115      run_sql "insert into kafka_message.USERTABLE2 select * from kafka_message.USERTABLE" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
   116      run_sql "create table kafka_message.check4(id int primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
   117      check_table_exists "kafka_message.USERTABLE2" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
   118      check_table_exists "kafka_message.check4" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90
   119  
   120      check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
   121  
   122      cleanup_process $CDC_BINARY
   123      stop_tidb_cluster
   124  }
   125  
   126  trap stop_tidb_cluster EXIT
   127  run_length_limit $*
   128  run_batch_size_limit $*
   129  check_logs $WORK_DIR
   130  echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"