github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/tests/integration_tests/consistent_replicate_gbk/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $CUR/../_utils/test_prepare
     7  WORK_DIR=$OUT_DIR/$TEST_NAME
     8  CDC_BINARY=cdc.test
     9  SINK_TYPE=$1
    10  
    11  # start the s3 server
    12  export MINIO_ACCESS_KEY=cdcs3accesskey
    13  export MINIO_SECRET_KEY=cdcs3secretkey
    14  export MINIO_BROWSER=off
    15  export AWS_ACCESS_KEY_ID=$MINIO_ACCESS_KEY
    16  export AWS_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
    17  export S3_ENDPOINT=127.0.0.1:24927
    18  rm -rf "$WORK_DIR"
    19  mkdir -p "$WORK_DIR"
    20  pkill -9 minio || true
    21  bin/minio server --address $S3_ENDPOINT "$WORK_DIR/s3" &
    22  MINIO_PID=$!
    23  i=0
    24  while ! curl -o /dev/null -v -s "http://$S3_ENDPOINT/"; do
    25  	i=$(($i + 1))
    26  	if [ $i -gt 30 ]; then
    27  		echo 'Failed to start minio'
    28  		exit 1
    29  	fi
    30  	sleep 2
    31  done
    32  
    33  stop_minio() {
    34  	kill -2 $MINIO_PID
    35  }
    36  
    37  stop() {
    38  	stop_minio
    39  	stop_tidb_cluster
    40  }
    41  
    42  s3cmd --access_key=$MINIO_ACCESS_KEY --secret_key=$MINIO_SECRET_KEY --host=$S3_ENDPOINT --host-bucket=$S3_ENDPOINT --no-ssl mb s3://logbucket
    43  
    44  function run() {
    45  	# we only support eventually consistent replication with MySQL sink
    46  	if [ "$SINK_TYPE" != "mysql" ]; then
    47  		return
    48  	fi
    49  
    50  	start_tidb_cluster --workdir $WORK_DIR
    51  
    52  	cd $WORK_DIR
    53  
    54  	run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY
    55  
    56  	SINK_URI="mysql://normal:123456@127.0.0.1:3306/"
    57  	changefeed_id=$(cdc cli changefeed create --sink-uri="$SINK_URI" --config="$CUR/conf/changefeed.toml" 2>&1 | tail -n2 | head -n1 | awk '{print $2}')
    58  
    59  	run_sql "CREATE DATABASE consistent_replicate_gbk;" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    60  	run_sql "CREATE TABLE consistent_replicate_gbk.GBKTABLE (id INT,name varchar(128),country char(32),city varchar(64),description text,image tinyblob,PRIMARY KEY (id)) ENGINE = InnoDB CHARSET = gbk;" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    61  	run_sql "INSERT INTO consistent_replicate_gbk.GBKTABLE VALUES (1, '测试', '中国', '上海', '你好,世界', 0xC4E3BAC3CAC0BDE7);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    62  	run_sql "CREATE table consistent_replicate_gbk.check1(id int primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    63  	check_table_exists "consistent_replicate_gbk.GBKTABLE" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
    64  	check_table_exists "consistent_replicate_gbk.check1" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 120
    65  	check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
    66  
    67  	# Inject the failpoint to prevent sink execution, but the global resolved can be moved forward.
    68  	# Then we can apply redo log to reach an eventual consistent state in downstream.
    69  	cleanup_process $CDC_BINARY
    70  	export GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)'
    71  	run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY
    72  	run_sql "create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    73  	run_sql "insert into consistent_replicate_gbk.GBKTABLE2 select * from consistent_replicate_gbk.GBKTABLE" ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    74  
    75  	# to ensure row changed events have been replicated to TiCDC
    76  	sleep 5
    77  
    78  	storage_path="s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/"
    79  	tmp_download_path=$WORK_DIR/cdc_data/redo/$changefeed_id
    80  	current_tso=$(cdc cli tso query --pd=http://$UP_PD_HOST_1:$UP_PD_PORT_1)
    81  	ensure 20 check_redo_resolved_ts $changefeed_id $current_tso $storage_path $tmp_download_path/meta
    82  	cleanup_process $CDC_BINARY
    83  
    84  	export GO_FAILPOINTS=''
    85  	export AWS_ACCESS_KEY_ID=$MINIO_ACCESS_KEY
    86  	export AWS_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
    87  	cdc redo apply --tmp-dir="$WORK_DIR/apply" \
    88  		--storage="$storage_path" \
    89  		--sink-uri="mysql://normal:123456@127.0.0.1:3306/"
    90  	check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml
    91  }
    92  
    93  trap stop EXIT
    94  run $*
    95  check_logs $WORK_DIR
    96  echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"