github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/tests/integration_tests/storage_csv_update/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $CUR/../_utils/test_prepare
     7  WORK_DIR=$OUT_DIR/$TEST_NAME
     8  CDC_BINARY=cdc.test
     9  SINK_TYPE=$1
    10  
    11  function run() {
    12  	if [ "$SINK_TYPE" != "storage" ]; then
    13  		return
    14  	fi
    15  
    16  	rm -rf $WORK_DIR && mkdir -p $WORK_DIR
    17  	start_tidb_cluster --workdir $WORK_DIR
    18  	cd $WORK_DIR
    19  	run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY
    20  
    21  	# Enable tidb extension to generate the commit ts.
    22  	SINK_URI="file://$WORK_DIR/storage_test?flush-interval=5s&enable-tidb-extension=true"
    23  	run_cdc_cli changefeed create --sink-uri="$SINK_URI" --config=$CUR/conf/changefeed.toml
    24  
    25  	run_sql_file $CUR/data/schema.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    26  	test_update $*
    27  
    28  	run_sql_file $CUR/data/data.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    29  	run_storage_consumer $WORK_DIR $SINK_URI $CUR/conf/changefeed.toml ""
    30  	sleep 8
    31  	check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml 100
    32  }
    33  
    34  function test_update() {
    35  	# test update
    36  	run_sql_file $CUR/data/update.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
    37  	sleep 30
    38  
    39  	table_dir="$WORK_DIR/storage_test/test/test_update"
    40  	ensure 60 ls "$table_dir" | grep -v "meta"
    41  
    42  	version_dir=$(ls $table_dir | grep -v "meta")
    43  	date_dir=$(ls $table_dir/$version_dir)
    44  	data_name=$(ls $table_dir/$version_dir/$date_dir | grep "csv")
    45  
    46  	file_path=$table_dir/$version_dir/$date_dir/$data_name
    47  
    48  	echo "start check update"
    49  	check_equal "update" "1" "$(cat $file_path | grep 'U' | wc -l)"
    50  	check_equal "insert" "7" "$(cat $file_path | grep 'I' | wc -l)"
    51  	check_equal "delete" "3" "$(cat $file_path | grep 'D' | wc -l)"
    52  }
    53  
    54  function check_equal() {
    55  	if [ "$2" != "$3" ]; then
    56  		echo "TEST: [$TEST_NAME] failed on $1, $2 expected, but $3 got"
    57  		exit 1
    58  	fi
    59  }
    60  
    61  trap stop_tidb_cluster EXIT
    62  run $*
    63  check_logs $WORK_DIR
    64  echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"