github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/tests/integration_tests/storage_cleanup/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eux
     4  
     5  CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $CUR/../_utils/test_prepare
     7  WORK_DIR=$OUT_DIR/$TEST_NAME
     8  CDC_BINARY=cdc.test
     9  SINK_TYPE=$1
    10  
    11  EXIST_FILES=()
    12  CLEANED_FILES=()
    13  function generate_single_table_files() {
    14  	local workdir=$1
    15  	local bucket=$2
    16  	local schema=$3
    17  	local table=$4
    18  	local day=$5
    19  	local file_cnt=$6
    20  	local should_clean=$7 # true or false
    21  
    22  	table_dir=$workdir/$bucket/$schema/$table/$day
    23  	mkdir -p $table_dir
    24  	for i in $(seq 1 $file_cnt); do
    25  		file=$table_dir/$i.data
    26  		touch $file
    27  		if [ "$should_clean" == "true" ]; then
    28  			CLEANED_FILES+=($file)
    29  		else
    30  			EXIST_FILES+=($file)
    31  		fi
    32  	done
    33  
    34  	mkdir -p $table_dir/meta
    35  	touch $table_dir/meta/CDC.index
    36  }
    37  
    38  function generate_historic_files() {
    39  	local target_bucket="storage_test"
    40  	yesterday=$(date -d "yesterday" +"%Y-%m-%d")             # should not be cleaned since file-expiration-days is 1
    41  	day_before_yesterday=$(date -d "2 days ago" +"%Y-%m-%d") # should be cleaned
    42  
    43  	# historic files of table in schema.sql
    44  	generate_single_table_files $WORK_DIR $target_bucket test multi_data_type $yesterday 10 false
    45  	generate_single_table_files $WORK_DIR $target_bucket test multi_charset $day_before_yesterday 10 true
    46  	generate_single_table_files $WORK_DIR $target_bucket test binary_columns $day_before_yesterday 10 true
    47  
    48  	# historic files of tables in test but not in schema.sql
    49  	generate_single_table_files $WORK_DIR $target_bucket test multi_data_type_dummy $day_before_yesterday 10 true
    50  	generate_single_table_files $WORK_DIR $target_bucket test multi_charset_dummy $day_before_yesterday 10 true
    51  	generate_single_table_files $WORK_DIR $target_bucket test binary_columns_dummy $yesterday 10 false
    52  
    53  	# historic files of table belongs to different schema
    54  	generate_single_table_files $WORK_DIR $target_bucket test2 multi_data_type $day_before_yesterday 10 true
    55  	generate_single_table_files $WORK_DIR $target_bucket test2 multi_charset $day_before_yesterday 10 true
    56  	generate_single_table_files $WORK_DIR $target_bucket test2 binary_columns $yesterday 10 false
    57  
    58  	# historic files in default bucket, which should not be cleaned
    59  	generate_single_table_files $WORK_DIR storage_test_default test multi_data_type 2022-01-01 10 false
    60  	generate_single_table_files $WORK_DIR storage_test_default test multi_charset 2022-01-02 10 false
    61  	generate_single_table_files $WORK_DIR storage_test_default test binary_columns 2022-01-03 10 false
    62  }
    63  
    64  function check_file_exists() {
    65  	local all_should_exist=$1
    66  	for f in ${EXIST_FILES[@]}; do
    67  		if [ ! -f $f ]; then
    68  			echo "file $f should exist but not"
    69  			exit 1
    70  		fi
    71  	done
    72  
    73  	for f in ${CLEANED_FILES[@]}; do
    74  		if [ "$all_should_exist" == "true" ]; then
    75  			if [ ! -f $f ]; then
    76  				echo "file $f should exist but not"
    77  				exit 1
    78  			fi
    79  		else
    80  			if [ -f $f ]; then
    81  				echo "file $f should not exist but exists"
    82  				exit 1
    83  			fi
    84  		fi
    85  	done
    86  }
    87  
    88  function run() {
    89  	if [ "$SINK_TYPE" != "storage" ]; then
    90  		return
    91  	fi
    92  
    93  	rm -rf $WORK_DIR && mkdir -p $WORK_DIR
    94  	start_tidb_cluster --workdir $WORK_DIR
    95  	cd $WORK_DIR
    96  	run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY
    97  
    98  	generate_historic_files
    99  
   100  	SINK_URI_DEFAULT="file://$WORK_DIR/storage_test_default?flush-interval=5s"
   101  	run_cdc_cli changefeed create --sink-uri="$SINK_URI_DEFAULT" -c "default-config-test" --config=$CUR/conf/changefeed-default.toml
   102  	sleep 20
   103  	check_file_exists true
   104  
   105  	SINK_URI="file://$WORK_DIR/storage_test?flush-interval=5s"
   106  	run_cdc_cli changefeed create --sink-uri="$SINK_URI" --config=$CUR/conf/changefeed.toml
   107  	sleep 20
   108  	check_file_exists false
   109  
   110  	run_sql_file $CUR/data/schema.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
   111  	run_sql_file $CUR/data/data.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT}
   112  	run_storage_consumer $WORK_DIR $SINK_URI $CUR/conf/changefeed.toml ""
   113  	sleep 8
   114  	check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml 100
   115  }
   116  
   117  trap stop_tidb_cluster EXIT
   118  run $*
   119  check_logs $WORK_DIR
   120  echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"