github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/ha_cases3/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $cur/../_utils/test_prepare
     7  WORK_DIR=$TEST_DIR/$TEST_NAME
     8  API_VERSION="v1alpha1"
     9  # import helper functions
    10  source $cur/../_utils/ha_cases_lib.sh
    11  
    12  function test_multi_task_running() {
    13  	echo "[$(date)] <<<<<< start test_multi_task_running >>>>>>"
    14  	cleanup
    15  	prepare_sql_multi_task
    16  	start_multi_tasks_cluster
    17  
    18  	# make sure task to step in "Sync" stage
    19  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \
    20  		"query-status test" \
    21  		"\"stage\": \"Running\"" 2 \
    22  		"\"unit\": \"Sync\"" 2
    23  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \
    24  		"query-status test2" \
    25  		"\"stage\": \"Running\"" 2 \
    26  		"\"unit\": \"Sync\"" 2
    27  
    28  	echo "use sync_diff_inspector to check full dump loader"
    29  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
    30  	check_sync_diff $WORK_DIR $cur/conf/diff_config_multi_task.toml
    31  
    32  	echo "flush logs to force rotate binlog file"
    33  	run_sql "flush logs;" $MYSQL_PORT1 $MYSQL_PASSWORD1
    34  	run_sql "flush logs;" $MYSQL_PORT2 $MYSQL_PASSWORD2
    35  
    36  	echo "apply increment data before restart dm-worker to ensure entering increment phase"
    37  	run_sql_file_withdb $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 $ha_test
    38  	run_sql_file_withdb $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 $ha_test
    39  	run_sql_file_withdb $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 $ha_test2
    40  	run_sql_file_withdb $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 $ha_test2
    41  
    42  	sleep 5 # wait for flush checkpoint
    43  	echo "use sync_diff_inspector to check increment data"
    44  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 50 || print_debug_status
    45  	check_sync_diff $WORK_DIR $cur/conf/diff_config_multi_task.toml 50 || print_debug_status
    46  	echo "[$(date)] <<<<<< finish test_multi_task_running >>>>>>"
    47  }
    48  
    49  function test_stop_task() {
    50  	echo "[$(date)] <<<<<< start test_stop_task >>>>>>"
    51  	test_multi_task_running
    52  
    53  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    54  		"start-relay -s $SOURCE_ID1 worker1" \
    55  		"\"result\": true" 2
    56  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    57  		"start-relay -s $SOURCE_ID2 worker2" \
    58  		"\"result\": true" 2
    59  
    60  	echo "start dumping SQLs into source"
    61  	load_data $MYSQL_PORT1 $MYSQL_PASSWORD1 "a" &
    62  	load_data $MYSQL_PORT2 $MYSQL_PASSWORD2 "b" &
    63  
    64  	task_name=(test test2)
    65  	task_config=(dm-task.yaml dm-task2.yaml)
    66  	for name in ${task_name[@]}; do
    67  		echo "stop tasks $name"
    68  		run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    69  			"stop-task $name" \
    70  			"\"result\": true" 3
    71  
    72  		run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    73  			"query-status $name" \
    74  			"\"result\": false" 1
    75  	done
    76  
    77  	sleep 1
    78  
    79  	run_dm_ctl_with_retry $WORK_DIR 127.0.0.1:$MASTER_PORT2 "list-member --worker" '"stage": "bound",' 2
    80  
    81  	for idx in $(seq 0 1); do
    82  		echo "start tasks $cur/conf/${task_config[$idx]}"
    83  		# if relay hasn't catch up with current sync pos(such as when workload too high), start-task may start with an error
    84  		# so we don't check "\"result\": true" here, and check it using query-status
    85  		run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    86  			"start-task $cur/conf/${task_config[$idx]}" \
    87  			"\"source\": \"$SOURCE_ID1\"" 1 \
    88  			"\"source\": \"$SOURCE_ID2\"" 1
    89  
    90  		run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    91  			"query-status ${task_name[$idx]}" \
    92  			"\"stage\": \"Running\"" 4
    93  	done
    94  
    95  	# waiting for syncing
    96  	wait
    97  	sleep 1
    98  
    99  	echo "use sync_diff_inspector to check increment data"
   100  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
   101  	check_sync_diff $WORK_DIR $cur/conf/diff_config_multi_task.toml
   102  	echo "[$(date)] <<<<<< finish test_stop_task >>>>>>"
   103  }
   104  
   105  function run() {
   106  	test_stop_task # TICASE-991, 984
   107  }
   108  
   109  cleanup_data $ha_test
   110  cleanup_data $ha_test2
   111  # also cleanup dm processes in case of last run failed
   112  cleanup_process $*
   113  run $*
   114  cleanup_process $*
   115  
   116  echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"