github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/ha_cases_2/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $cur/../_utils/test_prepare
     7  WORK_DIR=$TEST_DIR/$TEST_NAME
     8  API_VERSION="v1alpha1"
     9  # import helper functions
    10  source $cur/../_utils/ha_cases_lib.sh
    11  
    12  function test_running() {
    13  	echo "[$(date)] <<<<<< start test_running >>>>>>"
    14  	cleanup
    15  	prepare_sql
    16  	start_cluster
    17  
    18  	# make sure task to step in "Sync" stage
    19  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \
    20  		"query-status test" \
    21  		"\"stage\": \"Running\"" 2 \
    22  		"\"unit\": \"Sync\"" 2
    23  
    24  	echo "use sync_diff_inspector to check full dump loader"
    25  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
    26  
    27  	echo "flush logs to force rotate binlog file"
    28  	run_sql "flush logs;" $MYSQL_PORT1 $MYSQL_PASSWORD1
    29  	run_sql "flush logs;" $MYSQL_PORT2 $MYSQL_PASSWORD2
    30  
    31  	echo "apply increment data before restart dm-worker to ensure entering increment phase"
    32  	run_sql_file_withdb $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 $ha_test
    33  	run_sql_file_withdb $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 $ha_test
    34  
    35  	sleep 3 # wait for flush checkpoint
    36  	echo "use sync_diff_inspector to check increment data"
    37  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
    38  	echo "[$(date)] <<<<<< finish test_running >>>>>>"
    39  }
    40  
    41  # usage: test_kill_master_in_sync leader
    42  # or: test_kill_master_in_sync follower (default)
    43  function test_kill_master_in_sync() {
    44  	echo "[$(date)] <<<<<< start test_kill_master_in_sync >>>>>>"
    45  	test_running
    46  
    47  	echo "start dumping SQLs into source"
    48  	load_data $MYSQL_PORT1 $MYSQL_PASSWORD1 "a" &
    49  	load_data $MYSQL_PORT2 $MYSQL_PASSWORD2 "b" &
    50  
    51  	kill_process dm-master1
    52  	check_master_port_offline 1
    53  
    54  	echo "wait and check task running"
    55  	sleep 1
    56  	check_http_alive 127.0.0.1:$MASTER_PORT2/apis/${API_VERSION}/status/test '"stage": "Running"' 10
    57  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT2" \
    58  		"query-status test" \
    59  		"\"stage\": \"Running\"" 2
    60  
    61  	# waiting for syncing
    62  	wait
    63  
    64  	echo "wait for dm to sync"
    65  	sleep 1
    66  	echo "use sync_diff_inspector to check data now!"
    67  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
    68  	echo "[$(date)] <<<<<< finish test_kill_master_in_sync >>>>>>"
    69  }
    70  
    71  function test_kill_worker_in_sync() {
    72  	echo "[$(date)] <<<<<< start test_kill_worker_in_sync >>>>>>"
    73  	test_running
    74  
    75  	echo "start dumping SQLs into source"
    76  	load_data $MYSQL_PORT1 $MYSQL_PASSWORD1 "a" &
    77  	load_data $MYSQL_PORT2 $MYSQL_PASSWORD2 "b" &
    78  
    79  	echo "kill dm-worker1"
    80  	kill_process dm-worker1
    81  	echo "start worker3"
    82  	run_dm_worker $WORK_DIR/worker3 $WORKER3_PORT $cur/conf/dm-worker3.toml
    83  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER3_PORT
    84  
    85  	# start-relay halfway
    86  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    87  		"start-relay -s $SOURCE_ID1 worker3" \
    88  		"\"result\": true" 2
    89  
    90  	echo "kill dm-worker2"
    91  	kill_process dm-worker2
    92  	echo "start worker4"
    93  	run_dm_worker $WORK_DIR/worker4 $WORKER4_PORT $cur/conf/dm-worker4.toml
    94  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER4_PORT
    95  
    96  	echo "wait and check task running"
    97  	check_http_alive 127.0.0.1:$MASTER_PORT/apis/${API_VERSION}/status/test '"stage": "Running"' 10
    98  
    99  	echo "query-status from all dm-master"
   100  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT1" \
   101  		"query-status test" \
   102  		"\"stage\": \"Running\"" 3
   103  
   104  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT2" \
   105  		"query-status test" \
   106  		"\"stage\": \"Running\"" 3
   107  
   108  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT3" \
   109  		"query-status test" \
   110  		"\"stage\": \"Running\"" 3
   111  
   112  	# waiting for syncing
   113  	wait
   114  
   115  	echo "wait for dm to sync"
   116  	sleep 1
   117  
   118  	echo "use sync_diff_inspector to check data now!"
   119  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
   120  	echo "[$(date)] <<<<<< finish test_kill_worker_in_sync >>>>>>"
   121  }
   122  
   123  function run() {
   124  	test_kill_master_in_sync
   125  	test_kill_worker_in_sync
   126  }
   127  
   128  cleanup_data $ha_test
   129  cleanup_data $ha_test2
   130  # also cleanup dm processes in case of last run failed
   131  cleanup_process $*
   132  run $*
   133  cleanup_process $*
   134  
   135  echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"