github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/sequence_safe_mode/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $cur/../_utils/test_prepare
     7  WORK_DIR=$TEST_DIR/$TEST_NAME
     8  
     9  function run() {
    10  	run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    11  	check_contains 'Query OK, 2 rows affected'
    12  	run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    13  	check_contains 'Query OK, 3 rows affected'
    14  
    15  	export GO_FAILPOINTS='github.com/pingcap/tiflow/dm/syncer/ReSyncExit=return(true)'
    16  	run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml
    17  	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
    18  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    19  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    20  	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
    21  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
    22  	# operate mysql config to worker
    23  	cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml
    24  	cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml
    25  	sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml
    26  	sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml
    27  	dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1
    28  	dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2
    29  
    30  	run_sql_tidb "create database if not exists sequence_safe_mode_target;"
    31  	run_sql_tidb "create table sequence_safe_mode_target.t_target (id bigint auto_increment, uid int, name varchar(80), c_table varchar(255), c_source varchar(255), primary key (id, c_table, c_source), unique key(uid)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
    32  	dmctl_start_task "$cur/conf/dm-task.yaml" "--remove-meta"
    33  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
    34  
    35  	# DM-worker exit during re-sync after sharding group synced
    36  	run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    37  	run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    38  
    39  	check_port_offline $WORKER1_PORT 20
    40  	check_port_offline $WORKER2_PORT 20
    41  
    42  	export GO_FAILPOINTS=''
    43  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    44  	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
    45  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    46  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
    47  
    48  	sleep 2
    49  	echo "check sync diff after reset failpoint"
    50  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
    51  
    52  	pkill -hup dm-worker.test 2>/dev/null || true
    53  	wait_process_exit dm-worker.test
    54  
    55  	export GO_FAILPOINTS="github.com/pingcap/tiflow/dm/syncer/SequenceShardSyncedExecutionExit=return(true);github.com/pingcap/tiflow/dm/syncer/SafeModeInitPhaseSeconds=return(\"300s\")"
    56  
    57  	echo "restart DM-worker after set SequenceShardSyncedExecutionExit failpoint"
    58  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    59  	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
    60  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    61  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
    62  
    63  	sleep 2
    64  
    65  	# DM-worker exit when waiting for sharding group synced
    66  	run_sql_file $cur/data/db1.increment2.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    67  	run_sql_file $cur/data/db2.increment2.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    68  
    69  	OWNER_PORT=""
    70  	i=0
    71  	while [ $i -lt 10 ]; do
    72  		# we can't determine which DM-worker is the sharding lock owner, so we try both of them
    73  		# DM-worker1 is sharding lock owner and exits
    74  		if [ "$(check_port_return $WORKER1_PORT)" == "0" ]; then
    75  			echo "DM-worker1 is sharding lock owner and detects it offline"
    76  			export GO_FAILPOINTS="github.com/pingcap/tiflow/dm/syncer/SafeModeInitPhaseSeconds=return(\"0s\")"
    77  			run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    78  			check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    79  			check_instance_id="1"
    80  			OWNER_PORT=$WORKER1_PORT
    81  			break
    82  		fi
    83  		# DM-worker2 is sharding lock owner and exits
    84  		if [ "$(check_port_return $WORKER2_PORT)" == "0" ]; then
    85  			echo "DM-worker2 is sharding lock owner and detects it offline"
    86  			export GO_FAILPOINTS="github.com/pingcap/tiflow/dm/syncer/SafeModeInitPhaseSeconds=return(\"0s\")"
    87  			run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
    88  			check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
    89  			check_instance_id="2"
    90  			OWNER_PORT=$WORKER2_PORT
    91  			break
    92  		fi
    93  
    94  		((i += 1))
    95  		echo "wait for one of DM-worker offine failed, retry later" && sleep 1
    96  	done
    97  	if [ $i -ge 10 ]; then
    98  		echo "wait DM-worker offline timeout"
    99  		exit 1
   100  	fi
   101  
   102  	sleep 5
   103  	echo "check sync diff after restart DDL owner"
   104  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
   105  }
   106  
   107  cleanup_data sequence_safe_mode_target
   108  # also cleanup dm processes in case of last run failed
   109  cleanup_process $*
   110  run $*
   111  cleanup_process $*
   112  
   113  echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"