github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/import_v10x/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $cur/../_utils/test_prepare
     7  WORK_DIR=$TEST_DIR/$TEST_NAME
     8  
     9  function run() {
    10  	# load some data both into upstream and downstream.
    11  	run_sql_both_source "drop database if exists import_v10x"
    12  	run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    13  	check_contains 'Query OK, 2 rows affected'
    14  	run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    15  	check_contains 'Query OK, 3 rows affected'
    16  
    17  	run_sql_tidb "drop database if exists import_v10x"
    18  	run_sql_file $cur/data/db1.prepare.sql 127.0.0.1 $TIDB_PORT $TIDB_PASSWORD
    19  	check_contains 'Query OK, 2 rows affected'
    20  	run_sql_file $cur/data/db2.prepare.sql 127.0.0.1 $TIDB_PORT $TIDB_PASSWORD
    21  	check_contains 'Query OK, 3 rows affected'
    22  
    23  	# NOTE: to run a real v1.0.x cluster is a little bit complicated in CI.
    24  	# generate checkpoint data for v1.0.x.
    25  	run_sql_source1 "SHOW MASTER STATUS"
    26  	binlog_name1=$(grep "File" "$TEST_DIR/sql_res.$TEST_NAME.txt" | awk -F: '{print $2}' | tr -d ' ')
    27  	binlog_pos1=$(grep "Position" "$TEST_DIR/sql_res.$TEST_NAME.txt" | awk -F: '{print $2}' | tr -d ' ')
    28  	binlog_gtid1=$(grep "Executed_Gtid_Set" "$TEST_DIR/sql_res.$TEST_NAME.txt" | awk -F ': ' '{print $2}' | tr -d ' ')
    29  	echo "source1 master status", $binlog_name1, $binlog_pos1, $binlog_gtid1
    30  
    31  	run_sql_source2 "SHOW MASTER STATUS"
    32  	binlog_name2=$(grep "File" "$TEST_DIR/sql_res.$TEST_NAME.txt" | awk -F: '{print $2}' | tr -d ' ')
    33  	binlog_pos2=$(grep "Position" "$TEST_DIR/sql_res.$TEST_NAME.txt" | awk -F: '{print $2}' | tr -d ' ')
    34  	binlog_gtid2=$(grep "Executed_Gtid_Set" "$TEST_DIR/sql_res.$TEST_NAME.txt" | awk -F ': ' '{print $2}' | tr -d ' ')
    35  	echo "source2 master status", $binlog_name2, $binlog_pos2, $binlog_gtid2
    36  
    37  	cp $cur/data/v106_syncer_checkpoint.sql $WORK_DIR/v106_syncer_checkpoint.sql
    38  	sed -i "s/BINLOG_NAME1/$binlog_name1/g" $WORK_DIR/v106_syncer_checkpoint.sql
    39  	sed -i "s/BINLOG_POS1/$binlog_pos1/g" $WORK_DIR/v106_syncer_checkpoint.sql
    40  	sed -i "s/BINLOG_NAME2/$binlog_name2/g" $WORK_DIR/v106_syncer_checkpoint.sql
    41  	sed -i "s/BINLOG_POS2/$binlog_pos2/g" $WORK_DIR/v106_syncer_checkpoint.sql
    42  
    43  	run_sql_file $WORK_DIR/v106_syncer_checkpoint.sql 127.0.0.1 $TIDB_PORT $TIDB_PASSWORD
    44  
    45  	# generate/copy dm_worker_meta for v1.0.x.
    46  	mkdir -p $WORK_DIR/worker1
    47  	mkdir -p $WORK_DIR/worker2
    48  	cp -r $cur/dm_worker1_meta $WORK_DIR/worker1/dm_worker_meta
    49  	cp -r $cur/dm_worker2_meta $WORK_DIR/worker2/dm_worker_meta
    50  
    51  	# put source config files into `v1-sources-path`.
    52  	mkdir -p $WORK_DIR/master/v1-sources
    53  	cp $cur/conf/source1.yaml $WORK_DIR/master/v1-sources/source1.yaml
    54  	cp $cur/conf/source2.yaml $WORK_DIR/master/v1-sources/source2.yaml
    55  
    56  	# prepare key file for dm-master.
    57  	mkdir -p $WORK_DIR/master
    58  	cp $cur/conf/key.txt $WORK_DIR/master/
    59  
    60  	# start DM worker and master
    61  	run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml
    62  	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
    63  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    64  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    65  	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
    66  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
    67  
    68  	# check source created.
    69  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    70  		"operate-source show" \
    71  		"\"result\": true" 3 \
    72  		'"worker": "worker1"' 1 \
    73  		'"worker": "worker2"' 1
    74  
    75  	# check task running.
    76  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    77  		"query-status test" \
    78  		"\"stage\": \"Running\"" 3
    79  
    80  	# check task config, just a simple match
    81  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    82  		"config task test --path $WORK_DIR/task.yaml" \
    83  		"\"result\": true" 1
    84  
    85  	sed -i "s/password: '\*\*\*\*\*\*'/password: \"\"/g" $WORK_DIR/task.yaml
    86  	diff $cur/conf/task.yaml $WORK_DIR/task.yaml || exit 1
    87  
    88  	run_sql "show create table \`dm_meta\`.\`test_syncer_checkpoint\`" $TIDB_PORT $TIDB_PASSWORD
    89  	check_contains "\`exit_safe_binlog_name\` varchar(128) DEFAULT ''"
    90  	# different TiDB will output DEFAULT 0 or DEFAULT '0'
    91  	check_contains "\`exit_safe_binlog_pos\` int(10) unsigned DEFAULT "
    92  	check_contains "\`exit_safe_binlog_gtid\` text DEFAULT NULL"
    93  
    94  	run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    95  	run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    96  
    97  	# use sync_diff_inspector to check data now!
    98  	sleep 2
    99  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
   100  }
   101  
   102  cleanup_data import_v10x
   103  # also cleanup dm processes in case of last run failed
   104  cleanup_process $*
   105  run $*
   106  cleanup_process $*
   107  
   108  echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"