github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/dm_syncer/run.sh (about) 1 #!/bin/bash 2 3 set -eu 4 5 cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) 6 source $cur/../_utils/test_prepare 7 WORK_DIR=$TEST_DIR/$TEST_NAME 8 TASK_NAME="test" 9 10 function run() { 11 run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 12 check_contains 'Query OK, 2 rows affected' 13 run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 14 check_contains 'Query OK, 3 rows affected' 15 16 run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml 17 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 18 check_metric $MASTER_PORT 'start_leader_counter' 3 0 2 19 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 20 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 21 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 22 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 23 # operate mysql config to worker 24 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 25 cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml 26 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml 27 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml 28 dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 29 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 30 31 # start a task in `full` mode 32 cat $cur/conf/dm-task.yaml >$WORK_DIR/dm-task.yaml 33 dmctl_start_task $WORK_DIR/dm-task.yaml "--remove-meta" 34 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 35 36 dmctl_stop_task $TASK_NAME 37 38 run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 39 run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 40 41 # start a task in `incremental` mode 42 # using account with limited privileges 43 kill_dm_worker 44 run_sql_file $cur/data/db1.prepare.user.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 45 check_count 'Query OK, 0 rows affected' 7 46 run_sql_file $cur/data/db2.prepare.user.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 47 check_count 'Query OK, 0 rows affected' 7 48 49 cat $cur/conf/dm-syncer-1.toml >$WORK_DIR/dm-syncer-1.toml 50 cat $cur/conf/dm-syncer-2.toml >$WORK_DIR/dm-syncer-2.toml 51 cat $cur/conf/old_meta_file >$WORK_DIR/old_meta_file 52 # $worker1_run_source_1 > 0 means source1 is operated to worker1 53 worker1_run_source_1=$(sed "s/$SOURCE_ID1/$SOURCE_ID1\n/g" $WORK_DIR/worker1/log/dm-worker.log | grep -c "$SOURCE_ID1") || true 54 if [ $worker1_run_source_1 -gt 0 ]; then 55 name1=$(grep "Log: " $WORK_DIR/worker1/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') 56 pos1=$(grep "Pos: " $WORK_DIR/worker1/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') 57 name2=$(grep "Log: " $WORK_DIR/worker2/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') 58 pos2=$(grep "Pos: " $WORK_DIR/worker2/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') 59 else 60 name2=$(grep "Log: " $WORK_DIR/worker1/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') 61 pos2=$(grep "Pos: " $WORK_DIR/worker1/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') 62 name1=$(grep "Log: " $WORK_DIR/worker2/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') 63 pos1=$(grep "Pos: " $WORK_DIR/worker2/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2}' | tr -d ' ') 64 fi 65 sed -i "s/binlog-name-placeholder-1/\"$name1\"/g" $WORK_DIR/dm-syncer-1.toml 66 sed -i "s/binlog-pos-placeholder-1/$pos1/g" $WORK_DIR/dm-syncer-1.toml 67 sed -i "s/binlog-name-placeholder-2/\"$name2\"/g" $WORK_DIR/old_meta_file 68 sed -i "s/binlog-pos-placeholder-2/$pos2/g" $WORK_DIR/old_meta_file 69 run_dm_syncer $WORK_DIR/syncer1 $WORK_DIR/dm-syncer-1.toml 70 meta_file=$WORK_DIR/old_meta_file 71 run_dm_syncer $WORK_DIR/syncer2 $WORK_DIR/dm-syncer-2.toml $meta_file --syncer-config-format syncer2 72 73 # wait for dm_syncer to init and start 74 sleep 10 75 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 76 check_sync_diff $WORK_DIR $cur/conf/diff_config_blalist.toml 77 check_sync_diff $WORK_DIR $cur/conf/diff_config_route_rules.toml 78 # test block-allow-list by the way 79 run_sql "show databases;" $TIDB_PORT $TIDB_PASSWORD 80 check_not_contains "dm_syncer_ignore_db" 81 } 82 83 cleanup_data $TEST_NAME "dm_syncer_do_db" 84 # also cleanup dm processes in case of last run failed 85 cleanup_process $* 86 run $* 87 cleanup_process $* 88 89 echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"