github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/gtid/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $cur/../_utils/test_prepare
     7  WORK_DIR=$TEST_DIR/$TEST_NAME
     8  API_VERSION="v1alpha1"
     9  TASK_NAME="test"
    10  
    11  function advance_gtid_test() {
    12  	export GO_FAILPOINTS="github.com/pingcap/tiflow/dm/syncer/NotUpdateLatestGTID=return()"
    13  	run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    14  	run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    15  	cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml
    16  	cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml
    17  	sed -i "/from:/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml
    18  	sed -i "/from:/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml
    19  
    20  	# start DM worker and source one-by-one, make sure the source1 bound to worker1
    21  	run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml
    22  	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
    23  	check_metric $MASTER_PORT 'start_leader_counter' 3 0 2
    24  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    25  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    26  	dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1
    27  
    28  	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
    29  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
    30  	dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2
    31  
    32  	dmctl_start_task "$cur/conf/dm-task.yaml" "--remove-meta"
    33  	check_metric $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"test\",worker=\"worker1\"}" 10 1 3
    34  	check_metric $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"test\",worker=\"worker2\"}" 10 1 3
    35  
    36  	run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    37  	run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    38  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
    39  
    40  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    41  		"pause-task test" \
    42  		"\"result\": true" 3
    43  
    44  	run_sql_file $cur/data/db1.increment2.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    45  	run_sql_file $cur/data/db2.increment2.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    46  
    47  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    48  		"resume-task test" \
    49  		"\"result\": true" 3
    50  	check_metric $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"test\",worker=\"worker1\"}" 10 1 3
    51  	check_metric $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"test\",worker=\"worker2\"}" 10 1 3
    52  
    53  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
    54  
    55  	export GO_FAILPOINTS=""
    56  	cleanup_process $*
    57  	cleanup_data gtid
    58  }
    59  
    60  function run() {
    61  	advance_gtid_test
    62  
    63  	run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    64  	run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    65  
    66  	cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml
    67  	cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml
    68  	sed -i "/from:/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml
    69  	sed -i "/from:/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml
    70  
    71  	# start DM worker and source one-by-one, make sure the source1 bound to worker1
    72  	run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml
    73  	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
    74  	check_metric $MASTER_PORT 'start_leader_counter' 3 0 2
    75  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    76  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    77  	dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1
    78  
    79  	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
    80  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
    81  	dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2
    82  
    83  	dmctl_start_task "$cur/conf/dm-task.yaml" "--remove-meta"
    84  	check_metric $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"test\",worker=\"worker1\"}" 10 1 3
    85  	check_metric $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"test\",worker=\"worker2\"}" 10 1 3
    86  
    87  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
    88  
    89  	# TODO: when there's a purged gap, if starting gtid set covers gap, there should be no data lost
    90  	# if starting gtid set not fully covers gap, behaviour should be same whether we enable relay
    91  	# hard to reproduce in CI
    92  
    93  	# when there's a not purged gap, there should be no data lost
    94  	# we manually `set gtid_next = 'uuid:gtid'`to reproduce
    95  	gtid1=$(grep "GTID:" $WORK_DIR/worker1/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2,":",$3}' | tr -d ' ')
    96  	gtid2=$(grep "GTID:" $WORK_DIR/worker2/dumped_data.$TASK_NAME/metadata | awk -F: '{print $2,":",$3}' | tr -d ' ')
    97  	uuid1=$(echo $gtid1 | awk -F: '{print $1}')
    98  	uuid2=$(echo $gtid2 | awk -F: '{print $1}')
    99  	end_gtid_num1=$(echo $gtid1 | awk -F: '{print $2}' | awk -F- '{print $2}')
   100  	end_gtid_num2=$(echo $gtid2 | awk -F: '{print $2}' | awk -F- '{print $2}')
   101  	new_gtid1=${uuid1}:$((end_gtid_num1 + 3))
   102  	new_gtid2=${uuid2}:$((end_gtid_num2 + 3))
   103  	echo "new_gtid1 $new_gtid1 new_gtid2 $new_gtid2"
   104  
   105  	run_sql_source1 "SET gtid_next='$new_gtid1';insert into gtid.t1 values (3);SET gtid_next='AUTOMATIC';"
   106  	run_sql_source2 "SET gtid_next='$new_gtid2';insert into gtid.t2 values (3);SET gtid_next='AUTOMATIC'"
   107  	run_sql_both_source "flush logs"
   108  	run_sql_source1 "insert into gtid.t1 values (4)"
   109  	run_sql_source2 "insert into gtid.t2 values (4)"
   110  	# now Previous_gtids event is 09bec856-ba95-11ea-850a-58f2b4af5188:1-4:6
   111  
   112  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
   113  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
   114  		"stop-task test" \
   115  		"\"result\": true" 3
   116  
   117  	run_sql_source1 "insert into gtid.t1 values (5)"
   118  	run_sql_source2 "insert into gtid.t2 values (5)"
   119  	# now Previous_gtids event is 09bec856-ba95-11ea-850a-58f2b4af5188:1-6
   120  
   121  	# remove relay-dir, now relay starting point(syncer checkpoint) should be 09bec856-ba95-11ea-850a-58f2b4af5188:1-4:6
   122  	# check if relay correctly handle gap
   123  	pkill -hup dm-worker.test 2>/dev/null || true
   124  	check_port_offline $WORKER1_PORT 20
   125  	check_port_offline $WORKER2_PORT 20
   126  	rm -rf $WORK_DIR/worker1/relay_log || true
   127  	rm -rf $WORK_DIR/worker2/relay_log || true
   128  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
   129  	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
   130  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
   131  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
   132  
   133  	# we didn't lost 09bec856-ba95-11ea-850a-58f2b4af5188:5, which is insert into gtid.tx values (5)
   134  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
   135  		"start-task $cur/conf/dm-task.yaml" \
   136  		"\"result\": true" 3
   137  	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
   138  
   139  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
   140  		"stop-task test" \
   141  		"\"result\": true" 3
   142  
   143  	run_sql_source1 "insert into gtid.t1 values (7)"
   144  	run_sql_source2 "insert into gtid.t2 values (7)"
   145  	run_sql_both_source "flush logs"
   146  	run_sql_source1 "insert into gtid.t1 values (8)"
   147  	run_sql_source2 "insert into gtid.t2 values (8)"
   148  	sleep 2
   149  	run_sql_both_source "purge binary logs before '$(date '+%Y-%m-%d %H:%M:%S')'"
   150  
   151  	# remove relay-dir, now relay starting point(syncer checkpoint) should be 09bec856-ba95-11ea-850a-58f2b4af5188:1-6
   152  	# which is already purged
   153  	pkill -hup dm-worker.test 2>/dev/null || true
   154  	check_port_offline $WORKER1_PORT 20
   155  	check_port_offline $WORKER2_PORT 20
   156  	rm -rf $WORK_DIR/worker1/relay_log || true
   157  	rm -rf $WORK_DIR/worker2/relay_log || true
   158  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
   159  	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
   160  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
   161  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
   162  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
   163  		"start-task $cur/conf/dm-task.yaml"
   164  	# both with and without relay should error
   165  	# (different version of MySQL has different error message, only compare error code here)
   166  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
   167  		"query-status $TASK_NAME" \
   168  		"no relay pos match gtid" 1 \
   169  		"ERROR 1236 (HY000)" 1
   170  	check_metric $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"test\",worker=\"worker1\"}" 3 2 4
   171  	check_metric $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"test\",worker=\"worker2\"}" 3 2 4
   172  }
   173  
   174  cleanup_data gtid
   175  # also cleanup dm processes in case of last run failed
   176  cleanup_process $*
   177  run $*
   178  cleanup_process $*
   179  
   180  echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"