github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/validator_checkpoint/run.sh (about) 1 #!/bin/bash 2 3 set -eu 4 5 cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) 6 source $cur/../_utils/test_prepare 7 source $cur/../_utils/shardddl_lib.sh 8 WORK_DIR=$TEST_DIR/$TEST_NAME 9 10 db_name=$TEST_NAME 11 12 function prepare_for_test() { 13 cleanup_process $* 14 cleanup_data $db_name 15 cleanup_data_upstream $db_name 16 17 run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 18 run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 19 run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml 20 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 21 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 22 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 23 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 24 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 25 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 26 dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 27 cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml 28 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 29 } 30 31 function trigger_validator_flush() { 32 sleep 0.2 33 run_sql_source1 "alter table $db_name.t1_1 comment 'a';" # force flush checkpoint 34 run_sql_source2 "alter table $db_name.t2_1 comment 'a';" # force flush checkpoint 35 } 36 37 function run() { 38 # 39 # persis success for 2 source, and data of different source didn't interrupt with each other 40 # 41 echo "--> check persist checkpoint and data with 2 source" 42 export GO_FAILPOINTS='github.com/pingcap/tiflow/dm/syncer/SkipDML=return(5)' 43 prepare_for_test 44 dmctl_start_task $cur/conf/dm-task.yaml --remove-meta 45 # wait until task is in sync unit 46 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 47 "\"unit\": \"Sync\"" 2 48 run_sql_source1 "insert into $db_name.t1_1 values(1, 1)" 49 run_sql_source1 "insert into $db_name.t1_1 values(2, 2)" 50 run_sql_source1 "insert into $db_name.t1_1 values(6, 6)" 51 run_sql_source1 "update $db_name.t1_1 set value=22 where id=2" 52 run_sql_source1 "insert into $db_name.t1_2 values(6)" 53 run_sql_source1 "insert into $db_name.t1_3 values(6)" 54 run_sql_source2 "insert into $db_name.t2_1 values(1)" 55 run_sql_source2 "insert into $db_name.t2_2 values(6)" 56 trigger_validator_flush 57 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 58 "validation status test" \ 59 "processedRowsStatus\": \"insert\/update\/delete: 3\/1\/0\"" 1 \ 60 "processedRowsStatus\": \"insert\/update\/delete: 1\/0\/0\"" 1 \ 61 "pendingRowsStatus\": \"insert\/update\/delete: 1\/1\/0" 1 \ 62 "pendingRowsStatus\": \"insert\/update\/delete: 1\/0\/0" 1 \ 63 "new\/ignored\/resolved: 0\/0\/0" 2 64 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_pending_change where source='mysql-replica-01'" \ 65 "count(*): 2" 66 run_sql_tidb_with_retry "select count(*) 67 from dm_meta.test_validator_checkpoint a, dm_meta.test_validator_pending_change b 68 where a.source=b.source 69 and a.source='mysql-replica-01' 70 and a.revision=b.revision" \ 71 "count(*): 2" 72 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_pending_change where source='mysql-replica-02'" \ 73 "count(*): 1" 74 run_sql_tidb_with_retry "select count(*) 75 from dm_meta.test_validator_checkpoint a, dm_meta.test_validator_pending_change b 76 where a.source=b.source 77 and a.source='mysql-replica-02' 78 and a.revision=b.revision" \ 79 "count(*): 1" 80 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-01'" \ 81 "count(*): 3" 82 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-02'" \ 83 "count(*): 2" 84 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_error_change" \ 85 "count(*): 0" 86 # validator don't support relay right now, so we don't check binlog_name 87 run_sql_tidb_with_retry "select count(*) 88 from dm_meta.test_syncer_checkpoint a, dm_meta.test_validator_checkpoint b 89 where a.cp_schema='' 90 and a.id=b.source 91 and a.binlog_pos=b.binlog_pos 92 and a.binlog_gtid=b.binlog_gtid;" \ 93 "count(*): 2" 94 95 echo "--> check validator can restart from previous position on fail over" 96 restart_worker1 97 restart_worker2 98 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 99 "validation status test" \ 100 "processedRowsStatus\": \"insert\/update\/delete: 3\/1\/0\"" 1 \ 101 "processedRowsStatus\": \"insert\/update\/delete: 1\/0\/0\"" 1 \ 102 "pendingRowsStatus\": \"insert\/update\/delete: 1\/1\/0" 1 \ 103 "pendingRowsStatus\": \"insert\/update\/delete: 1\/0\/0" 1 \ 104 "new\/ignored\/resolved: 0\/0\/0" 2 105 # insert missed data manually 106 run_sql_tidb "insert into $db_name.t1_1 values(1, 1)" 107 run_sql_tidb "insert into $db_name.t1_1 values(2, 22)" 108 run_sql_tidb "insert into $db_name.t2_1 values(1)" 109 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 110 "validation status test" \ 111 "processedRowsStatus\": \"insert\/update\/delete: 3\/1\/0\"" 1 \ 112 "processedRowsStatus\": \"insert\/update\/delete: 1\/0\/0\"" 1 \ 113 "pendingRowsStatus\": \"insert\/update\/delete: 0\/0\/0" 2 \ 114 "new\/ignored\/resolved: 0\/0\/0" 2 115 trigger_validator_flush 116 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_pending_change" \ 117 "count(*): 0" 118 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-01'" \ 119 "count(*): 3" 120 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-02'" \ 121 "count(*): 2" 122 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_error_change" \ 123 "count(*): 0" 124 run_sql_tidb_with_retry "select count(*) 125 from dm_meta.test_syncer_checkpoint a, dm_meta.test_validator_checkpoint b 126 where a.cp_schema='' 127 and a.id=b.source 128 and a.binlog_pos=b.binlog_pos 129 and a.binlog_gtid=b.binlog_gtid;" \ 130 "count(*): 2" 131 132 # 133 # validator persist fail in the middle 134 # 135 echo "--> check validator persist fail in the middle" 136 export GO_FAILPOINTS='github.com/pingcap/tiflow/dm/syncer/SkipDML=return(10)' 137 export GO_FAILPOINTS="$GO_FAILPOINTS;github.com/pingcap/tiflow/dm/syncer/ValidatorFailOnPersist=return()" 138 prepare_for_test 139 dmctl_start_task $cur/conf/dm-task.yaml --remove-meta 140 # wait until task is in sync unit 141 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 142 "\"unit\": \"Sync\"" 2 143 for id in $(seq 4); do 144 run_sql_source1 "insert into $db_name.t1_1 values($id, $id)" 145 run_sql_source2 "insert into $db_name.t2_1 values($id)" 146 done 147 # trigger a success persist with 4 pending rows 148 trigger_validator_flush 149 run_sql_source1 "insert into $db_name.t1_1 values(5, 5)" 150 run_sql_source2 "insert into $db_name.t2_1 values(5)" 151 # this persist fails 152 trigger_validator_flush 153 # validator stopped because of error, stage of table is running 154 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 155 "validation status test" \ 156 "\"stage\": \"Stopped\"" 2 \ 157 "\"stage\": \"Running\"" 2 \ 158 "processedRowsStatus\": \"insert\/update\/delete: 5\/0\/0\"" 2 \ 159 "pendingRowsStatus\": \"insert\/update\/delete: 5\/0\/0" 2 \ 160 "new\/ignored\/resolved: 0\/0\/0" 2 161 # processed ins/upd/del should be 4/0/0 in both source 162 run_sql_tidb_with_retry "select concat_ws('/', procd_ins, procd_upd, procd_del) processed 163 from dm_meta.test_validator_checkpoint where source='mysql-replica-01'" \ 164 "processed: 4/0/0" 165 run_sql_tidb_with_retry "select concat_ws('/', procd_ins, procd_upd, procd_del) processed 166 from dm_meta.test_validator_checkpoint where source='mysql-replica-02'" \ 167 "processed: 4/0/0" 168 # 4 of valid persist, 2 of incomplete persist in source 01 169 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_pending_change where source='mysql-replica-01'" \ 170 "count(*): 6" 171 run_sql_tidb_with_retry "select count(*) 172 from dm_meta.test_validator_checkpoint a, dm_meta.test_validator_pending_change b 173 where a.source=b.source 174 and a.source='mysql-replica-01' 175 and a.revision=b.revision" \ 176 "count(*): 4" 177 # 4 of valid persist, 3 of incomplete persist in source 02 178 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_pending_change where source='mysql-replica-02'" \ 179 "count(*): 7" 180 run_sql_tidb_with_retry "select count(*) 181 from dm_meta.test_validator_checkpoint a, dm_meta.test_validator_pending_change b 182 where a.source=b.source 183 and a.source='mysql-replica-02' 184 and a.revision=b.revision" \ 185 "count(*): 4" 186 # 1 table for each source, 0 error 187 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-01'" \ 188 "count(*): 1" 189 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-02'" \ 190 "count(*): 1" 191 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_error_change" \ 192 "count(*): 0" 193 # validator should falls behind syncer 194 run_sql_tidb_with_retry "select count(*) 195 from dm_meta.test_syncer_checkpoint a, dm_meta.test_validator_checkpoint b 196 where a.cp_schema='' 197 and a.id=b.source 198 and a.binlog_pos!=b.binlog_pos" \ 199 "count(*): 2" 200 201 echo "--> after resume and persist again, meta data should be consistent" 202 dmctl_stop_task $cur/conf/dm-task.yaml 203 dmctl_start_task $cur/conf/dm-task.yaml 204 trigger_validator_flush 205 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 206 "validation status test" \ 207 "\"stage\": \"Running\"" 4 \ 208 "processedRowsStatus\": \"insert\/update\/delete: 5\/0\/0\"" 2 \ 209 "pendingRowsStatus\": \"insert\/update\/delete: 5\/0\/0" 2 \ 210 "new\/ignored\/resolved: 0\/0\/0" 2 211 # processed ins/upd/del should be 5/0/0 in both source 212 run_sql_tidb_with_retry "select concat_ws('/', procd_ins, procd_upd, procd_del) processed 213 from dm_meta.test_validator_checkpoint where source='mysql-replica-01'" \ 214 "processed: 5/0/0" 215 run_sql_tidb_with_retry "select concat_ws('/', procd_ins, procd_upd, procd_del) processed 216 from dm_meta.test_validator_checkpoint where source='mysql-replica-02'" \ 217 "processed: 5/0/0" 218 # 5 pending row for source 01 219 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_pending_change where source='mysql-replica-01'" \ 220 "count(*): 5" 221 run_sql_tidb_with_retry "select count(*) 222 from dm_meta.test_validator_checkpoint a, dm_meta.test_validator_pending_change b 223 where a.source=b.source 224 and a.source='mysql-replica-01' 225 and a.revision=b.revision" \ 226 "count(*): 5" 227 # 5 pending row for source 02 228 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_pending_change where source='mysql-replica-02'" \ 229 "count(*): 5" 230 run_sql_tidb_with_retry "select count(*) 231 from dm_meta.test_validator_checkpoint a, dm_meta.test_validator_pending_change b 232 where a.source=b.source 233 and a.source='mysql-replica-02' 234 and a.revision=b.revision" \ 235 "count(*): 5" 236 # 1 table for each source, 0 error 237 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-01'" \ 238 "count(*): 1" 239 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-02'" \ 240 "count(*): 1" 241 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_error_change" \ 242 "count(*): 0" 243 # validator should reach syncer 244 run_sql_tidb_with_retry "select count(*) 245 from dm_meta.test_syncer_checkpoint a, dm_meta.test_validator_checkpoint b 246 where a.cp_schema='' 247 and a.id=b.source 248 and a.binlog_pos=b.binlog_pos 249 and a.binlog_gtid=b.binlog_gtid;" \ 250 "count(*): 2" 251 252 echo "--> check validate success after insert data manually" 253 for id in $(seq 5); do 254 run_sql_tidb "insert into $db_name.t1_1 values($id, $id)" 255 run_sql_tidb "insert into $db_name.t2_1 values($id)" 256 done 257 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 258 "validation status test" \ 259 "\"stage\": \"Running\"" 4 \ 260 "processedRowsStatus\": \"insert\/update\/delete: 5\/0\/0\"" 2 \ 261 "pendingRowsStatus\": \"insert\/update\/delete: 0\/0\/0" 2 \ 262 "new\/ignored\/resolved: 0\/0\/0" 2 263 trigger_validator_flush 264 # processed ins/upd/del should be 5/0/0 in both source 265 run_sql_tidb_with_retry "select concat_ws('/', procd_ins, procd_upd, procd_del) processed 266 from dm_meta.test_validator_checkpoint where source='mysql-replica-01'" \ 267 "processed: 5/0/0" 268 run_sql_tidb_with_retry "select concat_ws('/', procd_ins, procd_upd, procd_del) processed 269 from dm_meta.test_validator_checkpoint where source='mysql-replica-02'" \ 270 "processed: 5/0/0" 271 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_pending_change" \ 272 "count(*): 0" 273 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-01'" \ 274 "count(*): 1" 275 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_table_status where source='mysql-replica-02'" \ 276 "count(*): 1" 277 run_sql_tidb_with_retry "select count(*) from dm_meta.test_validator_error_change" \ 278 "count(*): 0" 279 run_sql_tidb_with_retry "select count(*) 280 from dm_meta.test_syncer_checkpoint a, dm_meta.test_validator_checkpoint b 281 where a.cp_schema='' 282 and a.id=b.source 283 and a.binlog_pos=b.binlog_pos 284 and a.binlog_gtid=b.binlog_gtid;" \ 285 "count(*): 2" 286 } 287 288 run $* 289 cleanup_process $* 290 cleanup_data $db_name 291 cleanup_data_upstream $db_name 292 293 echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"