github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/all_mode/run.sh (about) 1 #!/bin/bash 2 3 set -eu 4 5 cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) 6 source $cur/../_utils/test_prepare 7 WORK_DIR=$TEST_DIR/$TEST_NAME 8 API_VERSION="v1alpha1" 9 ILLEGAL_CHAR_NAME='t-Ë!s`t' 10 11 function test_session_config() { 12 echo "[$(date)] <<<<<< start test_session_config >>>>>>" 13 run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 14 check_contains 'Query OK, 2 rows affected' 15 run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 16 check_contains 'Query OK, 3 rows affected' 17 18 # start DM worker and master 19 run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml 20 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 21 check_metric $MASTER_PORT 'start_leader_counter' 3 0 2 22 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 23 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 24 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 25 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 26 # operate mysql config to worker 27 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 28 cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml 29 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml 30 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml 31 dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 32 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 33 34 cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task.yaml 35 sed -i "s/name: test/name: $ILLEGAL_CHAR_NAME/g" $WORK_DIR/dm-task.yaml 36 37 # error config 38 # there should be a error message like "Incorrect argument type to variable 'tidb_retry_limit'" 39 # but different TiDB version output different message. so we only roughly match here 40 sed -i 's/tidb_retry_limit: "10"/tidb_retry_limit: "fjs"/g' $WORK_DIR/dm-task.yaml 41 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 42 "start-task $WORK_DIR/dm-task.yaml --remove-meta" \ 43 "tidb_retry_limit" 1 44 45 sed -i 's/tidb_retry_limit: "fjs"/tidb_retry_limit: "10"/g' $WORK_DIR/dm-task.yaml 46 dmctl_start_task "$WORK_DIR/dm-task.yaml" "--remove-meta" 47 run_sql_source1 "create table if not exists all_mode.t1 (c int); insert into all_mode.t1 (id, name) values (9, 'haha');" 48 49 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 50 51 run_sql_source1 "set session binlog_row_image = 'minimal'; update all_mode.t1 set name = 'hoho' where id = 9;" 52 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 53 "query-status $ILLEGAL_CHAR_NAME" \ 54 "upstream didn't log enough columns in binlog" 1 55 56 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 57 "stop-task $ILLEGAL_CHAR_NAME" \ 58 "\"result\": true" 3 59 60 cleanup_data all_mode 61 cleanup_process 62 echo "[$(date)] <<<<<< finish test_session_config >>>>>>" 63 64 } 65 66 function test_query_timeout() { 67 echo "[$(date)] <<<<<< start test_query_timeout >>>>>>" 68 export GO_FAILPOINTS="github.com/pingcap/tiflow/dm/syncer/BlockSyncStatus=return(\"5s\")" 69 70 cp $cur/conf/dm-master.toml $WORK_DIR/dm-master.toml 71 sed -i 's/rpc-timeout = "30s"/rpc-timeout = "3s"/g' $WORK_DIR/dm-master.toml 72 73 run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 74 check_contains 'Query OK, 2 rows affected' 75 run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 76 check_contains 'Query OK, 3 rows affected' 77 78 # start DM worker and master 79 run_dm_master $WORK_DIR/master $MASTER_PORT $WORK_DIR/dm-master.toml 80 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 81 check_metric $MASTER_PORT 'start_leader_counter' 3 0 2 82 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 83 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 84 # operate mysql config to worker 85 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 86 cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml 87 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml 88 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml 89 sed -i "s/enable-relay: true/enable-relay: false/g" $WORK_DIR/source1.yaml 90 dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 91 92 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 93 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 94 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 95 96 # don't know why CI has turned on Event Scheduler 97 run_sql_both_source 'SET GLOBAL event_scheduler = OFF;' 98 99 # there's only 1 row in result, which is for SHOW PROCESSLIST 100 run_sql_source1 'SHOW PROCESSLIST;' 101 check_rows_equal 1 102 103 run_sql_source2 'SHOW PROCESSLIST;' 104 check_rows_equal 1 105 106 run_sql_tidb 'SHOW PROCESSLIST;' 107 check_rows_equal 1 108 109 # start DM task only 110 cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task.yaml 111 sed -i "s/name: test/name: $ILLEGAL_CHAR_NAME/g" $WORK_DIR/dm-task.yaml 112 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 113 "start-task $WORK_DIR/dm-task.yaml --remove-meta" 114 check_metric $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker1\"}" 10 1 3 115 116 # `query-status` timeout 117 start_time=$(date +%s) 118 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 119 "query-status $ILLEGAL_CHAR_NAME" \ 120 "context deadline exceeded" 2 121 duration=$(($(date +%s) - $start_time)) 122 if [[ $duration -gt 10 ]]; then 123 echo "query-status takes too much time $duration" 124 exit 1 125 fi 126 127 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 128 129 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 130 "stop-task $ILLEGAL_CHAR_NAME" \ 131 "\"result\": true" 3 132 133 # there's only 1 row in result, which is for SHOW PROCESSLIST 134 run_sql_source1 'SHOW PROCESSLIST;' 135 check_rows_equal 1 136 137 run_sql_source2 'SHOW PROCESSLIST;' 138 check_rows_equal 1 139 140 run_sql_tidb 'SHOW PROCESSLIST;' 141 check_rows_equal 1 142 143 cleanup_process 144 cleanup_data all_mode 145 146 export GO_FAILPOINTS='' 147 echo "[$(date)] <<<<<< finish test_query_timeout >>>>>>" 148 } 149 150 function test_fail_job_between_event() { 151 echo "[$(date)] <<<<<< start test_fail_job_between_event >>>>>>" 152 run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 153 check_contains 'Query OK, 2 rows affected' 154 run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 155 check_contains 'Query OK, 3 rows affected' 156 157 # start DM worker and master 158 run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml 159 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 160 check_metric $MASTER_PORT 'start_leader_counter' 3 0 2 161 162 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 163 cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml 164 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml 165 sed -i "s/enable-gtid: true/enable-gtid: false/g" $WORK_DIR/source1.yaml 166 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml 167 168 # worker1 will be bound to source1 and fail when see the second row change in an event 169 inject_points=( 170 "github.com/pingcap/tiflow/dm/worker/TaskCheckInterval=return(\"500ms\")" 171 "github.com/pingcap/tiflow/dm/syncer/countJobFromOneEvent=return()" 172 "github.com/pingcap/tiflow/dm/syncer/flushFirstJob=return()" 173 "github.com/pingcap/tiflow/dm/syncer/failSecondJob=return()" 174 ) 175 export GO_FAILPOINTS="$(join_string \; ${inject_points[@]})" 176 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 177 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 178 dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 179 180 # worker2 will be bound to source2 and fail when see the second event in a GTID 181 inject_points=( 182 "github.com/pingcap/tiflow/dm/worker/TaskCheckInterval=return(\"500ms\")" 183 "github.com/pingcap/tiflow/dm/syncer/countJobFromOneGTID=return()" 184 "github.com/pingcap/tiflow/dm/syncer/flushFirstJob=return()" 185 "github.com/pingcap/tiflow/dm/syncer/failSecondJob=return()" 186 ) 187 export GO_FAILPOINTS="$(join_string \; ${inject_points[@]})" 188 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 189 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 190 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 191 192 dmctl_start_task "$cur/conf/dm-task.yaml" "--remove-meta" 193 194 run_sql_file $cur/data/db1.increment3.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 195 run_sql_file $cur/data/db2.increment3.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 196 check_log_contain_with_retry "failSecondJob" $WORK_DIR/worker1/log/dm-worker.log 197 check_log_contain_with_retry "failSecondJob" $WORK_DIR/worker2/log/dm-worker.log 198 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 199 "query-status test" \ 200 "\"result\": true" 3 201 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 202 203 cleanup_process 204 cleanup_data all_mode 205 206 export GO_FAILPOINTS='' 207 echo "[$(date)] <<<<<< finish test_fail_job_between_event >>>>>>" 208 } 209 210 function test_expression_filter() { 211 echo "[$(date)] <<<<<< start test_expression_filter >>>>>>" 212 run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 213 check_contains 'Query OK, 2 rows affected' 214 run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 215 check_contains 'Query OK, 3 rows affected' 216 217 # start DM worker and master 218 run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml 219 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 220 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 221 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 222 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 223 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 224 225 # operate mysql config to worker 226 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 227 cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml 228 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml 229 sed -i "s/enable-gtid: true/enable-gtid: false/g" $WORK_DIR/source1.yaml 230 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml 231 dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 232 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 233 234 dmctl_start_task "$cur/conf/dm-task-expression-filter.yaml" "--remove-meta" 235 236 run_sql_file $cur/data/db1.increment3.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 237 run_sql_file $cur/data/db2.increment3.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 238 239 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 240 run_sql_tidb "SELECT count(*) from all_mode.t1;" 241 check_contains "count(*): 6" 242 run_sql_source1 "DELETE FROM all_mode.t1 WHERE id = 30;" 243 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 3 "fail" 244 run_sql_tidb "SELECT count(*) from all_mode.t1;" 245 check_contains "count(*): 6" 246 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 247 "query-status test" \ 248 "\"result\": true" 3 249 250 cleanup_process 251 cleanup_data all_mode 252 echo "[$(date)] <<<<<< finish test_expression_filter >>>>>>" 253 } 254 255 function test_regexpr_router() { 256 echo "[$(date)] <<<<<< start test_regexpr_router $1 >>>>>>" 257 cleanup_process 258 cleanup_data all_mode 259 cleanup_data test2animal 260 cleanup_data test4s_2022 261 cleanup_data_upstream test2animal 262 cleanup_data_upstream test4s_2022 263 run_sql_file $cur/data/db1.regexpr.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 264 run_sql_file $cur/data/db2.regexpr.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 265 run_sql_tidb 'drop database if exists dtest2;' 266 run_sql_tidb 'create database dtest2;' 267 run_sql_tidb 'drop database if exists dtest4;' 268 run_sql_tidb 'create database dtest4;' 269 run_sql_tidb 'create table if not exists dtest2.dtable2(a int, b int);' 270 run_sql_tidb 'create table if not exists dtest4.dtable4(a int, b int);' 271 # start DM worker and master 272 run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml 273 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 274 check_metric $MASTER_PORT 'start_leader_counter' 3 0 2 275 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 276 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 277 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 278 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 279 # operate mysql config to worker 280 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 281 cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml 282 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml 283 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml 284 dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 285 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 286 cp $cur/conf/$1 $WORK_DIR/$1 287 sed -i "s/name: test/name: $ILLEGAL_CHAR_NAME/g" $WORK_DIR/$1 288 289 # error config 290 # there should be a error message like "Incorrect argument type to variable 'tidb_retry_limit'" 291 # but different TiDB version output different message. so we only roughly match here 292 sed -i 's/tidb_retry_limit: "10"/tidb_retry_limit: "fjs"/g' $WORK_DIR/$1 293 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 294 "start-task $WORK_DIR/$1 --remove-meta" \ 295 "tidb_retry_limit" 1 296 297 sed -i 's/tidb_retry_limit: "fjs"/tidb_retry_limit: "10"/g' $WORK_DIR/$1 298 dmctl_start_task "$WORK_DIR/$1" "--remove-meta" 299 300 check_sync_diff $WORK_DIR $cur/conf/regexpr_diff_config.toml 301 302 cleanup_process 303 cleanup_data test2animal 304 cleanup_data test4s_2022 305 cleanup_data_upstream test2animal 306 cleanup_data_upstream test4s_2022 307 echo "[$(date)] <<<<<< finish test_regexpr_router $1 >>>>>>" 308 } 309 310 function test_json_expression() { 311 echo "[$(date)] <<<<<< start test_json_expression >>>>>>" 312 run_sql_file $cur/data/db3.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 313 314 # start DM worker and master 315 run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml 316 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 317 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 318 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 319 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 320 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 321 322 # operate mysql config to worker 323 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 324 cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml 325 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml 326 sed -i "s/enable-gtid: true/enable-gtid: false/g" $WORK_DIR/source1.yaml 327 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml 328 dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 329 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 330 331 dmctl_start_task "$cur/conf/dm-task-expression-filter.yaml" "--remove-meta" 332 333 run_sql_file $cur/data/db3.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 334 335 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 336 337 cleanup_process 338 cleanup_data all_mode 339 echo "[$(date)] <<<<<< finish test_json_expression >>>>>>" 340 } 341 342 function run() { 343 run_sql_both_source "SET @@GLOBAL.SQL_MODE='ANSI_QUOTES,NO_AUTO_VALUE_ON_ZERO'" 344 run_sql_source1 "SET @@global.time_zone = '+01:00';" 345 run_sql_source2 "SET @@global.time_zone = '+02:00';" 346 test_expression_filter 347 test_json_expression 348 test_fail_job_between_event 349 test_session_config 350 test_query_timeout 351 test_regexpr_router regexpr-task.yaml 352 test_regexpr_router regexpr-task-lightning.yaml 353 354 inject_points=( 355 "github.com/pingcap/tiflow/dm/worker/TaskCheckInterval=return(\"500ms\")" 356 "github.com/pingcap/tiflow/dm/relay/NewUpstreamServer=return(true)" 357 ) 358 export GO_FAILPOINTS="$(join_string \; ${inject_points[@]})" 359 360 # manually create target table with two extra field 361 run_sql_tidb "drop database if exists all_mode;" 362 run_sql_tidb "create database all_mode;" 363 run_sql_tidb "drop table if exists all_mode.no_diff;" 364 run_sql_tidb "create table all_mode.no_diff(id int NOT NULL PRIMARY KEY, dt datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP);" 365 366 run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 367 check_contains 'Query OK, 2 rows affected' 368 run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 369 check_contains 'Query OK, 3 rows affected' 370 371 # create table with unsupported charset 372 run_sql_source1 "create table all_mode.no_diff2(id int primary key, name varchar(20)) charset=greek;" 373 run_sql_source1 "insert into all_mode.no_diff2 values(1, 'αβγ');" 374 375 # start DM worker and master 376 # set log level of DM-master to info, because debug level will let etcd print KV, thus expose the password in task config 377 run_dm_master_info_log $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml 378 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 379 check_metric $MASTER_PORT 'start_leader_counter' 3 0 2 380 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 381 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 382 383 # operate mysql config to worker 384 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 385 cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml 386 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml 387 sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml 388 # make sure source1 is bound to worker1 389 dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 390 391 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 392 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 393 dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 394 395 # start DM task only 396 cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task.yaml 397 sed -i "s/name: test/name: $ILLEGAL_CHAR_NAME/g" $WORK_DIR/dm-task.yaml 398 399 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 400 "start-task $WORK_DIR/dm-task.yaml --remove-meta" \ 401 "Unknown character set: 'greek'" 402 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 403 "query-status $ILLEGAL_CHAR_NAME" \ 404 "Unknown character set: 'greek'" 1 405 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 406 "stop-task $WORK_DIR/dm-task.yaml" 407 408 run_sql_tidb "create table all_mode.no_diff2(id int primary key, name varchar(20)) charset=utf8mb4;" 409 410 dmctl_start_task "$WORK_DIR/dm-task.yaml" "--remove-meta" 411 # check task has started 412 check_metric $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker1\"}" 10 1 3 413 check_metric $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker2\"}" 10 1 3 414 415 # use sync_diff_inspector to check full dump loader 416 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 417 # check data of no_diff2 418 run_sql_tidb_with_retry "select count(1) from all_mode.no_diff2 where name = 'αβγ'" "count(1): 1" 419 420 # check create view(should be skipped by func `skipSQLByPattern`) will not stop sync task 421 run_sql_source1 "create view all_mode.t1_v as select * from all_mode.t1 where id=0;" 422 sleep 1 423 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 424 "query-status -s $SOURCE_ID1" \ 425 "\"result\": true" 2 \ 426 "\"unit\": \"Sync\"" 1 \ 427 "\"stage\": \"Running\"" 2 428 429 run_sql_source1 "SHOW SLAVE HOSTS;" 430 check_contains 'Slave_UUID' 431 432 run_sql_tidb "set time_zone = '+04:00';SELECT count(*) from all_mode.no_diff where dt = ts;" 433 check_contains "count(*): 3" 434 435 # check default session config 436 check_log_contain_with_retry '\\"tidb_txn_mode\\":\\"optimistic\\"' $WORK_DIR/worker1/log/dm-worker.log 437 check_log_contain_with_retry '\\"tidb_txn_mode\\":\\"optimistic\\"' $WORK_DIR/worker2/log/dm-worker.log 438 439 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 440 "pause-task $ILLEGAL_CHAR_NAME" \ 441 "\"result\": true" 3 442 echo 'create table all_mode.no_diff(id int NOT NULL PRIMARY KEY);' >${WORK_DIR}/schema.sql 443 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 444 "operate-schema set -s mysql-replica-01 $ILLEGAL_CHAR_NAME -d all_mode -t no_diff ${WORK_DIR}/schema.sql" \ 445 "\"result\": true" 2 446 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" "resume-task $ILLEGAL_CHAR_NAME" "\"result\": true" 3 447 448 # restart dm-worker1 449 pkill -hup -f dm-worker1.toml 2>/dev/null || true 450 wait_pattern_exit dm-worker1.toml 451 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml 452 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 453 # make sure worker1 have bound a source, and the source should same with bound before 454 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 455 "query-status $ILLEGAL_CHAR_NAME" \ 456 "worker1" 1 457 458 # restart dm-worker2 459 pkill -hup -f dm-worker2.toml 2>/dev/null || true 460 wait_pattern_exit dm-worker2.toml 461 run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml 462 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT 463 check_metric $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker1\"}" 10 1 3 464 check_metric $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker2\"}" 10 1 3 465 466 sleep 10 467 echo "after restart dm-worker, task should resume automatically" 468 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 469 "start-task $WORK_DIR/dm-task.yaml" \ 470 "\"result\": false" 1 \ 471 "subtasks with name $ILLEGAL_CHAR_NAME for sources \[mysql-replica-01 mysql-replica-02\] already exist" 1 472 sleep 2 473 474 # wait for task running 475 check_http_alive 127.0.0.1:$MASTER_PORT/apis/${API_VERSION}/status/$ILLEGAL_CHAR_NAME '"stage": "Running"' 10 476 sleep 2 # still wait for subtask running on other dm-workers 477 478 # kill tidb 479 pkill -hup tidb-server 2>/dev/null || true 480 wait_process_exit tidb-server 481 482 # dm-worker execute sql failed, and will try auto resume task 483 run_sql_file $cur/data/db2.increment0.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 484 sleep 2 485 check_log_contains $WORK_DIR/worker2/log/dm-worker.log "dispatch auto resume task" 486 487 # restart tidb, and task will recover success 488 run_tidb_server 4000 $TIDB_PASSWORD 489 sleep 2 490 491 # test after pause and resume relay, relay could continue from syncer's checkpoint 492 run_sql_source1 "flush logs" 493 run_sql_file $cur/data/db1.increment0.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 494 495 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 496 497 # check compatibility after incremental sync 498 run_sql_tidb "set time_zone = '+04:00';SELECT count(*) from all_mode.no_diff where dt = ts;" 499 check_contains "count(*): 4" 500 501 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 502 "pause-relay -s mysql-replica-01" \ 503 "\"result\": true" 2 504 # we used failpoint to imitate an upstream switching, which purged whole relay dir 505 run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 506 run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 507 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 508 "resume-relay -s mysql-replica-01" \ 509 "\"result\": true" 2 510 511 sleep 2 512 # relay should continue pulling from syncer's checkpoint, so only pull the latest binlog 513 server_uuid=$(tail -n 1 $WORK_DIR/worker1/relay_log/server-uuid.index) 514 echo "relay logs $(ls $WORK_DIR/worker1/relay_log/$server_uuid)" 515 relay_log_num=$(ls $WORK_DIR/worker1/relay_log/$server_uuid | grep -v 'relay.meta' | wc -l) 516 [ $relay_log_num -eq 1 ] 517 518 # use sync_diff_inspector to check data now! 519 check_sync_diff $WORK_DIR $cur/conf/diff_config.toml 520 521 # test block-allow-list by the way 522 run_sql "show databases;" $TIDB_PORT $TIDB_PASSWORD 523 check_not_contains "ignore_db" 524 check_contains "all_mode" 525 526 echo "check dump files have been cleaned" 527 # source1 contains unsupported charset, so dump files is uncleaned. files are 528 # all_mode.no_diff2-schema.sql all_mode.t1-schema.sql 529 # all_mode.no_diff-schema.sql metadata 530 # all_mode-schema-create.sql 531 [ $(ls $WORK_DIR/worker1/dumped_data.$ILLEGAL_CHAR_NAME | wc -l) -eq 5 ] 532 ls $WORK_DIR/worker2/dumped_data.$ILLEGAL_CHAR_NAME && exit 1 || echo "worker2 auto removed dump files" 533 534 echo "check no password in log" 535 check_log_not_contains $WORK_DIR/master/log/dm-master.log "123456" 536 check_log_not_contains $WORK_DIR/worker1/log/dm-worker.log "123456" 537 check_log_not_contains $WORK_DIR/worker2/log/dm-worker.log "123456" 538 539 # test drop table if exists 540 run_sql_source1 "drop table if exists \`all_mode\`.\`tb1\`;" 541 run_sql_source1 "drop table if exists \`all_mode\`.\`tb1\`;" 542 run_sql_source2 "drop table if exists \`all_mode\`.\`tb2\`;" 543 run_sql_source2 "drop table if exists \`all_mode\`.\`tb2\`;" 544 check_log_not_contains $WORK_DIR/worker1/log/dm-worker.log "Error .* Table .* doesn't exist" 545 check_log_not_contains $WORK_DIR/worker2/log/dm-worker.log "Error .* Table .* doesn't exist" 546 547 # test Db not exists should be reported 548 run_sql_tidb "drop database all_mode" 549 run_sql_source1 "create table all_mode.db_error (c int primary key);" 550 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 551 "query-status $ILLEGAL_CHAR_NAME" \ 552 "Error 1049 (42000): Unknown database" 1 553 554 # stop task, task state should be cleaned 555 run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 556 "stop-task $ILLEGAL_CHAR_NAME" \ 557 "\"result\": true" 3 558 check_metric_not_contains $WORKER1_PORT "dm_worker_task_state{source_id=\"mysql-replica-01\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker1\"}" 3 559 check_metric_not_contains $WORKER2_PORT "dm_worker_task_state{source_id=\"mysql-replica-02\",task=\"$ILLEGAL_CHAR_NAME\",worker=\"worker2\"}" 3 560 561 # all unit without error. 562 check_metric_not_contains $WORKER1_PORT "dm_mydumper_exit_with_error_count" 3 563 check_metric_not_contains $WORKER1_PORT "dm_loader_exit_with_error_count" 3 564 check_metric_not_contains $WORKER1_PORT "dm_syncer_exit_with_error_count" 3 565 566 # check syncer metrics 567 check_two_metric_equal $WORKER1_PORT 'dm_syncer_binlog_file{node="master"' 'dm_syncer_binlog_file{node="syncer"' 3 568 check_two_metric_equal $WORKER2_PORT 'dm_syncer_binlog_file{node="master"' 'dm_syncer_binlog_file{node="syncer"' 3 569 export GO_FAILPOINTS='' 570 571 run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" 572 run_sql_both_source "SET @@global.time_zone = 'SYSTEM';" 573 574 test_source_and_target_with_empty_gtid 575 } 576 577 function prepare_test_empty_gtid() { 578 run_sql 'DROP DATABASE if exists all_mode;' $TIDB_PORT $TIDB_PASSWORD 579 run_sql 'DROP DATABASE if exists all_mode;' $MYSQL_PORT1 $MYSQL_PASSWORD1 580 run_sql 'DROP DATABASE if exists xxx;' $MYSQL_PORT1 $MYSQL_PASSWORD1 581 run_sql 'CREATE DATABASE all_mode;' $MYSQL_PORT1 $MYSQL_PASSWORD1 582 run_sql "CREATE TABLE all_mode.t1(i TINYINT, j INT UNIQUE KEY);" $MYSQL_PORT1 $MYSQL_PASSWORD1 583 584 run_sql 'reset master;' $MYSQL_PORT1 $MYSQL_PASSWORD1 585 } 586 587 function test_source_and_target_with_empty_gtid() { 588 echo "[$(date)] <<<<<< start test_source_and_target_with_empty_gtid >>>>>>" 589 cleanup_process 590 cleanup_data all_mode 591 prepare_test_empty_gtid 592 593 cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml 594 cp $cur/conf/dm-master.toml $WORK_DIR/ 595 cp $cur/conf/dm-worker1.toml $WORK_DIR/ 596 cp $cur/conf/dm-task-no-gtid.yaml $WORK_DIR/ 597 598 # start DM worker and master 599 run_dm_master $WORK_DIR/master $MASTER_PORT $WORK_DIR/dm-master.toml 600 check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT 601 run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $WORK_DIR/dm-worker1.toml 602 check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT 603 604 # operate mysql config to worker 605 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 606 "operate-source create $WORK_DIR/source1.yaml" \ 607 "\"result\": true" 2 \ 608 "\"source\": \"$SOURCE_ID1\"" 1 609 610 echo "check master alive" 611 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 612 "list-member" \ 613 "\"alive\": true" 1 614 615 # check current master gtid is empty 616 # len indicates the number of non-empty fields 617 len=$(echo "show master status;" | MYSQL_PWD=$MYSQL_PASSWORD1 mysql -uroot -h127.0.0.1 -P$MYSQL_PORT1 | awk 'FNR == 2 {print NF}') 618 if [ "$len" = 2 ]; then 619 echo "gtid is empty" 620 else 621 exit 1 622 fi 623 624 echo "start task and check stage" 625 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 626 "start-task $WORK_DIR/dm-task-no-gtid.yaml --remove-meta=true" \ 627 "\"result\": true" 2 628 629 run_sql 'INSERT INTO all_mode.t1 VALUES (1,1001);' $MYSQL_PORT1 $MYSQL_PASSWORD1 630 631 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 632 "query-status test" \ 633 "\"result\": true" 2 \ 634 "\"unit\": \"Sync\"" 1 \ 635 "\"stage\": \"Running\"" 2 636 637 echo "check data" 638 check_sync_diff $WORK_DIR $cur/conf/diff_config-1.toml 639 640 # check checkpoint matches master when the last event is a ddl 641 # 1. ddl that dm will sync 642 run_sql_source1 "create table all_mode.t2(c int primary key)" 643 run_sql_tidb_with_retry "show create table all_mode.t2" "CREATE TABLE" 644 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 645 "query-status test" \ 646 '"synced": true' 1 647 # 2. ddl cannot be parsed and should be skipped 648 run_sql_source1 "create FUNCTION all_mode.hello (s CHAR(20)) RETURNS CHAR(50) DETERMINISTIC RETURN 'a';" 649 check_log_contain_with_retry "RETURNS char(50)" $WORK_DIR/worker1/log/dm-worker.log 650 sleep 30 # we rely on heartbeat event to flush checkpoint here, below too 651 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 652 "query-status test" \ 653 '"synced": true' 1 654 # 3. ddl can be parsed and dm don't handle 655 run_sql_source1 "analyze table all_mode.t1" 656 check_log_contain_with_retry "analyze table" $WORK_DIR/worker1/log/dm-worker.log 657 sleep 30 658 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 659 "query-status test" \ 660 '"synced": true' 1 661 # 4. ddl that is filtered 662 run_sql_source1 "create database xxx" 663 check_log_contain_with_retry "CREATE DATABASE IF NOT EXISTS" $WORK_DIR/worker1/log/dm-worker.log 664 sleep 30 665 run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \ 666 "query-status test" \ 667 '"synced": true' 1 668 669 echo "<<<<<< test_source_and_target_with_empty_gtid success! >>>>>>" 670 } 671 672 cleanup_data_upstream all_mode 673 cleanup_data all_mode 674 # also cleanup dm processes in case of last run failed 675 cleanup_process $* 676 run $* 677 cleanup_process $* 678 679 echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"