github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/extend_column/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $cur/../_utils/test_prepare
     7  WORK_DIR=$TEST_DIR/$TEST_NAME
     8  db="extend_column"
     9  tb="t"
    10  yb="y"
    11  
    12  function run_case() {
    13  	cleanup_data extend_column extend_column1 extend_column2
    14  	# table `y` has extend and generate column
    15  	# table `t` has extend and different instance
    16  	run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    17  	run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    18  
    19  	# start DM worker and master
    20  	run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml
    21  	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
    22  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    23  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    24  	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
    25  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
    26  
    27  	# operate mysql config to worker
    28  	cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml
    29  	dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1
    30  	cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml
    31  	dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2
    32  
    33  	# start DM task in all mode
    34  	cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task.yaml
    35  	sed -i "s/import-mode-placeholder/$1/g" $WORK_DIR/dm-task.yaml
    36  
    37  	run_sql_tidb "create database extend_column;"
    38  	run_sql_tidb "create table extend_column.t (c1 int primary key, c2 int, c3 int);"
    39  
    40  	# 4 errors for each upstream table
    41  	# 1 warnings: MySQL 8.0
    42  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    43  		"start-task $WORK_DIR/dm-task.yaml --remove-meta" \
    44  		"does not exist in downstream table" 2 \
    45  		"must contain extended columns" 2 \
    46  		'"severity": "fail"' 4 \
    47  		'"severity": "warn"' 0
    48  
    49  	# create table in tidb
    50  	run_sql_file $cur/data/tidb.prepare.sql $TIDB_HOST $TIDB_PORT $TIDB_PASSWORD
    51  	dmctl_start_task "$WORK_DIR/dm-task.yaml" "--remove-meta"
    52  
    53  	# check load data
    54  	# only extend table
    55  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb};" "count(1): 4"
    56  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c_table='1';" "count(1): 2"
    57  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c_table='2';" "count(1): 2"
    58  	# cross mysql instance
    59  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb};" "count(1): 4"
    60  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c_table='1' and c_schema='extend_column1' and c_source='replica01';" "count(1): 2"
    61  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c_table='2' and c_schema='extend_column2' and c_source='replica02';" "count(1): 2"
    62  
    63  	# check incremental data
    64  	run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    65  	run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    66  
    67  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb};" "count(1): 6"
    68  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb};" "count(1): 6"
    69  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=3 and c_table='1';" "count(1): 1"
    70  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=3 and c_table='2';" "count(1): 1"
    71  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=3 and c_table='1' and c_schema='extend_column1' and c_source='replica01';" "count(1): 1"
    72  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=3 and c_table='2' and c_schema='extend_column2' and c_source='replica02';" "count(1): 1"
    73  
    74  	# check update data
    75  	run_sql_file $cur/data/db1.update.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    76  	run_sql_file $cur/data/db2.update.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    77  
    78  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=3 and c_table='1';" "count(1): 1"
    79  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=4 and c_table='1';" "count(1): 0"
    80  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=3 and c_table='2';" "count(1): 0"
    81  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=4 and c_table='2';" "count(1): 1"
    82  
    83  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=3 and c_table='1' and c_schema='extend_column1' and c_source='replica01';" "count(1): 1"
    84  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=4 and c_table='1' and c_schema='extend_column1' and c_source='replica01';" "count(1): 0"
    85  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=3 and c_table='2' and c_schema='extend_column2' and c_source='replica02';" "count(1): 0"
    86  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=4 and c_table='2' and c_schema='extend_column2' and c_source='replica02';" "count(1): 1"
    87  
    88  	# check delete data
    89  	run_sql_file $cur/data/db1.delete.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    90  	run_sql_file $cur/data/db2.delete.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2
    91  
    92  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb};" "count(1): 5"
    93  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=1 and c_table='1';" "count(1): 1"
    94  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=2 and c_table='1';" "count(1): 1"
    95  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=3 and c_table='1';" "count(1): 1"
    96  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=4 and c_table='1';" "count(1): 0"
    97  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=1 and c_table='2';" "count(1): 1"
    98  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=2 and c_table='2';" "count(1): 1"
    99  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=3 and c_table='2';" "count(1): 0"
   100  	run_sql_tidb_with_retry "select count(1) from ${db}.${yb} where c1=4 and c_table='2';" "count(1): 0"
   101  
   102  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb};" "count(1): 4"
   103  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=1 and c_table='1' and c_schema='extend_column1' and c_source='replica01';" "count(1): 1"
   104  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=2 and c_table='1' and c_schema='extend_column1' and c_source='replica01';" "count(1): 1"
   105  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=3 and c_table='1' and c_schema='extend_column1' and c_source='replica01';" "count(1): 1"
   106  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=4 and c_table='1' and c_schema='extend_column1' and c_source='replica01';" "count(1): 0"
   107  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1=1 and c_table='2' and c_schema='extend_column2' and c_source='replica02';" "count(1): 1"
   108  	run_sql_tidb_with_retry "select count(1) from ${db}.${tb} where c1>1 and c_table='2' and c_schema='extend_column2' and c_source='replica02';" "count(1): 0"
   109  
   110  	cleanup_process $*
   111  }
   112  
   113  function run() {
   114  	for import_mode in "sql" "loader"; do
   115  		echo "running extend_column case with import_mode: $import_mode"
   116  		run_case $import_mode
   117  	done
   118  }
   119  
   120  cleanup_process $*
   121  run $*
   122  
   123  echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"