github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/tests/expression_filter/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -eu
     4  
     5  cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
     6  source $cur/../_utils/test_prepare
     7  WORK_DIR=$TEST_DIR/$TEST_NAME
     8  API_VERSION="v1alpha1"
     9  
    10  function complex_behaviour() {
    11  	cleanup_data expr_filter
    12  
    13  	run_dm_master $WORK_DIR/master $MASTER_PORT1 $cur/conf/dm-master.toml
    14  	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT1
    15  
    16  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    17  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    18  	dmctl_operate_source create $cur/conf/source1.yaml $SOURCE_ID1
    19  
    20  	run_sql_file $cur/data/db1.prepare2.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    21  
    22  	# test no permission
    23  	chmod -w $WORK_DIR/worker1
    24  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    25  		"start-task $cur/conf/dm-task2.yaml"
    26  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    27  		"query-status test" \
    28  		"dumpling runs with error" 1 \
    29  		"permission denied" 1
    30  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    31  		"stop-task test"
    32  	chmod +w $WORK_DIR/worker1
    33  
    34  	dmctl_start_task_standalone $cur/conf/dm-task2.yaml
    35  
    36  	run_sql_file $cur/data/db1.increment2.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
    37  
    38  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    39  		"query-status test" \
    40  		"\"result\": true" 2 \
    41  		"\"synced\": true" 1
    42  
    43  	run_sql_tidb "select count(0) from expr_filter.t2"
    44  	check_contains "count(0): 6"
    45  	run_sql_tidb "select count(1) from expr_filter.t2 where should_skip = 0"
    46  	check_contains "count(1): 6"
    47  	run_sql_tidb "select count(2) from expr_filter.t2 where should_skip = 1"
    48  	check_contains "count(2): 0"
    49  
    50  	run_sql_tidb "select count(3) from expr_filter.t3"
    51  	check_contains "count(3): 1"
    52  	run_sql_tidb "select count(4) from expr_filter.t3 where should_skip = 1"
    53  	check_contains "count(4): 0"
    54  
    55  	run_sql_tidb "select count(5) from expr_filter.t4"
    56  	check_contains "count(5): 1"
    57  	run_sql_tidb "select count(6) from expr_filter.t4 where should_skip = 1"
    58  	check_contains "count(6): 0"
    59  
    60  	run_sql_tidb "select count(7) from expr_filter.t5"
    61  	check_contains "count(7): 4"
    62  	run_sql_tidb "select count(8) from expr_filter.t5 where should_skip = 1"
    63  	check_contains "count(8): 0"
    64  
    65  	run_sql_tidb "select count(9) from expr_filter.t6 where name = 'Müller…' and msg = 'Müller…' and name2 = 'Müller…'"
    66  	check_contains "count(9): 2"
    67  	run_sql_tidb "select count(10) from expr_filter.t6 where name != 'Müller…'"
    68  	check_contains "count(10): 0"
    69  
    70  	run_sql_tidb "select count(11) from expr_filter.t7 where r = 'a' and s = 2"
    71  	check_contains "count(11): 1"
    72  	run_sql_tidb "select count(12) from expr_filter.t7 where r = 'a' and s = 3"
    73  	check_contains "count(12): 0"
    74  
    75  	insert_num=$(grep -o '"number of filtered insert"=[0-9]\+' $WORK_DIR/worker1/log/dm-worker.log | grep -o '[0-9]\+' | awk '{n += $1}; END{print n}')
    76  	[ $insert_num -eq 6 ]
    77  	update_num=$(grep -o '"number of filtered update"=[0-9]\+' $WORK_DIR/worker1/log/dm-worker.log | grep -o '[0-9]\+' | awk '{n += $1}; END{print n}')
    78  	[ $update_num -eq 4 ]
    79  
    80  	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
    81  		"stop-task test"
    82  
    83  	ls $WORK_DIR/worker1/schema-tracker* && exit 1 || echo "schema tracker path has been cleaned"
    84  
    85  	cleanup_process $*
    86  	cleanup_data expr_filter
    87  }
    88  
    89  function run() {
    90  	complex_behaviour
    91  
    92  	run_dm_master $WORK_DIR/master $MASTER_PORT1 $cur/conf/dm-master.toml
    93  	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT1
    94  
    95  	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
    96  	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
    97  	dmctl_operate_source create $cur/conf/source1.yaml $SOURCE_ID1
    98  
    99  	run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
   100  
   101  	dmctl_start_task_standalone $cur/conf/dm-task.yaml
   102  
   103  	run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
   104  
   105  	# too many expression filter will slow the sync
   106  	sleep 5
   107  	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
   108  		"query-status test" \
   109  		"\"result\": true" 2 \
   110  		"\"synced\": true" 1
   111  
   112  	run_sql_source1 "select count(9) from expr_filter.t1"
   113  	check_contains "count(9): 60"
   114  
   115  	run_sql_tidb "select count(10) from expr_filter.t1"
   116  	check_contains "count(10): 30"
   117  
   118  	run_sql_tidb "select count(11) from expr_filter.t1 where should_skip = 1"
   119  	check_contains "count(11): 0"
   120  
   121  	insert_num=$(grep -o '"number of filtered insert"=[0-9]\+' $WORK_DIR/worker1/log/dm-worker.log | grep -o '[0-9]\+' | awk '{n += $1}; END{print n}')
   122  	[ $insert_num -eq 30 ]
   123  }
   124  
   125  cleanup_data expr_filter
   126  # also cleanup dm processes in case of last run failed
   127  cleanup_process $*
   128  run $*
   129  cleanup_process $*
   130  
   131  echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"