github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/tests/cli/run.sh (about)

     1  #!/bin/bash
     2  
     3  set -e
     4  
     5  CUR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
     6  source $CUR/../_utils/test_prepare
     7  WORK_DIR=$OUT_DIR/$TEST_NAME
     8  CDC_BINARY=cdc.test
     9  SINK_TYPE=$1
    10  TLS_DIR=$( cd $CUR/../_certificates && pwd )
    11  
    12  function check_changefeed_state() {
    13      changefeedid=$1
    14      expected=$2
    15      output=$(cdc cli changefeed query --simple --changefeed-id $changefeedid --pd=http://$UP_PD_HOST_1:$UP_PD_PORT_1 2>&1)
    16      state=$(echo $output | grep -oE "\"state\": \"[a-z]+\""|tr -d '" '|awk -F':' '{print $(NF)}')
    17      if [ "$state" != "$expected" ]; then
    18          echo "unexpected state $output, expected $expected"
    19          exit 1
    20      fi
    21  }
    22  
    23  function check_changefeed_count() {
    24      pd_addr=$1
    25      expected=$2
    26      feed_count=$(cdc cli changefeed list --pd=$pd_addr|jq '.|length')
    27      if [[ "$feed_count" != "$expected" ]]; then
    28          echo "[$(date)] <<<<< unexpect changefeed count! expect ${expected} got ${feed_count} >>>>>"
    29          exit 1
    30      fi
    31      echo "changefeed count ${feed_count} check pass, pd_addr: $pd_addr"
    32  }
    33  
    34  function run() {
    35      rm -rf $WORK_DIR && mkdir -p $WORK_DIR
    36  
    37      start_tidb_cluster --workdir $WORK_DIR --multiple-upstream-pd true
    38  
    39      cd $WORK_DIR
    40      pd_addr="http://$UP_PD_HOST_1:$UP_PD_PORT_1"
    41  
    42      # record tso before we create tables to skip the system table DDLs
    43      start_ts=$(run_cdc_cli tso query --pd=http://$UP_PD_HOST_1:$UP_PD_PORT_1)
    44      run_sql "CREATE table test.simple(id int primary key, val int);"
    45      run_sql "CREATE table test.\`simple-dash\`(id int primary key, val int);"
    46  
    47      run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY
    48  
    49      TOPIC_NAME="ticdc-cli-test-$RANDOM"
    50      case $SINK_TYPE in
    51          kafka) SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&kafka-version=${KAFKA_VERSION}";;
    52          *) SINK_URI="mysql://normal:123456@127.0.0.1:3306/";;
    53      esac
    54  
    55      uuid="custom-changefeed-name"
    56      run_cdc_cli changefeed create --start-ts=$start_ts --sort-engine=memory --sink-uri="$SINK_URI" --tz="Asia/Shanghai" -c="$uuid"
    57      if [ "$SINK_TYPE" == "kafka" ]; then
    58        run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&version=${KAFKA_VERSION}"
    59      fi
    60  
    61      run_cdc_cli changefeed cyclic create-marktables \
    62          --cyclic-upstream-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/"
    63  
    64      # Make sure changefeed is created.
    65      check_table_exists tidb_cdc.repl_mark_test_simple ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
    66      check_table_exists tidb_cdc."\`repl_mark_test_simple-dash\`" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
    67  
    68      check_changefeed_state $uuid "normal"
    69  
    70      check_changefeed_count http://${UP_PD_HOST_1}:${UP_PD_PORT_1} 1
    71      check_changefeed_count http://${UP_PD_HOST_2}:${UP_PD_PORT_2} 1
    72      check_changefeed_count http://${UP_PD_HOST_3}:${UP_PD_PORT_3} 1
    73      check_changefeed_count http://${UP_PD_HOST_1}:${UP_PD_PORT_1},http://${UP_PD_HOST_2}:${UP_PD_PORT_2},http://${UP_PD_HOST_3}:${UP_PD_PORT_3} 1
    74  
    75      # Make sure changefeed can not be created if the name is already exists.
    76      set +e
    77      exists=$(run_cdc_cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" --changefeed-id="$uuid" 2>&1 | grep -oE 'already exists')
    78      set -e
    79      if [[ -z $exists ]]; then
    80          echo "[$(date)] <<<<< unexpect output got ${exists} >>>>>"
    81          exit 1
    82      fi
    83  
    84      # Update changefeed failed because changefeed is running
    85  cat - >"$WORK_DIR/changefeed.toml" <<EOF
    86  case-sensitive = false
    87  [mounter]
    88  worker-num = 4
    89  EOF
    90      set +e
    91      update_result=$(cdc cli changefeed update --pd=$pd_addr --config="$WORK_DIR/changefeed.toml" --no-confirm --changefeed-id $uuid)
    92      set -e
    93      if [[ ! $update_result == *"can only update changefeed config when it is stopped"* ]]; then
    94          echo "update changefeed config should fail when changefeed is running, got $update_result"
    95      fi
    96  
    97      # Pause changefeed
    98      run_cdc_cli changefeed --changefeed-id $uuid pause && sleep 3
    99      jobtype=$(run_cdc_cli changefeed --changefeed-id $uuid query 2>&1 | grep 'admin-job-type' | grep -oE '[0-9]' | head -1)
   100      if [[ $jobtype != 1 ]]; then
   101          echo "[$(date)] <<<<< unexpect admin job type! expect 1 got ${jobtype} >>>>>"
   102          exit 1
   103      fi
   104      check_changefeed_state $uuid "stopped"
   105  
   106      # Update changefeed
   107      run_cdc_cli changefeed update --pd=$pd_addr --config="$WORK_DIR/changefeed.toml" --no-confirm --changefeed-id $uuid
   108      changefeed_info=$(run_cdc_cli changefeed query --changefeed-id $uuid 2>&1)
   109      if [[ ! $changefeed_info == *"\"case-sensitive\": false"* ]]; then
   110          echo "[$(date)] <<<<< changefeed info is not updated as expected ${changefeed_info} >>>>>"
   111          exit 1
   112      fi
   113      if [[ ! $changefeed_info == *"\"worker-num\": 4"* ]]; then
   114          echo "[$(date)] <<<<< changefeed info is not updated as expected ${changefeed_info} >>>>>"
   115          exit 1
   116      fi
   117      if [[ ! $changefeed_info == *"\"sort-engine\": \"memory\""* ]]; then
   118          echo "[$(date)] <<<<< changefeed info is not updated as expected ${changefeed_info} >>>>>"
   119          exit 1
   120      fi
   121  
   122      jobtype=$(run_cdc_cli changefeed --changefeed-id $uuid query 2>&1 | grep 'admin-job-type' | grep -oE '[0-9]' | head -1)
   123      if [[ $jobtype != 1 ]]; then
   124          echo "[$(date)] <<<<< unexpect admin job type! expect 1 got ${jobtype} >>>>>"
   125          exit 1
   126      fi
   127  
   128      # Resume changefeed
   129      run_cdc_cli changefeed --changefeed-id $uuid resume && sleep 3
   130      jobtype=$(run_cdc_cli changefeed --changefeed-id $uuid query 2>&1 | grep 'admin-job-type' | grep -oE '[0-9]' | head -1)
   131      if [[ $jobtype != 2 ]]; then
   132          echo "[$(date)] <<<<< unexpect admin job type! expect 2 got ${jobtype} >>>>>"
   133          exit 1
   134      fi
   135      check_changefeed_state $uuid "normal"
   136  
   137      # Remove changefeed
   138      run_cdc_cli changefeed --changefeed-id $uuid remove && sleep 3
   139      jobtype=$(run_cdc_cli changefeed --changefeed-id $uuid query 2>&1 | grep 'admin-job-type' | grep -oE '[0-9]' | head -1)
   140      if [[ $jobtype != 3 ]]; then
   141          echo "[$(date)] <<<<< unexpect admin job type! expect 3 got ${jobtype} >>>>>"
   142          exit 1
   143      fi
   144      check_changefeed_state $uuid "removed"
   145  
   146      set +e
   147      # Make sure changefeed can not be created if a removed changefeed with the same name exists
   148      create_log=$(run_cdc_cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" --changefeed-id="$uuid" 2>&1)
   149      set -e
   150      exists=$(echo $create_log | grep -oE 'already exists')
   151      if [[ -z $exists ]]; then
   152          echo "[$(date)] <<<<< unexpect output got ${create_log} >>>>>"
   153          exit 1
   154      fi
   155  
   156      # force remove the changefeed, and re create a new one with the same name
   157      run_cdc_cli changefeed --changefeed-id $uuid remove --force && sleep 3
   158      run_cdc_cli changefeed create --sink-uri="$SINK_URI" --tz="Asia/Shanghai" -c="$uuid" && sleep 3
   159      check_changefeed_state $uuid "normal"
   160  
   161      # Make sure bad sink url fails at creating changefeed.
   162      badsink=$(run_cdc_cli changefeed create --start-ts=$start_ts --sink-uri="mysql://badsink" 2>&1 | grep -oE 'fail')
   163      if [[ -z $badsink ]]; then
   164          echo "[$(date)] <<<<< unexpect output got ${badsink} >>>>>"
   165          exit 1
   166      fi
   167  
   168      # Test Kafka SSL connection.
   169      if [ "$SINK_TYPE" == "kafka" ]; then
   170          SSL_TOPIC_NAME="ticdc-cli-test-ssl-$RANDOM"
   171          SINK_URI="kafka://127.0.0.1:9093/$SSL_TOPIC_NAME?ca=${TLS_DIR}/ca.pem&cert=${TLS_DIR}/client.pem&key=${TLS_DIR}/client-key.pem&kafka-version=${KAFKA_VERSION}"
   172          run_cdc_cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" --tz="Asia/Shanghai"
   173      fi
   174  
   175      # Smoke test unsafe commands
   176      echo "y" | run_cdc_cli unsafe delete-service-gc-safepoint
   177      run_cdc_cli unsafe reset --no-confirm
   178  
   179      # Smoke test change log level
   180      curl -X POST -d '"warn"' http://127.0.0.1:8300/admin/log
   181      sleep 3
   182      # make sure TiCDC does not panic
   183      curl http://127.0.0.1:8300/status
   184  
   185      cleanup_process $CDC_BINARY
   186  }
   187  
   188  trap stop_tidb_cluster EXIT
   189  run $*
   190  check_logs $WORK_DIR
   191  echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>"