github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/tests/_utils/run_services (about)

     1  #!/bin/sh
     2  #
     3  # Copyright 2019 PingCAP, Inc.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # See the License for the specific language governing permissions and
    14  # limitations under the License.
    15  
    16  set -eu
    17  
    18  export PD_PEER_ADDR="127.0.0.1:2380"
    19  export PD_ADDR="127.0.0.1:2379"
    20  export TIDB_IP="127.0.0.1"
    21  export TIDB_PORT="4000"
    22  export TIDB_ADDR="127.0.0.1:4000"
    23  export TIDB_STATUS_ADDR="127.0.0.1:10080"
    24  # actual tikv_addr are TIKV_ADDR${i}
    25  export TIKV_ADDR="127.0.0.1:2016"
    26  export TIKV_STATUS_ADDR="127.0.0.1:2018"
    27  export TIKV_COUNT=3
    28  export TIFLASH_STATUS="127.0.0.1:17000"
    29  export TIFLASH_HTTP="127.0.0.1:8125"
    30  export IMPORTER_ADDR="127.0.0.1:8808"
    31  export TIKV_PIDS="${TEST_DIR:?}/tikv_pids.txt"
    32  
    33  cleanup_data() {
    34      # Clean up data
    35      for svc in "br" "tidb" "tiflash" "tikv" "pd" "importer"; do
    36          find "$TEST_DIR" -maxdepth 1 -name "${svc}*" -type d -exec echo delete {} \; -exec rm -rf {} \; 2> /dev/null
    37      done
    38  }
    39  
    40  stop() {
    41      svc=$1
    42      killall -v -1 "$svc" 2>/dev/null || return 0
    43      sleep 1 # give some grace shutdown period
    44      killall -v -9 "$svc" &>/dev/null || return 0
    45  }
    46  
    47  stop_services() {
    48      for svc in "br" "tidb-server" "tiflash" "TiFlashMain" "tikv-server" "pd-server" "cdc" "minio" "tikv-importer"; do
    49          stop $svc &
    50      done
    51      sleep 2 # give some time for the OS to reap all processes
    52      lsof -n -P -i :2379 -i :4000 -i :10080 -i :20161 -i :20162 -i :20163 -i :20181 -i :20182 -i :20183 -i :17000 -i :8125 || true
    53  }
    54  
    55  start_services() {
    56      max_retry=3
    57      for retry_time in $(seq 1 $max_retry); do
    58          # run it in a subshell so the failure won't stop execution.
    59          if ( start_services_impl "$@" ); then
    60              return 0
    61          fi
    62          stop_services
    63          echo "Failed to start services, but let's retry it after $(( $retry_time * 30 )) seconds"
    64          sleep $(( $retry_time * 30 ))
    65      done
    66      echo "Failed to start services after retry $max_retry times."
    67      return 1
    68  }
    69  
    70  start_pd() {
    71      echo "Starting PD..."
    72      mkdir -p "$TEST_DIR/pd"
    73      bin/pd-server \
    74          --client-urls "https://$PD_ADDR" \
    75          --peer-urls "https://$PD_PEER_ADDR" \
    76          --log-file "$TEST_DIR/pd.log" \
    77          --data-dir "$TEST_DIR/pd" \
    78          --config $PD_CONFIG &
    79      # wait until PD is online...
    80      i=0
    81      while ! run_curl "https://$PD_ADDR/pd/api/v1/version"; do
    82         i=$((i+1))
    83         if [ "$i" -gt 20 ]; then
    84            echo 'Failed to start PD'
    85            return 1
    86         fi
    87         sleep 3
    88      done
    89  }
    90  
    91  kv_outage() {
    92      dur=""
    93      id=()
    94      scale_in=false
    95      scale_out=false
    96      until [ $# -eq 0 ]; do
    97          case $1 in
    98              --duration | -d) shift; dur=$1 ;;
    99              --id | -i) shift; id+=("$1") ;;
   100              --scale-out) scale_out=true ;;
   101              --kill) scale_in=true ;;
   102          esac
   103          shift
   104      done
   105  
   106      $scale_out || { 
   107          for i in "${id[@]}"; do
   108              target=$(cat "${TIKV_PIDS}_$i" | awk '{print $1}')
   109              echo "killing TiKV $target(#$i)"
   110              kill "$target" || true
   111              sleep 1
   112              kill -9 "$target" || true
   113          done 
   114      }
   115      $scale_in || $scale_out || sleep "$dur"
   116      $scale_in || {
   117          for i in "${id[@]}"; do
   118              if [ -e "${TIKV_PIDS}_$i" ]; then
   119                  TIKV_CONFIG=$(cat "${TIKV_PIDS}_$i" | awk '{print $2}')
   120              else
   121                  TIKV_CONFIG=${TIKV_CONFIG:-"tests/config/tikv.toml"}
   122              fi
   123              start_tikv "$i"
   124          done
   125          # let tikv start up completely if backup is finished before tikv restarts
   126          ensure_tikv
   127          # sometimes even though a tikv node is stopped, pd also show is_intialized in ensure_tikv
   128          sleep 1
   129      }
   130  }
   131  
   132  start_tikv() {
   133      i=$1
   134      echo "Starting TiKV($i)..."
   135      mkdir -p "$TEST_DIR/tikv${i}"
   136      bin/tikv-server \
   137          --pd "$PD_ADDR" \
   138          -A "$TIKV_ADDR$i" \
   139          --status-addr "$TIKV_STATUS_ADDR$i" \
   140          --log-file "$TEST_DIR/tikv${i}.log" \
   141          --log-level info \
   142          -C "$TIKV_CONFIG" \
   143          -s "$TEST_DIR/tikv${i}" &
   144      pid=$!
   145      echo -e "$pid\t$TIKV_CONFIG" > "${TIKV_PIDS}_${i}"
   146  }
   147  
   148  ensure_tikv() {
   149      echo "Waiting initializing TiKV..."
   150      while ! run_curl "https://$PD_ADDR/pd/api/v1/cluster/status" | grep '"is_initialized": true'; do
   151         i=$((i+1))
   152         if [ "$i" -gt 20 ]; then
   153            echo 'Failed to initialize TiKV cluster'
   154            return 1
   155         fi
   156         sleep 5
   157      done
   158  }
   159  
   160  start_tidb() {
   161      echo "Starting TiDB..."
   162      bin/tidb-server \
   163          -P 4000 \
   164          --status 10080 \
   165          --advertise-address="127.0.0.1" \
   166          --store tikv \
   167          --path "$PD_ADDR" \
   168          --config "$TIDB_CONFIG" \
   169          --log-file "$TEST_DIR/tidb.log" &
   170  
   171      echo "Verifying TiDB is started..."
   172      i=0
   173      while ! run_curl "https://$TIDB_IP:10080/status"; do
   174          i=$((i+1))
   175          if [ "$i" -gt 50 ]; then
   176              echo 'Failed to start TiDB'
   177              return 1
   178          fi
   179          sleep 3
   180      done
   181  }
   182  
   183  start_importer() {
   184      echo "Starting Importer..."
   185      bin/tikv-importer \
   186          --addr "$IMPORTER_ADDR" \
   187          --import-dir "$TEST_DIR/importer" \
   188          --log-file "$TEST_DIR/importer.log" \
   189          --config "tests/config/importer.toml" &
   190  }
   191  
   192  
   193  start_services_impl() {
   194      stop_services || true
   195      cleanup_data || true
   196  
   197      TIDB_CONFIG="tests/config/tidb.toml"
   198      TIKV_CONFIG="tests/config/tikv.toml"
   199      PD_CONFIG="tests/config/pd.toml"
   200      RUN_TIFLASH=true
   201  
   202      while [[ $# -gt 0 ]]
   203      do
   204          local key="$1"
   205  
   206          case $key in
   207              --tidb-cfg)
   208              TIDB_CONFIG="$2"
   209              shift # past argument
   210              shift # past value
   211              ;;
   212              --no-tiflash)
   213              RUN_TIFLASH=false
   214              shift # past argument
   215              ;;
   216              *)    # unknown option
   217              echo "Unknown args $1"
   218              exit 1
   219              ;;
   220          esac
   221      done
   222  
   223      rm -f "${TIKV_PIDS}*"
   224  
   225      start_pd
   226      # When using TDE, we add the master key to a file, and this master key is used to encrypt data key
   227      echo -e "3b5896b5be691006e0f71c3040a29495ddcad20b14aff61806940ebd780d3c62" > "$TEST_DIR/master-key-file"
   228      for i in $(seq $TIKV_COUNT); do
   229          start_tikv "$i"
   230      done
   231      ensure_tikv
   232      start_tidb
   233      start_importer
   234  
   235      if $RUN_TIFLASH; then
   236          start_tiflash
   237      fi
   238  
   239      i=0
   240      while ! run_curl "https://$PD_ADDR/pd/api/v1/cluster/status" | grep -q "\"is_initialized\": true"; do
   241          i=$((i+1))
   242          if [ "$i" -gt 20 ]; then
   243              echo 'Failed to bootstrap cluster'
   244              return 1
   245          fi
   246          sleep 3
   247      done
   248  }
   249  
   250  start_tiflash() {
   251      echo "Starting TiFlash..."
   252      tests/_utils/make_tiflash_config
   253      LD_LIBRARY_PATH=bin/ bin/tiflash server --config-file="$TEST_DIR/tiflash.toml" &
   254  
   255      i=0
   256      while ! run_curl https://$TIFLASH_HTTP 1>/dev/null 2>&1; do
   257          i=$((i+1))
   258          if [ "$i" -gt 20 ]; then
   259              echo "failed to start tiflash"
   260              return 1
   261          fi
   262          echo "TiFlash seems doesn't started, retrying..."
   263          sleep 3
   264      done
   265  
   266      echo "TiFlash started."
   267  }