github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/tests/br_other/run.sh (about)

     1  #!/bin/sh
     2  #
     3  # Copyright 2019 PingCAP, Inc.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # See the License for the specific language governing permissions and
    14  # limitations under the License.
    15  
    16  set -eu
    17  DB="$TEST_NAME"
    18  
    19  run_sql "CREATE DATABASE $DB;"
    20  
    21  run_sql "CREATE TABLE $DB.usertable1 ( \
    22    YCSB_KEY varchar(64) NOT NULL, \
    23    FIELD0 varchar(10) DEFAULT NULL, \
    24    PRIMARY KEY (YCSB_KEY) \
    25  ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
    26  
    27  for i in `seq 1 100`
    28  do
    29  run_sql "INSERT INTO $DB.usertable1 VALUES (\"a$i\", \"bbbbbbbbbb\");"
    30  done
    31  
    32  # backup full
    33  echo "backup start..."
    34  run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB"
    35  
    36  # Test debug decode
    37  run_br -s "local://$TEST_DIR/$DB" debug decode --field "Schemas"
    38  run_br -s "local://$TEST_DIR/$DB" debug decode --field "EndVersion"
    39  # Ensure compatibility
    40  run_br -s "local://$TEST_DIR/$DB" validate decode --field "end-version"
    41  
    42  # Test redact-log and redact-info-log compalibility
    43  run_br -s "local://$TEST_DIR/$DB" debug decode --field "Schemas" --redact-log=true
    44  run_br -s "local://$TEST_DIR/$DB" debug decode --field "Schemas" --redact-info-log=true
    45  
    46  # Test validate backupmeta
    47  run_br debug backupmeta validate -s "local://$TEST_DIR/$DB"
    48  run_br debug backupmeta validate -s "local://$TEST_DIR/$DB" --offset 100
    49  
    50  # Test validate checksum
    51  run_br validate checksum -s "local://$TEST_DIR/$DB"
    52  
    53  # Test validate checksum
    54  for sst in $TEST_DIR/$DB/*.sst; do
    55      echo "corrupted!" >> $sst
    56      echo "$sst corrupted!"
    57      break
    58  done
    59  
    60  corrupted=0
    61  run_br validate checksum -s "local://$TEST_DIR/$DB" || corrupted=1
    62  if [ "$corrupted" -ne "1" ];then
    63      echo "TEST: [$TEST_NAME] failed!"
    64      exit 1
    65  fi
    66  
    67  # backup full with ratelimit = 1 to make sure this backup task won't finish quickly
    68  echo "backup start to test lock file"
    69  PPROF_PORT=6080
    70  GO_FAILPOINTS="github.com/pingcap/br/pkg/utils/determined-pprof-port=return($PPROF_PORT)" \
    71  run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB/lock" \
    72      --remove-schedulers \
    73      --ratelimit 1 \
    74      --ratelimit-unit 1 \
    75      --concurrency 4 &> $TEST_DIR/br-other-stdout.log & # It will be killed after test finish.
    76  
    77  # record last backup pid
    78  _pid=$!
    79  
    80  # give the former backup some time to write down lock file (and initialize signal listener).
    81  sleep 1
    82  pkill -10 -P $_pid
    83  echo "starting pprof..."
    84  
    85  # give the former backup some time to write down lock file (and start pprof server).
    86  sleep 1
    87  run_curl "https://localhost:$PPROF_PORT/debug/pprof/trace?seconds=1" &>/dev/null
    88  echo "pprof started..."
    89  
    90  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | grep '"disable": false'
    91  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."enable-location-replacement"' | grep "false"
    92  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."max-pending-peer-count"' | grep "2147483647"
    93  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."max-merge-region-size"' | grep -E "^0$"
    94  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."max-merge-region-keys"' | grep -E "^0$"
    95  
    96  backup_fail=0
    97  echo "another backup start expect to fail due to last backup add a lockfile"
    98  run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB/lock" --concurrency 4 || backup_fail=1
    99  if [ "$backup_fail" -ne "1" ];then
   100      echo "TEST: [$TEST_NAME] test backup lock file failed!"
   101      exit 1
   102  fi
   103  
   104  # check is there still exists scheduler not in pause.
   105  pause_schedulers=$(run_curl https://$PD_ADDR/pd/api/v1/schedulers?status="paused" | grep "scheduler" | wc -l)
   106  if [ "$pause_schedulers" -lt "3" ];then
   107    echo "TEST: [$TEST_NAME] failed because paused scheduler are not enough"
   108    exit 1
   109  fi
   110  
   111  if ps -p $_pid > /dev/null
   112  then
   113     echo "$_pid is running"
   114     # kill last backup progress (Don't send SIGKILL, or we might stuck PD in no scheduler state.)
   115     pkill -P $_pid
   116     echo "$_pid is killed @ $(date)"
   117  else
   118     echo "TEST: [$TEST_NAME] test backup lock file failed! the last backup finished"
   119     exit 1
   120  fi
   121  
   122  
   123  # make sure we won't stuck in non-scheduler state, even we send a SIGTERM to it.
   124  # give enough time to BR so it can gracefully stop.
   125  sleep 30
   126  if run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '[."schedulers-v2"][0][0]' | grep -q '"disable": true'
   127  then
   128    echo "TEST: [$TEST_NAME] failed because scheduler has been removed"
   129    exit 1
   130  fi
   131  
   132  
   133  default_pd_values='{
   134    "max-merge-region-keys": 200000,
   135    "max-merge-region-size": 20,
   136    "leader-schedule-limit": 4,
   137    "region-schedule-limit": 2048
   138  }'
   139  
   140  for key in $(echo $default_pd_values | jq 'keys[]'); do
   141    if ! run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq ".[$key]" | grep -q $(echo $default_pd_values | jq ".[$key]"); then
   142      run_curl https://$PD_ADDR/pd/api/v1/config/schedule
   143      echo "[$TEST_NAME] failed due to PD config isn't reset after restore"
   144      exit 1
   145    fi
   146  done
   147  
   148  
   149  # check is there still exists scheduler in pause.
   150  pause_schedulers=$(curl https://$PD_ADDR/pd/api/v1/schedulers?status="paused" | grep "scheduler" | wc -l)
   151   # There shouldn't be any paused schedulers since BR gracfully shutdown.
   152   if [ "$pause_schedulers" -ne "0" ];then
   153    echo "TEST: [$TEST_NAME] failed because paused scheduler has changed"
   154    exit 1
   155  fi
   156  
   157  pd_settings=6
   158  
   159  # balance-region scheduler enabled
   160  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."schedulers-v2"[] | {disable: .disable, type: ."type" | select (.=="balance-region")}' | grep '"disable": false' || ((pd_settings--))
   161  # balance-leader scheduler enabled
   162  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."schedulers-v2"[] | {disable: .disable, type: ."type" | select (.=="balance-leader")}' | grep '"disable": false' || ((pd_settings--))
   163  # hot region scheduler enabled
   164  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."schedulers-v2"[] | {disable: .disable, type: ."type" | select (.=="hot-region")}' | grep '"disable": false' || ((pd_settings--))
   165  # location replacement enabled
   166  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."enable-location-replacement"' | grep "true" || ((pd_settings--))
   167  
   168  # we need reset pd config to default
   169  # until pd has the solution to temporary set these scheduler/configs.
   170  run_br validate reset-pd-config-as-default --pd $PD_ADDR
   171  
   172  # max-merge-region-size set to default 20
   173  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."max-merge-region-size"' | grep "20" || ((pd_settings--))
   174  
   175  # max-merge-region-keys set to default 200000
   176  run_curl https://$PD_ADDR/pd/api/v1/config/schedule | jq '."max-merge-region-keys"' | grep "200000" || ((pd_settings--))
   177  
   178  if [ "$pd_settings" -ne "6" ];then
   179      echo "TEST: [$TEST_NAME] test validate reset pd config failed!"
   180      exit 1
   181  fi
   182  
   183  
   184  # Test version
   185  run_br --version
   186  run_br -V
   187  
   188  run_sql "DROP DATABASE $DB;"