github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/tests/br_log_restore/run.sh (about)

     1  #!/bin/bash
     2  #
     3  # Copyright 2020 PingCAP, Inc.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # See the License for the specific language governing permissions and
    14  # limitations under the License.
    15  
    16  # skip log restore test because ticdc doesn't support this feature anymore.
    17  exit 0
    18  
    19  set -eux
    20  DB="$TEST_NAME"
    21  TABLE="usertable"
    22  DB_COUNT=3
    23  BUCKET="cdcs3"
    24  CDC_COUNT=3
    25  
    26  # start the s3 server
    27  export MINIO_ACCESS_KEY=brs3accesskey
    28  export MINIO_SECRET_KEY=brs3secretkey
    29  export MINIO_BROWSER=off
    30  export AWS_ACCESS_KEY_ID=$MINIO_ACCESS_KEY
    31  export AWS_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
    32  export S3_ENDPOINT=127.0.0.1:24928
    33  rm -rf "$TEST_DIR/$DB"
    34  mkdir -p "$TEST_DIR/$DB"
    35  bin/minio server --address $S3_ENDPOINT "$TEST_DIR/$DB" &
    36  i=0
    37  while ! curl -o /dev/null -s "http://$S3_ENDPOINT/"; do
    38      i=$(($i+1))
    39      if [ $i -gt 30 ]; then
    40          echo 'Failed to start minio'
    41          exit 1
    42      fi
    43      sleep 2
    44  done
    45  
    46  bin/mc config --config-dir "$TEST_DIR/$TEST_NAME" \
    47      host add minio http://$S3_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
    48  bin/mc mb --config-dir "$TEST_DIR/$TEST_NAME" minio/$BUCKET
    49  
    50  # Start cdc servers
    51  run_cdc server --pd=https://$PD_ADDR --log-file=ticdc.log --addr=0.0.0.0:18301 --advertise-addr=127.0.0.1:18301 &
    52  trap 'cat ticdc.log' ERR
    53  
    54  # TODO: remove this after TiCDC supports TiDB clustered index
    55  run_sql "set @@global.tidb_enable_clustered_index=0"
    56  # TiDB global variables cache 2 seconds
    57  sleep 2
    58  
    59  # create change feed for s3 log
    60  run_cdc cli changefeed create --pd=https://$PD_ADDR --sink-uri="s3://$BUCKET/$DB?endpoint=http://$S3_ENDPOINT" --changefeed-id="simple-replication-task"
    61  
    62  start_ts=$(run_sql "show master status;" | grep Position | awk -F ':' '{print $2}' | xargs)
    63  
    64  # Fill in the database
    65  for i in $(seq $DB_COUNT); do
    66      run_sql "CREATE DATABASE $DB${i};"
    67      go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
    68  done
    69  
    70  for i in $(seq $DB_COUNT); do
    71      row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
    72  done
    73  
    74  # test drop & create schema/table, finally only db2 has one row
    75  run_sql "create schema ${DB}_DDL1;"
    76  run_sql "create table ${DB}_DDL1.t1 (a int primary key, b varchar(10));"
    77  run_sql "insert into ${DB}_DDL1.t1 values (1, 'x');"
    78  
    79  run_sql "drop schema ${DB}_DDL1;"
    80  run_sql "create schema ${DB}_DDL1;"
    81  run_sql "create schema ${DB}_DDL2;"
    82  
    83  run_sql "create table ${DB}_DDL2.t2 (a int primary key, b varchar(10));"
    84  run_sql "insert into ${DB}_DDl2.t2 values (2, 'x');"
    85  
    86  run_sql "drop table ${DB}_DDL2.t2;"
    87  run_sql "create table ${DB}_DDL2.t2 (a int primary key, b varchar(10));"
    88  run_sql "insert into ${DB}_DDL2.t2 values (3, 'x');"
    89  run_sql "delete from ${DB}_DDL2.t2 where a = 3;"
    90  run_sql "insert into ${DB}_DDL2.t2 values (4, 'x');"
    91  
    92  end_ts=$(run_sql "show master status;" | grep Position | awk -F ':' '{print $2}' | xargs)
    93  
    94  
    95  # if we restore with ts range [start_ts, end_ts], then the below record won't be restored.
    96  run_sql "insert into ${DB}_DDL2.t2 values (5, 'x');"
    97  
    98  wait_time=0
    99  checkpoint_ts=$(run_cdc cli changefeed query -c simple-replication-task --pd=https://$PD_ADDR | jq '.status."checkpoint-ts"')
   100  while [ "$checkpoint_ts" -lt "$end_ts" ]; do
   101      echo "waiting for cdclog syncing... (checkpoint_ts = $checkpoint_ts; end_ts = $end_ts)"
   102      if [ "$wait_time" -gt 300 ]; then
   103          echo "cdc failed to sync after 300s, please check the CDC log."
   104          exit 1
   105      fi
   106      sleep 5
   107      wait_time=$(( wait_time + 5 ))
   108      checkpoint_ts=$(run_cdc cli changefeed query -c simple-replication-task --pd=https://$PD_ADDR | jq '.status."checkpoint-ts"')
   109  done
   110  
   111  # remove the change feed, because we don't want to record the drop ddl.
   112  echo "Y" | run_cdc cli unsafe reset --pd=https://$PD_ADDR
   113  
   114  for i in $(seq $DB_COUNT); do
   115      run_sql "DROP DATABASE $DB${i};"
   116  done
   117  run_sql "DROP DATABASE ${DB}_DDL1"
   118  run_sql "DROP DATABASE ${DB}_DDL2"
   119  
   120  # restore full
   121  export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/local/FailIngestMeta=return("notleader")'
   122  echo "restore start..."
   123  run_br restore cdclog -s "s3://$BUCKET/$DB" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \
   124      --log-file "restore.log" --log-level "info" --start-ts $start_ts --end-ts $end_ts
   125  
   126  for i in $(seq $DB_COUNT); do
   127      row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
   128  done
   129  
   130  fail=false
   131  row_count=$(run_sql "SELECT COUNT(*) FROM ${DB}_DDL2.t2 WHERE a=4;" | awk '/COUNT/{print $2}')
   132  if [ "$row_count" -ne "1" ]; then
   133      fail=true
   134      echo "TEST: [$TEST_NAME] fail on dml&ddl drop test."
   135  fi
   136  
   137  
   138  # record a=5 shouldn't be restore, because we set -end-ts without this record.
   139  row_count=$(run_sql "SELECT COUNT(*) FROM ${DB}_DDL2.t2 WHERE a=5;" | awk '/COUNT/{print $2}')
   140  if [ "$row_count" -ne "0" ]; then
   141      fail=true
   142      echo "TEST: [$TEST_NAME] fail on ts range test."
   143  fi
   144  
   145  export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/local/FailIngestMeta=return("epochnotmatch")'
   146  echo "restore again to restore a=5 record..."
   147  run_br restore cdclog -s "s3://$BUCKET/$DB" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \
   148      --log-file "restore.log" --log-level "info" --start-ts $end_ts
   149  
   150  # record a=5 should be restore, because we set -end-ts without this record.
   151  row_count=$(run_sql "SELECT COUNT(*) FROM ${DB}_DDL2.t2 WHERE a=5;" | awk '/COUNT/{print $2}')
   152  if [ "$row_count" -ne "1" ]; then
   153      fail=true
   154      echo "TEST: [$TEST_NAME] fail on recover ts range test."
   155  fi
   156  
   157  # record a=3 should be deleted
   158  row_count=$(run_sql "SELECT COUNT(*) FROM ${DB}_DDL2.t2 WHERE a=3;" | awk '/COUNT/{print $2}')
   159  if [ "$row_count" -ne "0" ]; then
   160      fail=true
   161      echo "TEST: [$TEST_NAME] fail on key not deleted."
   162  fi
   163  
   164  
   165  for i in $(seq $DB_COUNT); do
   166      if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then
   167          fail=true
   168          echo "TEST: [$TEST_NAME] fail on database $DB${i}"
   169      fi
   170      echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}"
   171  done
   172  
   173  if $fail; then
   174      echo "TEST: [$TEST_NAME] failed!"
   175      exit 1
   176  fi
   177  
   178  for i in $(seq $DB_COUNT); do
   179      run_sql "DROP DATABASE $DB${i};"
   180  done
   181  
   182  run_sql "DROP DATABASE ${DB}_DDL1"
   183  run_sql "DROP DATABASE ${DB}_DDL2"