github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/tests/lightning_checkpoint_chunks/run.sh (about)

     1  #!/bin/sh
     2  #
     3  # Copyright 2019 PingCAP, Inc.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # See the License for the specific language governing permissions and
    14  # limitations under the License.
    15  
    16  set -euE
    17  
    18  # Populate the mydumper source
    19  DBPATH="$TEST_DIR/cpch.mydump"
    20  CHUNK_COUNT=5
    21  ROW_COUNT=1000
    22  
    23  do_run_lightning() {
    24      run_lightning -d "$DBPATH" --enable-checkpoint=1 --config "tests/$TEST_NAME/$1.toml"
    25  }
    26  
    27  verify_checkpoint_noop() {
    28      # After everything is done, there should be no longer new calls to WriteEngine/CloseAndRecv
    29      # (and thus `kill_lightning_after_one_chunk` will spare this final check)
    30      echo "******** Verify checkpoint no-op ********"
    31      do_run_lightning config
    32      run_sql 'SELECT count(i), sum(i) FROM cpch_tsr.tbl;'
    33      check_contains "count(i): $(($ROW_COUNT*$CHUNK_COUNT))"
    34      check_contains "sum(i): $(( $ROW_COUNT*$CHUNK_COUNT*(($CHUNK_COUNT+2)*$ROW_COUNT + 1)/2 ))"
    35      run_sql 'SELECT count(*) FROM `tidb_lightning_checkpoint_test_cpch.1234567890.bak`.table_v7 WHERE status >= 200'
    36      check_contains "count(*): 1"
    37  }
    38  
    39  mkdir -p $DBPATH
    40  echo 'CREATE DATABASE cpch_tsr;' > "$DBPATH/cpch_tsr-schema-create.sql"
    41  echo 'CREATE TABLE tbl(i BIGINT UNSIGNED PRIMARY KEY);' > "$DBPATH/cpch_tsr.tbl-schema.sql"
    42  for i in $(seq "$CHUNK_COUNT"); do
    43      rm -f "$DBPATH/cpch_tsr.tbl.$i.sql"
    44      for j in $(seq "$ROW_COUNT"); do
    45          # the values run from ($ROW_COUNT + 1) to $CHUNK_COUNT*($ROW_COUNT + 1).
    46          echo "INSERT INTO tbl VALUES($(($i*$ROW_COUNT+$j)));" >> "$DBPATH/cpch_tsr.tbl.$i.sql"
    47      done
    48  done
    49  
    50  # Set the failpoint to kill the lightning instance as soon as
    51  # one file (after writing totally $ROW_COUNT rows) is imported.
    52  # If checkpoint does work, this should kill exactly $CHUNK_COUNT instances of lightnings.
    53  TASKID_FAILPOINTS="github.com/pingcap/br/pkg/lightning/SetTaskID=return(1234567890)"
    54  export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/br/pkg/lightning/restore/FailIfImportedChunk=return($ROW_COUNT)"
    55  
    56  # Start importing the tables.
    57  run_sql 'DROP DATABASE IF EXISTS cpch_tsr'
    58  run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cpch'
    59  run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cpch.1234567890.bak`'
    60  
    61  set +e
    62  for i in $(seq "$CHUNK_COUNT"); do
    63      echo "******** Importing Chunk Now (step $i/$CHUNK_COUNT) ********"
    64      do_run_lightning config 2> /dev/null
    65      [ $? -ne 0 ] || exit 1
    66  done
    67  set -e
    68  
    69  verify_checkpoint_noop
    70  
    71  # Next, test kill lightning via signal mechanism
    72  run_sql 'DROP DATABASE IF EXISTS cpch_tsr'
    73  run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cpch'
    74  run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cpch.1234567890.bak`'
    75  
    76  # Set the failpoint to kill the lightning instance as soon as one chunk is imported, via signal mechanism
    77  # If checkpoint does work, this should only kill $CHUNK_COUNT instances of lightnings.
    78  export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/br/pkg/lightning/restore/KillIfImportedChunk=return($ROW_COUNT)"
    79  
    80  for i in $(seq "$CHUNK_COUNT"); do
    81      echo "******** Importing Chunk Now (step $i/$CHUNK_COUNT) ********"
    82      do_run_lightning config
    83  done
    84  
    85  set +e
    86  i=0
    87  wait_max_time=20
    88  while [ $i -lt $wait_max_time ]; do
    89      lightning_proc=$(ps -ef|grep "[b]in/tidb-lightning\\.test.*$TEST_NAME")
    90      ret="$?"
    91      if [ "$ret" -eq 0 ]; then
    92          echo "lightning is still running: $lightning_proc"
    93          sleep 1
    94      else
    95          break
    96      fi
    97  done
    98  if [ $i -ge $wait_max_time ]; then
    99      echo "wait lightning exit failed"
   100      exit 1
   101  fi
   102  set -e
   103  
   104  verify_checkpoint_noop
   105  
   106  # Repeat, but using the file checkpoint
   107  run_sql 'DROP DATABASE IF EXISTS cpch_tsr'
   108  run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cpch'
   109  rm -f "$TEST_DIR"/cpch.pb*
   110  
   111  # Set the failpoint to kill the lightning instance as soon as one chunk is imported
   112  # If checkpoint does work, this should only kill $CHUNK_COUNT instances of lightnings.
   113  export GO_FAILPOINTS="$TASKID_FAILPOINTS;github.com/pingcap/br/pkg/lightning/restore/FailIfImportedChunk=return($ROW_COUNT)"
   114  set +e
   115  for i in $(seq "$CHUNK_COUNT"); do
   116      echo "******** Importing Chunk using File checkpoint Now (step $i/$CHUNK_COUNT) ********"
   117      do_run_lightning file 2> /dev/null
   118      [ $? -ne 0 ] || exit 1
   119  done
   120  set -e
   121  
   122  echo "******** Verify File checkpoint no-op ********"
   123  do_run_lightning file
   124  run_sql 'SELECT count(i), sum(i) FROM cpch_tsr.tbl;'
   125  check_contains "count(i): $(($ROW_COUNT*$CHUNK_COUNT))"
   126  check_contains "sum(i): $(( $ROW_COUNT*$CHUNK_COUNT*(($CHUNK_COUNT+2)*$ROW_COUNT + 1)/2 ))"
   127  [ ! -e "$TEST_DIR/cpch.pb" ]
   128  [ -e "$TEST_DIR/cpch.pb.1234567890.bak" ]