github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/tests/lightning_checkpoint/run.sh (about)

     1  #!/bin/sh
     2  #
     3  # Copyright 2019 PingCAP, Inc.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # See the License for the specific language governing permissions and
    14  # limitations under the License.
    15  
    16  set -euE
    17  
    18  # Populate the mydumper source
    19  DBPATH="$TEST_DIR/cppk.mydump"
    20  TABLE_COUNT=9
    21  CHUNK_COUNT=50
    22  
    23  mkdir -p $DBPATH
    24  echo 'CREATE DATABASE cppk_tsr;' > "$DBPATH/cppk_tsr-schema-create.sql"
    25  INNER_QUERY='0'
    26  OUTER_QUERY='0'
    27  for i in $(seq "$TABLE_COUNT"); do
    28      case $i in
    29          1)
    30              INDICES="PRIMARY KEY"
    31              ;;
    32          2)
    33              INDICES="UNIQUE"
    34              ;;
    35          3)
    36              INDICES=", INDEX(j)"
    37              ;;
    38          4)
    39              INDICES=", PRIMARY KEY(i, j)"
    40              ;;
    41          5)
    42              INDICES=", UNIQUE KEY(j)"
    43              ;;
    44          6)
    45              INDICES=", PRIMARY KEY(j)"
    46              ;;
    47          *)
    48              INDICES=""
    49              ;;
    50      esac
    51      echo "CREATE TABLE tbl$i(i TINYINT, j INT $INDICES);" > "$DBPATH/cppk_tsr.tbl$i-schema.sql"
    52      INNER_QUERY="$INNER_QUERY, (SELECT sum(j) FROM cppk_tsr.tbl$i) as s$i"
    53      OUTER_QUERY="$OUTER_QUERY + coalesce(s$i, 0)"
    54      for j in $(seq "$CHUNK_COUNT"); do
    55          echo "INSERT INTO tbl$i VALUES ($i,${j}000),($i,${j}001);" > "$DBPATH/cppk_tsr.tbl$i.$j.sql"
    56      done
    57  done
    58  PARTIAL_IMPORT_QUERY="SELECT *, $OUTER_QUERY AS s FROM (SELECT $INNER_QUERY) _"
    59  
    60  for BACKEND in importer local; do
    61    if [ "$BACKEND" = 'local' ]; then
    62      check_cluster_version 4 0 0 'local backend' || continue
    63    fi
    64  
    65    # Set the failpoint to kill the lightning instance as soon as one table is imported
    66    # If checkpoint does work, this should only kill 9 instances of lightnings.
    67    SLOWDOWN_FAILPOINTS='github.com/pingcap/br/pkg/lightning/restore/SlowDownImport=sleep(250)'
    68    export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/br/pkg/lightning/restore/FailBeforeIndexEngineImported=return"
    69  
    70    # Start importing the tables.
    71    run_sql 'DROP DATABASE IF EXISTS cppk_tsr'
    72    run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk'
    73    run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cppk.1357924680.bak`'
    74  
    75    # panic after saving index engine checkpoint status before saving table checkpoint status
    76    set +e
    77    for i in $(seq "$TABLE_COUNT"); do
    78        echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********"
    79        run_lightning -d "$DBPATH" --backend $BACKEND --enable-checkpoint=1 2> /dev/null
    80        [ $? -ne 0 ] || exit 1
    81    done
    82    set -e
    83  
    84    export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS"
    85    set +e
    86    for i in $(seq "$TABLE_COUNT"); do
    87        echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********"
    88        run_lightning -d "$DBPATH" --backend $BACKEND --enable-checkpoint=1 2> /dev/null
    89    done
    90    set -e
    91  
    92    # Start importing the tables.
    93    run_sql 'DROP DATABASE IF EXISTS cppk_tsr'
    94    run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk'
    95    run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cppk.1357924680.bak`'
    96  
    97    export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/br/pkg/lightning/SetTaskID=return(1357924680);github.com/pingcap/br/pkg/lightning/restore/FailIfIndexEngineImported=return(1)"
    98  
    99    set +e
   100    for i in $(seq "$TABLE_COUNT"); do
   101        echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********"
   102        run_lightning -d "$DBPATH" --backend $BACKEND --enable-checkpoint=1 2> /dev/null
   103        [ $? -ne 0 ] || exit 1
   104    done
   105    set -e
   106  
   107    # After everything is done, there should be no longer new calls to ImportEngine
   108    # (and thus `kill_lightning_after_one_import` will spare this final check)
   109    echo "******** Verify checkpoint no-op ********"
   110    run_lightning -d "$DBPATH" --backend $BACKEND --enable-checkpoint=1
   111    run_sql "$PARTIAL_IMPORT_QUERY"
   112    check_contains "s: $(( (1000 * $CHUNK_COUNT + 1001) * $CHUNK_COUNT * $TABLE_COUNT ))"
   113    run_sql 'SELECT count(*) FROM `tidb_lightning_checkpoint_test_cppk.1357924680.bak`.table_v7 WHERE status >= 200'
   114    check_contains "count(*): $TABLE_COUNT"
   115  
   116    # Ensure there is no dangling open engines
   117    ls -lA "$TEST_DIR"/importer/.temp/
   118    [ -z "$(ls -A "$TEST_DIR"/importer/.temp/)" ]
   119  
   120  done