github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/plugins/plugin_test.sh (about)

     1  #!/bin/bash
     2  set -e
     3  set -o pipefail
     4  
     5  plugin=$1
     6  plugin_config=$2
     7  secondary_plugin_config=$3
     8  MINIMUM_API_VERSION="0.3.0"
     9  
    10  # ----------------------------------------------
    11  # Test suite setup
    12  # This will put small amounts of data in the
    13  # plugin destination location
    14  # ----------------------------------------------
    15  if [ $# -lt 2 ] || [ $# -gt 3 ]
    16    then
    17      echo "Usage: plugin_test.sh [path_to_executable] [plugin_config] [optional_config_for_secondary_destination]"
    18      exit 1
    19  fi
    20  
    21  if [[ "$plugin_config" != /* ]] ; then
    22      echo "Must provide an absolute path to the plugin config"
    23      exit 1
    24  fi
    25  
    26  # This should be time_second=$(date +"%Y%m%d%H%M%S") but concurrent
    27  # runs require randomness to ensure they don't collide with each other
    28  # in writing/deleting backups. We ensure there are 14 characters
    29  # through the expression below.
    30  time_second=$(expr 99999999999999 - $(od -vAn -N5 -tu < /dev/urandom | tr -d ' \n'))
    31  current_date=$(echo $time_second | cut -c 1-8)
    32  
    33  testdir="/tmp/testseg/backups/${current_date}/${time_second}"
    34  testfile="$testdir/testfile_$time_second.txt"
    35  testdata="$testdir/testdata_$time_second.txt"
    36  test_no_data="$testdir/test_no_data_$time_second.txt"
    37  testdatasmall="$testdir/testdatasmall_$time_second.txt"
    38  testdatalarge="$testdir/testdatalarge_$time_second.txt"
    39  
    40  logdir="/tmp/test_bench_logs"
    41  
    42  text="this is some text"
    43  data=`LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 1000 ; echo`
    44  data_large=`LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 1000000 ; echo`
    45  mkdir -p $testdir
    46  mkdir -p $logdir
    47  echo $text > $testfile
    48  
    49  # ----------------------------------------------
    50  # Cleanup functions
    51  # ----------------------------------------------
    52  cleanup_test_dir() {
    53    if [ $# -ne 1 ]
    54    then
    55      echo "Must call cleanup_test_dir with only 1 argument"
    56      exit 1
    57    fi
    58  
    59    testdir_to_clean=$1
    60  
    61    $plugin cleanup_plugin_for_backup $plugin_config $testdir_to_clean coordinator \"-1\"
    62    $plugin cleanup_plugin_for_backup $plugin_config $testdir_to_clean segment_host
    63    $plugin cleanup_plugin_for_backup $plugin_config $testdir_to_clean segment \"0\"
    64    echo "[PASSED - CLEANUP] cleanup_plugin_for_backup"
    65  
    66    $plugin cleanup_plugin_for_restore $plugin_config $testdir_to_clean coordinator \"-1\"
    67    $plugin cleanup_plugin_for_restore $plugin_config $testdir_to_clean segment_host
    68    $plugin cleanup_plugin_for_restore $plugin_config $testdir_to_clean segment \"0\"
    69    echo "[PASSED - CLEANUP] cleanup_plugin_for_restore"
    70  }
    71  
    72  echo "# ----------------------------------------------"
    73  echo "# Starting gpbackup plugin tests"
    74  echo "# ----------------------------------------------"
    75  
    76  # ----------------------------------------------
    77  # Check API version
    78  # ----------------------------------------------
    79  
    80  echo "[RUNNING] plugin_api_version"
    81  api_version=`$plugin plugin_api_version`
    82  # `awk` call returns 1 for true, 0 for false (contrary to bash logic)
    83  if (( 0 == $(echo "$MINIMUM_API_VERSION $api_version" | awk '{print ($1 <= $2)}') )) ; then
    84    echo "Plugin API version is less than the minimum supported version $MINIMUM_API_VERSION"
    85    exit 1
    86  fi
    87  echo "[PASSED] plugin_api_version"
    88  
    89  echo "[RUNNING] --version"
    90  native_version=`$plugin --version`
    91  echo "$native_version" | grep --regexp '.* version .*' > /dev/null 2>&1
    92  if [[ ! $? -eq 0 ]]; then
    93    echo "Plugin --version is not in expected format of <plugin name> version <version>"
    94    exit 1
    95  fi
    96  echo "[PASSED] --version"
    97  
    98  # ----------------------------------------------
    99  # Setup and Backup/Restore file functions
   100  # ----------------------------------------------
   101  
   102  echo "[RUNNING] setup_plugin_for_backup on coordinator"
   103  $plugin setup_plugin_for_backup $plugin_config $testdir coordinator \"-1\"
   104  echo "[RUNNING] setup_plugin_for_backup on segment_host"
   105  $plugin setup_plugin_for_backup $plugin_config $testdir segment_host
   106  echo "[RUNNING] setup_plugin_for_backup on segment 0"
   107  $plugin setup_plugin_for_backup $plugin_config $testdir segment \"0\"
   108  
   109  echo "[RUNNING] backup_file"
   110  $plugin backup_file $plugin_config $testfile
   111  # plugins should leave copies of the files locally when they run backup_file
   112  test -f $testfile
   113  
   114  echo "[RUNNING] setup_plugin_for_restore on coordinator"
   115  $plugin setup_plugin_for_restore $plugin_config $testdir coordinator \"-1\"
   116  echo "[RUNNING] setup_plugin_for_restore on segment_host"
   117  $plugin setup_plugin_for_restore $plugin_config $testdir segment_host
   118  echo "[RUNNING] setup_plugin_for_restore on segment 0"
   119  $plugin setup_plugin_for_restore $plugin_config $testdir segment \"0\"
   120  
   121  echo "[RUNNING] restore_file"
   122  rm $testfile
   123  $plugin restore_file $plugin_config $testfile
   124  output=`cat $testfile`
   125  if [ "$output" != "$text" ]; then
   126    echo "Failed to backup and restore file using plugin"
   127    exit 1
   128  fi
   129  
   130  echo "[RUNNING] attempting to restore_file of non-existent file should fail"
   131  set +e
   132  $plugin restore_file $plugin_config "$testdir/there_is_no_file_to_restore" > /dev/null 2>&1
   133  nonexist_file_restore=$(echo $?)
   134  set -e
   135  if [ "$nonexist_file_restore" == "0" ]; then
   136    echo "Failed, when trying to restore a file that does not exist, should have error"
   137    exit 1
   138  fi
   139  
   140  
   141  if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then
   142    rm $testfile
   143    echo "[RUNNING] restore_file (from secondary destination)"
   144    $plugin restore_file $secondary_plugin_config $testfile
   145    output=`cat $testfile`
   146    if [ "$output" != "$text" ]; then
   147      echo "Failed to backup and restore file using plugin from secondary destination"
   148      exit 1
   149    fi
   150  fi
   151  echo "[PASSED] setup_plugin_for_backup"
   152  echo "[PASSED] backup_file"
   153  echo "[PASSED] setup_plugin_for_restore"
   154  echo "[PASSED] restore_file"
   155  cleanup_test_dir $testdir
   156  
   157  # ----------------------------------------------
   158  # Backup/Restore data functions
   159  # ----------------------------------------------
   160  
   161  echo "[RUNNING] backup_data"
   162  echo $data | $plugin backup_data $plugin_config $testdata
   163  echo "[RUNNING] restore_data"
   164  output=`$plugin restore_data $plugin_config $testdata`
   165  
   166  if [ "$output" != "$data" ]; then
   167    echo "Failed to backup and restore data using plugin"
   168    exit 1
   169  fi
   170  
   171  if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then
   172    echo "[RUNNING] restore_data (from secondary destination)"
   173    output=`$plugin restore_data $secondary_plugin_config $testdata`
   174  
   175    if [ "$output" != "$data" ]; then
   176      echo "Failed to backup and restore data using plugin"
   177      exit 1
   178    fi
   179  fi
   180  echo "[PASSED] backup_data"
   181  echo "[PASSED] restore_data"
   182  cleanup_test_dir $testdir
   183  
   184  echo "[RUNNING] backup_data with no data"
   185  echo -n "" | $plugin backup_data $plugin_config $test_no_data
   186  echo "[RUNNING] restore_data with no data"
   187  output=`$plugin restore_data $plugin_config $test_no_data`
   188  
   189  if [ "$output" != "" ]; then
   190    echo "Failed to backup and restore data using plugin"
   191    exit 1
   192  fi
   193  
   194  if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then
   195    echo "[RUNNING] restore_data with no data (from secondary destination)"
   196    output=`$plugin restore_data $secondary_plugin_config $test_no_data`
   197  
   198    if [ "$output" != "" ]; then
   199      echo "Failed to backup and restore data using plugin"
   200      exit 1
   201    fi
   202  fi
   203  echo "[PASSED] backup_data with no data"
   204  echo "[PASSED] restore_data with no data"
   205  cleanup_test_dir $testdir
   206  
   207  # ----------------------------------------------
   208  # Restore subset data functions
   209  # ----------------------------------------------
   210  if [[ "$plugin" == *gpbackup_ddboost_plugin ]]; then
   211    echo "[RUNNING] backup_data of small data for subset restore"
   212    echo $data | $plugin backup_data $plugin_config $testdatasmall
   213    echo "1 3 10" > "$testdir/offsets"
   214    echo "[RUNNING] restore_data_subset"
   215    echo $plugin restore_data_subset $plugin_config $testdatasmall "$testdir/offsets"
   216    output=`$plugin restore_data_subset $plugin_config $testdatasmall "$testdir/offsets"`
   217    data_subset=$(echo $data | cut -c4-10)
   218    if [ "$output" != "$data_subset" ]; then
   219      echo "Failure in restore_data_subset of small data using plugin"
   220      exit 1
   221    fi
   222    echo "[PASSED] restore_data_subset with small data"
   223  
   224    echo "[RUNNING] backup_data of large data for subset restore"
   225    echo $data_large | $plugin backup_data $plugin_config $testdatalarge
   226    echo "1 900000 900001" > "$testdir/offsets"
   227    echo "[RUNNING] restore_data_subset"
   228    output=`$plugin restore_data_subset $plugin_config $testdatalarge "$testdir/offsets"`
   229    data_part=$(echo $data_large | cut -c900001-900001)
   230    if [ "$output" != "$data_part" ]; then
   231      echo "Failure restore_data_subset of one partition from large data"
   232      exit 1
   233    fi
   234    echo "[PASSED] restore_data_subset of one partition from large data"
   235  
   236    echo "2 0 700000 900000 900001" > "$testdir/offsets"
   237    echo "[RUNNING] restore_data_subset"
   238    output=`$plugin restore_data_subset $plugin_config $testdatalarge "$testdir/offsets"`
   239    data_part1=$(echo $data_large | cut -c1-700000)
   240    data_part2=$(echo $data_large | cut -c900001-900001)
   241    if [ "$output" != "$data_part1$data_part2" ]; then
   242      echo "Failure restore_data_subset of two partitions from large data"
   243      exit 1
   244    fi
   245    echo "[PASSED] restore_data_subset of two partitions from large data"
   246    cleanup_test_dir $testdir
   247  fi
   248  
   249  # ----------------------------------------------
   250  # Delete backup directory function
   251  # ----------------------------------------------
   252  time_second_for_del=$(expr 99999999999999 - $(od -vAn -N5 -tu < /dev/urandom | tr -d ' \n'))
   253  current_date_for_del=$(echo $time_second_for_del | cut -c 1-8)
   254  
   255  time_second_for_del2=$(expr $time_second_for_del + 1)
   256  current_date_for_del2=$(echo $time_second_for_del2 | cut -c 1-8)
   257  
   258  testdir_for_del="/tmp/testseg/backups/${current_date_for_del}/${time_second_for_del}"
   259  testdata_for_del="$testdir_for_del/testdata_$time_second_for_del.txt"
   260  testfile_for_del="$testdir_for_del/testfile_$time_second_for_del.txt"
   261  
   262  testdir_for_del2="/tmp/testseg/backups/${current_date_for_del2}/${time_second_for_del2}"
   263  testdata_for_del2="$testdir_for_del2/testdata_$time_second_for_del2.txt"
   264  testfile_for_del2="$testdir_for_del2/testfile_$time_second_for_del2.txt"
   265  
   266  mkdir -p $testdir_for_del
   267  mkdir -p $testdir_for_del2
   268  
   269  echo $text > $testfile_for_del
   270  echo $text > $testfile_for_del2
   271  
   272  echo "[RUNNING] delete_backup"
   273  
   274  # first backup
   275  $plugin setup_plugin_for_backup $plugin_config $testdir_for_del coordinator \"-1\"
   276  $plugin setup_plugin_for_backup $plugin_config $testdir_for_del segment_host
   277  $plugin setup_plugin_for_backup $plugin_config $testdir_for_del segment \"0\"
   278  
   279  echo $data | $plugin backup_data $plugin_config $testdata_for_del
   280  $plugin backup_file $plugin_config $testfile_for_del
   281  
   282  # second backup
   283  $plugin setup_plugin_for_backup $plugin_config $testdir_for_del2 coordinator \"-1\"
   284  $plugin setup_plugin_for_backup $plugin_config $testdir_for_del2 segment_host
   285  $plugin setup_plugin_for_backup $plugin_config $testdir_for_del2 segment \"0\"
   286  
   287  echo $data | $plugin backup_data $plugin_config $testdata_for_del2
   288  $plugin backup_file $plugin_config $testfile_for_del2
   289  
   290  # here's the real test: can we delete and confirm deletion, while sibling backup remains?
   291  $plugin delete_backup $plugin_config $time_second_for_del
   292  
   293  set +e
   294  # test deletion from local server
   295  output_data_restore=$($plugin restore_data $plugin_config $testdata_for_del 2>/dev/null)
   296  retval_data_restore=$(echo $?)
   297  if [ "${output_data_restore}" = "${data}"  ] || [ "$retval_data_restore" = "0" ] ; then
   298    echo "Failed to delete backup data from local server using plugin"
   299    exit 1
   300  fi
   301  $plugin restore_file $plugin_config $testfile_for_del 2>/dev/null
   302  retval_file_restore=$(echo $?)
   303  if [ "$retval_file_restore" = "0" ] ; then
   304    echo "Failed to delete backup file from local server using plugin"
   305    exit 1
   306  fi
   307  
   308  # test deletion from remote server
   309  if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then
   310    output_data_restore=$($plugin restore_data $secondary_plugin_config $testdata_for_del 2>/dev/null)
   311    retval_data_restore=$(echo $?)
   312    if [ "${output_data_restore}" = "${data}"  ] || [ "$retval_data_restore" = "0" ] ; then
   313      echo "Failed to delete backup data from remote server using plugin"
   314      exit 1
   315    fi
   316    $plugin restore_file $secondary_plugin_config $testfile_for_del 2>/dev/null
   317    retval_file_restore=$(echo $?)
   318    if [ "$retval_file_restore" = "0" ] ; then
   319      echo "Failed to delete backup file from remote server using plugin"
   320      exit 1
   321    fi
   322  fi
   323  
   324  # confirm sibling backup remains
   325  log_sibling_restore="$logdir/sib_restore"
   326  output_data_restore=$($plugin restore_data $plugin_config $testdata_for_del2 2>$log_sibling_restore)
   327  retval_data_restore=$(echo $?)
   328  if [ "${output_data_restore}" != "${data}"  ] || [ "$retval_data_restore" != "0" ] ; then
   329    echo
   330    cat $log_sibling_restore
   331    echo
   332    echo "Failed to leave behind a sibling backup after a sibling delete from local server using plugin"
   333    exit 1
   334  fi
   335  
   336  set -e
   337  echo "[PASSED] delete_backup"
   338  cleanup_test_dir $testdir_for_del
   339  
   340  # ----------------------------------------------
   341  # Replicate backup function
   342  # ----------------------------------------------
   343  if [[ "$plugin" == *gpbackup_ddboost_plugin ]] && [[ "$plugin_config" == *ddboost_config.yaml ]]; then
   344      time_second_for_repl=$(expr 99999999999999 - $(od -vAn -N5 -tu < /dev/urandom | tr -d ' \n'))
   345      current_date_for_repl=$(echo $time_second_for_repl | cut -c 1-8)
   346  
   347      testdir_for_repl="/tmp/testseg/backups/${current_date_for_repl}/${time_second_for_repl}"
   348      testdata_for_repl="$testdir_for_repl/testdata_$time_second_for_repl.txt"
   349  
   350      mkdir -p $testdir_for_repl
   351  
   352      echo "[RUNNING] backup_data without replication to replicate later"
   353      $plugin setup_plugin_for_backup $plugin_config $testdir_for_repl coordinator \"-1\"
   354      $plugin setup_plugin_for_backup $plugin_config $testdir_for_repl segment_host
   355      $plugin setup_plugin_for_backup $plugin_config $testdir_for_repl segment \"0\"
   356  
   357      echo $data | $plugin backup_data $plugin_config $testdata_for_repl
   358      echo "[PASSED] backup_data without replication to replicate later"
   359  
   360      echo "[RUNNING] replicate_backup"
   361      $plugin replicate_backup $plugin_config $time_second_for_repl
   362  
   363      set +e
   364      echo "[RUNNING] replicate_backup second time to verify error"
   365      output=`$plugin replicate_backup $plugin_config $time_second_for_repl 2>&1`
   366      echo "replicate backup second time, output: $output"
   367      if [[ ! "$output" == *"DDBoost open file in exclusive mode failed on remote"* ]]; then
   368          echo "Error doesn't match when trying to replicate already replicated backup"
   369          exit 1
   370      fi
   371      echo "[PASSED] replicate_backup second time to verify error"
   372      set -e
   373  
   374      if [ -n "$secondary_plugin_config" ]; then
   375          echo "[RUNNING] restore_data (from secondary destination)"
   376          output=`$plugin restore_data $secondary_plugin_config $testdata_for_repl`
   377  
   378          if [ "$output" != "$data" ]; then
   379              echo "Failed to replicate backup using plugin"
   380              exit 1
   381          fi
   382      fi
   383      echo "[PASSED] replicate backup"
   384      cleanup_test_dir $testdir_for_repl
   385  fi
   386  
   387  set +e
   388  echo "[RUNNING] fails with unknown command"
   389  $plugin unknown_command &> /dev/null
   390  if [ $? -eq 0 ] ; then
   391    echo "Plugin should exit non-zero when provided an unknown command"
   392    exit 1
   393  fi
   394  echo "[PASSED] fails with unknown command"
   395  set -e
   396  
   397  # ----------------------------------------------
   398  # Run test gpbackup and gprestore with plugin
   399  # ----------------------------------------------
   400  #gpbackup --dbname $test_db --plugin-config $plugin_config $further_options > $log_file
   401  
   402  test_backup_and_restore_with_plugin() {
   403      flags=$1
   404      restore_filter=$2
   405      test_db=plugin_test_db
   406      log_file="$logdir/plugin_test_log_file"
   407  
   408      psql -X -d postgres -qc "DROP DATABASE IF EXISTS $test_db" 2>/dev/null
   409      createdb $test_db
   410      psql -X -d $test_db -qc "CREATE TABLE test1(i int) DISTRIBUTED RANDOMLY; INSERT INTO test1 select generate_series(1,50000)"
   411      if [ "$restore_filter" == "restore-filter" ] ; then
   412        psql -X -d $test_db -qc "CREATE TABLE test2(i int) DISTRIBUTED RANDOMLY; INSERT INTO test2 VALUES(3333)"
   413        flags_restore="--include-table public.test2"
   414      fi
   415  
   416      set +e
   417      # save the encrypt key file, if it exists
   418      if [ -f "$COORDINATOR_DATA_DIRECTORY/.encrypt" ] ; then
   419          mv $COORDINATOR_DATA_DIRECTORY/.encrypt /tmp/.encrypt_saved
   420      fi
   421      echo "gpbackup_ddboost_plugin: 66706c6c6e677a6965796f68343365303133336f6c73366b316868326764" > $COORDINATOR_DATA_DIRECTORY/.encrypt
   422  
   423      echo "[RUNNING] gpbackup with test database (using ${flags} ${flags_restore})"
   424      gpbackup --dbname $test_db --plugin-config $plugin_config $flags &> $log_file
   425      if [ ! $? -eq 0 ]; then
   426          echo
   427          cat $log_file
   428          echo
   429          echo "gpbackup failed. Check gpbackup log file in ~/gpAdminLogs for details."
   430          exit 1
   431      fi
   432      timestamp=`head -10 $log_file | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}"`
   433      dropdb $test_db
   434  
   435      echo "[RUNNING] gprestore with test database"
   436      gprestore --timestamp $timestamp --plugin-config $plugin_config --create-db $flags_restore &> $log_file
   437      if [ ! $? -eq 0 ]; then
   438          echo
   439          cat $log_file
   440          echo
   441          echo "gprestore failed. Check gprestore log file in ~/gpAdminLogs for details."
   442          exit 1
   443      fi
   444  
   445      if [ "$restore_filter" == "restore-filter" ] ; then
   446        result=`psql -X -d $test_db -tc "SELECT table_name FROM information_schema.tables WHERE table_schema='public'" | xargs`
   447        if [ "$result" == *"test1"* ]; then
   448            echo "Expected relation test1 to not exist"
   449            exit 1
   450        fi
   451        result=`psql -X -d $test_db -tc "SELECT * FROM test2" | xargs`
   452        if [ "$flags" != "--metadata-only" ] && [ "$result" != "3333" ]; then
   453            echo "Expected relation test2 value: 3333, got %result"
   454            exit 1
   455        fi
   456      else
   457        result=`psql -X -d $test_db -tc "SELECT count(*) FROM test1" | xargs`
   458        if [ "$flags" != "--metadata-only" ] && [ "$result" != "50000" ]; then
   459            echo "Expected to restore 50000 rows, got $result"
   460            exit 1
   461        fi
   462      fi
   463  
   464      if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then
   465          dropdb $test_db
   466          echo "[RUNNING] gprestore with test database from secondary destination"
   467          gprestore --timestamp $timestamp --plugin-config $secondary_plugin_config --create-db &> $log_file
   468          if [ ! $? -eq 0 ]; then
   469              echo
   470              cat $log_file
   471              echo
   472              echo "gprestore from secondary destination failed. Check gprestore log file in ~/gpAdminLogs for details."
   473              exit 1
   474          fi
   475          result=`psql -X -d $test_db -tc "SELECT count(*) FROM test1" | xargs`
   476          if [ "$flags" != "--metadata-only" ] && [ "$result" != "50000" ]; then
   477            echo "Expected to restore 50000 rows, got $result"
   478            exit 1
   479          fi
   480      fi
   481      # replace the encrypt key file to its proper location
   482      if [ -f "/tmp/.encrypt_saved" ] ; then
   483          mv /tmp/.encrypt_saved $COORDINATOR_DATA_DIRECTORY/.encrypt
   484      fi
   485      set -e
   486      echo "[PASSED] gpbackup and gprestore (using ${flags})"
   487  }
   488  
   489  test_backup_and_restore_with_plugin "--single-data-file --no-compression --copy-queue-size 4" "--copy-queue-size 4"
   490  test_backup_and_restore_with_plugin "--no-compression --single-data-file"
   491  test_backup_and_restore_with_plugin "--no-compression"
   492  test_backup_and_restore_with_plugin "--metadata-only"
   493  test_backup_and_restore_with_plugin "--no-compression --single-data-file" "restore-filter"
   494  
   495  
   496  # ----------------------------------------------
   497  # Cleanup test artifacts
   498  # ----------------------------------------------
   499  echo "Cleaning up leftover test artifacts"
   500  
   501  dropdb $test_db
   502  rm -r $logdir
   503  rm -r /tmp/testseg
   504  
   505  if (( 1 == $(echo "0.4.0 $api_version" | awk '{print ($1 > $2)}') )) ; then
   506    echo "[SKIPPING] cleanup of uploaded test artifacts using plugins (only compatible with version >= 0.4.0)"
   507  else
   508    $plugin delete_backup $plugin_config $time_second
   509  fi
   510  
   511  echo "# ----------------------------------------------"
   512  echo "# Finished gpbackup plugin tests"
   513  echo "# ----------------------------------------------"