github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/ci/scripts/scale_perf_tests.bash (about)

     1  #!/bin/bash
     2  
     3  set -ex
     4  
     5  # setup cluster and install gpbackup tools using gppkg
     6  ccp_src/scripts/setup_ssh_to_cluster.sh
     7  out=$(ssh -t cdw 'source env.sh && psql postgres -c "select version();"')
     8  GPDB_VERSION=$(echo ${out} | sed -n 's/.*Greenplum Database \([0-9]\).*/\1/p')
     9  mkdir -p /tmp/untarred
    10  tar -xzf gppkgs/gpbackup-gppkgs.tar.gz -C /tmp/untarred
    11  scp /tmp/untarred/gpbackup_tools*gp${GPDB_VERSION}*${OS}*.gppkg cdw:/home/gpadmin
    12  scp gpbackup/ci/scripts/analyze_run.py cdw:/home/gpadmin/analyze_run.py
    13  scp gpbackup/ci/scale/sql/scaletestdb_bigschema_ddl.sql cdw:/home/gpadmin/scaletestdb_bigschema_ddl.sql
    14  scp gpbackup/ci/scale/sql/scaletestdb_wideschema_ddl.sql cdw:/home/gpadmin/scaletestdb_wideschema_ddl.sql
    15  scp gpbackup/ci/scale/sql/etl_job.sql cdw:/home/gpadmin/etl_job.sql
    16  scp gpbackup/ci/scale/sql/pull_rowcount.sql cdw:/home/gpadmin/pull_rowcount.sql
    17  scp gpbackup/ci/scale/sql/valid_metadata.sql cdw:/home/gpadmin/valid_metadata.sql
    18  scp -r gpbackup/ci/scale/gpload_yaml cdw:/home/gpadmin/gpload_yaml
    19  
    20  set +x
    21  printf "%s" "${GOOGLE_CREDENTIALS}" > "/tmp/keyfile.json"
    22  set -x
    23  
    24  scp /tmp/keyfile.json cdw:/home/gpadmin/keyfile.json && rm -f /tmp/keyfile.json
    25  
    26  cat <<SCRIPT > /tmp/run_tests.bash
    27  #!/bin/bash
    28  
    29  source env.sh
    30  # set format for logging
    31  export TIMEFORMAT="TEST RUNTIME: %E"
    32  export RESULTS_LOG_FILE=${RESULTS_LOG_FILE}
    33  
    34  # set parameters for reference time DB
    35  export RESULTS_DATABASE_HOST=${RESULTS_DATABASE_HOST}
    36  export RESULTS_DATABASE_USER=${RESULTS_DATABASE_USER}
    37  export RESULTS_DATABASE_NAME=${RESULTS_DATABASE_NAME}
    38  export RESULTS_DATABASE_PASSWORD=${RESULTS_DATABASE_PASSWORD}
    39  
    40  # set GCS credentials and mount gcs bucket with gcsfuse
    41  export GOOGLE_APPLICATION_CREDENTIALS=/home/gpadmin/keyfile.json
    42  gcloud auth activate-service-account --key-file=/home/gpadmin/keyfile.json
    43  rm -rf /home/gpadmin/bucket && mkdir /home/gpadmin/bucket
    44  gcsfuse --implicit-dirs dp-gpbackup-scale-test-data /home/gpadmin/bucket
    45  
    46  
    47  # Double the vmem protect limit default on the coordinator segment to
    48  # prevent query cancels on large table creations (e.g. scale_db1.sql)
    49  gpconfig -c gp_vmem_protect_limit -v 16384 --masteronly
    50  gpconfig -c client_min_messages -v error
    51  gpstop -air
    52  
    53  # only install if not installed already
    54  is_installed_output=\$(source env.sh; gppkg -q gpbackup*gp*.gppkg)
    55  set +e
    56  echo \$is_installed_output | grep 'is installed'
    57  if [ \$? -ne 0 ] ; then
    58    set -e
    59    gppkg -i gpbackup*gp*.gppkg
    60  fi
    61  set -e
    62  
    63  # capture installed versions for later storage in run stats
    64  gpstart --version > /home/gpadmin/gpversion.txt
    65  gpbackup --version > /home/gpadmin/gpbversion.txt
    66  export GPDB_VERSION=\$(cat /home/gpadmin/gpversion.txt)
    67  export GPB_VERSION=\$(cat /home/gpadmin/gpbversion.txt)
    68  
    69  ### Data scale tests ###
    70  echo "## Loading data into database for scale tests ##"
    71  createdb scaletestdb
    72  psql -d scaletestdb -q -f scaletestdb_bigschema_ddl.sql
    73  gpload -f /home/gpadmin/gpload_yaml/customer.yml
    74  gpload -f /home/gpadmin/gpload_yaml/lineitem.yml
    75  gpload -f /home/gpadmin/gpload_yaml/orders.yml
    76  gpload -f /home/gpadmin/gpload_yaml/orders_2.yml
    77  gpload -f /home/gpadmin/gpload_yaml/orders_3.yml
    78  gpload -f /home/gpadmin/gpload_yaml/nation.yml
    79  gpload -f /home/gpadmin/gpload_yaml/part.yml
    80  gpload -f /home/gpadmin/gpload_yaml/partsupp.yml
    81  gpload -f /home/gpadmin/gpload_yaml/region.yml
    82  gpload -f /home/gpadmin/gpload_yaml/supplier.yml
    83  
    84  # clean out credentials after data is loaded
    85  rm -f /home/gpadmin/keyfile.json
    86  
    87  echo "## Capturing row counts for comparison ##"
    88  psql -d scaletestdb -f /home/gpadmin/pull_rowcount.sql -o /home/gpadmin/rowcounts_orig.txt
    89  
    90  
    91  #####################################################################
    92  ##################################################################### 
    93  echo "## Performing single-data-file, --no-compression, --copy-queue-size 8 backup test ##"
    94  # BACKUP
    95  rm -f $RESULTS_LOG_FILE
    96  (time gpbackup --dbname scaletestdb --include-schema big --backup-dir /data/gpdata/ --single-data-file --no-compression --copy-queue-size 8) > $RESULTS_LOG_FILE 2>&1
    97  timestamp=\$(head -10 "\$RESULTS_LOG_FILE" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
    98  echo "gpb_single_data_file_copy_q8 timestamp backed up: \$timestamp"
    99  
   100  # conduct runtime analysis
   101  python /home/gpadmin/analyze_run.py gpb_single_data_file_copy_q8
   102  #####################################################################
   103  
   104  #####################################################################
   105  echo "## Performing single-data-file, --no-compression, --copy-queue-size 8 restore test ##"
   106  # RESTORE
   107  rm -f $RESULTS_LOG_FILE
   108  (time gprestore --timestamp "\$timestamp" --include-schema big --backup-dir /data/gpdata/ --create-db --redirect-db copyqueuerestore8 --copy-queue-size 8) > $RESULTS_LOG_FILE 2>&1
   109  echo "gpr_single_data_file_copy_q8 timestamp restored: \$timestamp"
   110  
   111  # compare round-trip row counts
   112  psql -d copyqueuerestore8 -f /home/gpadmin/pull_rowcount.sql -o /home/gpadmin/rowcounts_gpr_single_data_file_copy_q8.txt
   113  ROWCOUNTS_DIFF=\$(diff -w /home/gpadmin/rowcounts_orig.txt /home/gpadmin/rowcounts_gpr_single_data_file_copy_q8.txt)
   114  if [ "\$ROWCOUNTS_DIFF" != "" ] 
   115  then
   116    echo "Failed result from gpr_single_data_file_copy_q8 -- mismatched row counts.  Exiting early with failure code."
   117    fusermount -u /home/gpadmin/bucket
   118    exit 1
   119  fi
   120  
   121  # conduct runtime analysis
   122  python /home/gpadmin/analyze_run.py gpr_single_data_file_copy_q8
   123  
   124  # clean out redirected database before proceeding further
   125  yes y | gpbackup_manager delete-backup "\$timestamp"
   126  dropdb copyqueuerestore8
   127  #####################################################################
   128  #####################################################################
   129  
   130  #####################################################################
   131  ##################################################################### 
   132  echo "## Performing backup for data scale test ##"
   133  # BACKUP
   134  rm -f $RESULTS_LOG_FILE
   135  (time gpbackup --dbname scaletestdb --include-schema big --backup-dir /data/gpdata/) > $RESULTS_LOG_FILE 2>&1
   136  timestamp=\$(head -10 "\$RESULTS_LOG_FILE" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   137  echo "gpb_scale_multi_data_file timestamp backed up: \$timestamp"
   138  
   139  # conduct runtime analysis
   140  python /home/gpadmin/analyze_run.py gpb_scale_multi_data_file
   141  #####################################################################
   142  
   143  #####################################################################
   144  echo "## Performing restore for data scale test ##"
   145  # RESTORE
   146  rm -f $RESULTS_LOG_FILE
   147  (time gprestore --timestamp "\$timestamp" --include-schema big --backup-dir /data/gpdata/ --create-db --redirect-db scalemultifile --jobs=4) > $RESULTS_LOG_FILE 2>&1
   148  echo "gpr_scale_multi_data_file timestamp restored: \$timestamp"
   149  
   150  # compare round-trip row counts
   151  psql -d scalemultifile -f /home/gpadmin/pull_rowcount.sql -o /home/gpadmin/rowcounts_gpr_scale_multi_data_file.txt
   152  ROWCOUNTS_DIFF=\$(diff -w /home/gpadmin/rowcounts_orig.txt /home/gpadmin/rowcounts_gpr_scale_multi_data_file.txt)
   153  if [ "\$ROWCOUNTS_DIFF" != "" ] 
   154  then
   155    echo "Failed result from gpr_scale_multi_data_file -- mismatched row counts.  Exiting early with failure code."
   156    fusermount -u /home/gpadmin/bucket
   157    exit 1
   158  fi
   159  
   160  # conduct runtime analysis
   161  python /home/gpadmin/analyze_run.py gpr_scale_multi_data_file
   162  
   163  # clean out redirected database before proceeding further
   164  yes y | gpbackup_manager delete-backup "\$timestamp"
   165  dropdb scalemultifile
   166  #####################################################################
   167  #####################################################################
   168  
   169  #####################################################################
   170  ##################################################################### 
   171  echo "## Performing backup for data scale test with zstd ##"
   172  # BACKUP
   173  rm -f $RESULTS_LOG_FILE
   174  (time gpbackup --dbname scaletestdb --include-schema big --backup-dir /data/gpdata/ --compression-type zstd) > $RESULTS_LOG_FILE 2>&1
   175  timestamp=\$(head -10 "\$RESULTS_LOG_FILE" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   176  echo "gpb_scale_multi_data_file_zstd timestamp backed up: \$timestamp"
   177  
   178  # conduct runtime analysis
   179  python /home/gpadmin/analyze_run.py gpb_scale_multi_data_file_zstd
   180  #####################################################################
   181  
   182  #####################################################################
   183  echo "## Performing restore for data scale test with zstd ##"
   184  # RESTORE
   185  rm -f $RESULTS_LOG_FILE
   186  (time gprestore --timestamp "\$timestamp" --include-schema big --backup-dir /data/gpdata/ --create-db --redirect-db scalemultifilezstd --jobs=4) > $RESULTS_LOG_FILE 2>&1
   187  echo "gpr_scale_multi_data_file_zstd timestamp restored: \$timestamp"
   188  
   189  # compare round-trip row counts
   190  psql -d scalemultifilezstd -f /home/gpadmin/pull_rowcount.sql -o /home/gpadmin/rowcounts_gpr_scale_multi_data_file_zstd.txt
   191  ROWCOUNTS_DIFF=\$(diff -w /home/gpadmin/rowcounts_orig.txt /home/gpadmin/rowcounts_gpr_scale_multi_data_file_zstd.txt)
   192  if [ "\$ROWCOUNTS_DIFF" != "" ] 
   193  then
   194    echo "Failed result from gpr_scale_multi_data_file_zstd -- mismatched row counts.  Exiting early with failure code."
   195    fusermount -u /home/gpadmin/bucket
   196    exit 1
   197  fi
   198  
   199  # conduct runtime analysis
   200  python /home/gpadmin/analyze_run.py gpr_scale_multi_data_file_zstd
   201  
   202  # clean out redirected database before proceeding further
   203  yes y | gpbackup_manager delete-backup "\$timestamp"
   204  dropdb scalemultifilezstd
   205  #####################################################################
   206  #####################################################################
   207  
   208  #####################################################################
   209  ##################################################################### 
   210  echo "## Performing single-data-file backup for data scale test ##"
   211  # BACKUP
   212  rm -f $RESULTS_LOG_FILE
   213  (time gpbackup --dbname scaletestdb --include-schema big --backup-dir /data/gpdata/ --single-data-file) > $RESULTS_LOG_FILE 2>&1
   214  timestamp=\$(head -10 "\$RESULTS_LOG_FILE" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   215  echo "gpb_scale_single_data_file timestamp backed up: \$timestamp"
   216  
   217  # conduct runtime analysis
   218  python /home/gpadmin/analyze_run.py gpb_scale_single_data_file
   219  #####################################################################
   220  
   221  #####################################################################
   222  echo "## Performing single-data-file restore for data scale test ##"
   223  # RESTORE
   224  rm -f $RESULTS_LOG_FILE
   225  (time gprestore --timestamp "\$timestamp" --include-schema big --backup-dir /data/gpdata/ --create-db --redirect-db scalesinglefile) > $RESULTS_LOG_FILE 2>&1
   226  echo "gpr_scale_single_data_file timestamp restored: \$timestamp"
   227  
   228  # compare round-trip row counts
   229  psql -d scalesinglefile -f /home/gpadmin/pull_rowcount.sql -o /home/gpadmin/rowcounts_gpr_scale_single_data_file.txt
   230  ROWCOUNTS_DIFF=\$(diff -w /home/gpadmin/rowcounts_orig.txt /home/gpadmin/rowcounts_gpr_scale_single_data_file.txt)
   231  if [ "\$ROWCOUNTS_DIFF" != "" ] 
   232  then
   233    echo "Failed result from gpr_scale_single_data_file -- mismatched row counts.  Exiting early with failure code."
   234    fusermount -u /home/gpadmin/bucket
   235    exit 1
   236  fi
   237  
   238  # conduct runtime analysis
   239  python /home/gpadmin/analyze_run.py gpr_scale_single_data_file
   240  
   241  # clean out redirected database before proceeding further
   242  yes y | gpbackup_manager delete-backup "\$timestamp"
   243  dropdb scalesinglefile
   244  #####################################################################
   245  #####################################################################
   246  
   247  #####################################################################
   248  ##################################################################### 
   249  echo "## Performing single-data-file backup for data scale test with zstd ##"
   250  # BACKUP
   251  rm -f $RESULTS_LOG_FILE
   252  (time gpbackup --dbname scaletestdb --include-schema big --backup-dir /data/gpdata/ --single-data-file --compression-type zstd) > $RESULTS_LOG_FILE 2>&1
   253  timestamp=\$(head -10 "\$RESULTS_LOG_FILE" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   254  echo "gpb_scale_single_data_file_zstd timestamp backed up: \$timestamp"
   255  
   256  # conduct runtime analysis
   257  python /home/gpadmin/analyze_run.py gpb_scale_single_data_file_zstd
   258  #####################################################################
   259  
   260  #####################################################################
   261  echo "## Performing single-data-file restore for data scale test with zstd ##"
   262  # RESTORE
   263  rm -f $RESULTS_LOG_FILE
   264  (time gprestore --timestamp "\$timestamp" --include-schema big --backup-dir /data/gpdata/ --create-db --redirect-db scalesinglefilezstd) > $RESULTS_LOG_FILE 2>&1
   265  echo "gpr_scale_single_data_file_zstd timestamp restored: \$timestamp"
   266  
   267  # compare round-trip row counts
   268  psql -d scalesinglefilezstd -f /home/gpadmin/pull_rowcount.sql -o /home/gpadmin/rowcounts_gpr_scale_single_data_file_zstd.txt
   269  ROWCOUNTS_DIFF=\$(diff -w /home/gpadmin/rowcounts_orig.txt /home/gpadmin/rowcounts_gpr_scale_single_data_file_zstd.txt)
   270  if [ "\$ROWCOUNTS_DIFF" != "" ] 
   271  then
   272    echo "Failed result from gpr_scale_single_data_file_zstd -- mismatched row counts.  Exiting early with failure code."
   273    fusermount -u /home/gpadmin/bucket
   274    exit 1
   275  fi
   276  
   277  # conduct runtime analysis
   278  python /home/gpadmin/analyze_run.py gpr_scale_single_data_file_zstd
   279  
   280  # clean out redirected database before proceeding further
   281  yes y | gpbackup_manager delete-backup "\$timestamp"
   282  dropdb scalesinglefilezstd
   283  #####################################################################
   284  #####################################################################
   285  
   286  #####################################################################
   287  ##################################################################### 
   288  # TEST GPBACKUP UNDER VARIOUS PRESSURES
   289  #####################################################################
   290  ##################################################################### 
   291  
   292  #####################################################################
   293  #####################################################################
   294  echo "## Performing backup with moderate number of jobs while database is being edited ##"
   295  # BACKUP
   296  rm -f $RESULTS_LOG_FILE
   297  echo "RESULTS_LOG_FILE: \$RESULTS_LOG_FILE"
   298  (time gpbackup --dbname scaletestdb --include-schema big --backup-dir /data/gpdata --jobs=16 ) > \$RESULTS_LOG_FILE 2>&1 &
   299  echo "Backup initiated in the background."
   300  # check log for lock acquisition before proceeding
   301  set +e # turn off exit on error so grep doesn't halt the whole script
   302  TIMEOUT_COUNTER=0
   303  while true
   304  do
   305      sleep 1
   306      LOCKSGREP=\$(grep "Locks acquired: .* 100\.00\%" \$RESULTS_LOG_FILE)
   307      if [ "\$LOCKSGREP" != "" ]; then
   308          echo "All locks acquired.  Proceeding with ETL job."
   309          break
   310      fi
   311  
   312      if ((\$TIMEOUT_COUNTER > 100)); then
   313          echo "Test timed out waiting for lock acquisition"
   314          exit 1
   315      fi
   316      echo "\$TIMEOUT_COUNTER"
   317      ((TIMEOUT_COUNTER=\$TIMEOUT_COUNTER+1))
   318  done
   319  
   320  # begin ETL job
   321  psql -d scaletestdb -f /home/gpadmin/etl_job.sql > /dev/null
   322  
   323  # check log for backup completion before proceeding
   324  TIMEOUT_COUNTER=0
   325  while true
   326  do
   327      sleep 1
   328      COMPGREP=\$(grep "Backup completed successfully" \$RESULTS_LOG_FILE)
   329      if [ "\$COMPGREP" != "" ]; then
   330          break
   331      fi
   332  
   333      if ((\$TIMEOUT_COUNTER > 10000)); then
   334          echo "Test timed out waiting for backup completion"
   335          exit 1
   336      fi
   337      ((TIMEOUT_COUNTER=\$TIMEOUT_COUNTER+1))
   338  done
   339  set -e
   340  
   341  timestamp=\$(head -10 "\$RESULTS_LOG_FILE" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   342  echo "gpb_distr_snap_edit_data timestamp backed up: \$timestamp"
   343  
   344  # conduct runtime analysis
   345  python /home/gpadmin/analyze_run.py gpb_distr_snap_edit_data
   346  #####################################################################
   347  
   348  #####################################################################
   349  echo "## Performing restore with moderate number of jobs on backup done while database is edited ##"
   350  # RESTORE
   351  rm -f $RESULTS_LOG_FILE
   352  dropdb scaletestdb
   353  (time gprestore --timestamp "\$timestamp" --include-schema big --backup-dir /data/gpdata --create-db --redirect-db newscaletestdb --jobs=16) > \$RESULTS_LOG_FILE 2>&1
   354  echo "gpr_distr_snap_edit_data timestamp restored: \$timestamp"
   355  
   356  # compare round-trip row counts
   357  psql -d newscaletestdb -f /home/gpadmin/pull_rowcount.sql -o /home/gpadmin/rowcounts_gpr_distr_snap_edit_data.txt
   358  ROWCOUNTS_DIFF=\$(diff -w /home/gpadmin/rowcounts_orig.txt /home/gpadmin/rowcounts_gpr_distr_snap_edit_data.txt)
   359  if [ "\$ROWCOUNTS_DIFF" != "" ] 
   360  then
   361    echo "Failed result from gpr_distr_snap_edit_data -- mismatched row counts.  Exiting early with failure code."
   362    fusermount -u /home/gpadmin/bucket
   363    exit 1
   364  fi
   365  
   366  # conduct runtime analysis
   367  python /home/gpadmin/analyze_run.py gpr_distr_snap_edit_data
   368  
   369  # clean out redirected database before proceeding further
   370  yes y | gpbackup_manager delete-backup "\$timestamp"
   371  #####################################################################
   372  #####################################################################
   373  
   374  #####################################################################
   375  ##################################################################### 
   376  echo "## Performing backup with high number of jobs on cluster with high-concurrency load ##"
   377  # BACKUP
   378  rm -f $RESULTS_LOG_FILE
   379  (time gpbackup --dbname newscaletestdb --include-schema big --backup-dir /data/gpdata --jobs=32 ) > \$RESULTS_LOG_FILE 2>&1 &
   380  # check log for lock acquisition before proceeding
   381  set +e set +e # turn off exit on error so grep doesn't halt the whole script
   382  TIMEOUT_COUNTER=0
   383  while true
   384  do
   385      sleep 1
   386      LOCKSGREP=\$(grep "Locks acquired: .* 100\.00\%" \$RESULTS_LOG_FILE)
   387      if [ "\$LOCKSGREP" != "" ]; then
   388          echo "All locks acquired.  Proceeding with data load"
   389          break
   390      fi
   391  
   392      if ((\$TIMEOUT_COUNTER > 100)); then
   393          echo "Test timed out waiting for lock acquisition"
   394          exit 1
   395      fi
   396      ((TIMEOUT_COUNTER=\$TIMEOUT_COUNTER+1))
   397  done
   398  
   399  # load data into a separate database to apply high concurrent load to cluster
   400  createdb scaletestdb
   401  psql -d scaletestdb -q -f scaletestdb_bigschema_ddl.sql
   402  gpload -f /home/gpadmin/gpload_yaml/lineitem.yml
   403  gpload -f /home/gpadmin/gpload_yaml/orders_3.yml
   404  
   405  # check log for backup completion before proceeding
   406  TIMEOUT_COUNTER=0
   407  while true
   408  do
   409      sleep 1
   410      COMPGREP=\$(grep "Backup completed successfully" \$RESULTS_LOG_FILE)
   411      if [ "\$COMPGREP" != "" ]; then
   412          break
   413      fi
   414  
   415      if ((\$TIMEOUT_COUNTER > 10000)); then
   416          echo "Test timed out waiting for backup completion"
   417          exit 1
   418      fi
   419      ((TIMEOUT_COUNTER=\$TIMEOUT_COUNTER+1))
   420  done
   421  set -e
   422  
   423  timestamp=\$(head -10 "\$RESULTS_LOG_FILE" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   424  echo "gpb_distr_snap_high_conc timestamp backed up: \$timestamp"
   425  
   426  # conduct runtime analysis
   427  python /home/gpadmin/analyze_run.py gpb_distr_snap_high_conc
   428  #####################################################################
   429  
   430  #####################################################################
   431  echo "## Performing restore with high number of jobs on backup done while cluster had high-concurrency load ##"
   432  # RESTORE
   433  rm -f $RESULTS_LOG_FILE
   434  dropdb scaletestdb
   435  (time gprestore --timestamp "\$timestamp" --include-schema big --backup-dir /data/gpdata --create-db --redirect-db scaletestdb --jobs=32) > \$RESULTS_LOG_FILE 2>&1
   436  echo "gpr_distr_snap_high_conc timestamp restored: \$timestamp"
   437  
   438  # compare round-trip row counts
   439  psql -d scaletestdb -f /home/gpadmin/pull_rowcount.sql -o /home/gpadmin/rowcounts_gpr_distr_snap_high_conc.txt
   440  ROWCOUNTS_DIFF=\$(diff -w /home/gpadmin/rowcounts_orig.txt /home/gpadmin/rowcounts_gpr_distr_snap_high_conc.txt)
   441  if [ "\$ROWCOUNTS_DIFF" != "" ] 
   442  then
   443    echo "Failed result from gpb_distr_snap_high_conc -- mismatched row counts.  Exiting early with failure code."
   444    fusermount -u /home/gpadmin/bucket
   445    exit 1
   446  fi
   447  
   448  # conduct runtime analysis
   449  python /home/gpadmin/analyze_run.py gpr_distr_snap_high_conc
   450  
   451  # clean out redirected database before proceeding further
   452  yes y | gpbackup_manager delete-backup "\$timestamp"
   453  dropdb newscaletestdb
   454  #####################################################################
   455  #####################################################################
   456  
   457  #####################################################################
   458  ##################################################################### 
   459  # METADATA-ONLY FROM HERE ON
   460  echo "## Loading wide schema for metadata tests"
   461  psql -d scaletestdb -q -f scaletestdb_wideschema_ddl.sql
   462  #####################################################################
   463  ##################################################################### 
   464  
   465  #####################################################################
   466  ##################################################################### 
   467  echo "## Performing first backup with metadata-only ##"
   468  # BACKUP
   469  rm -f $RESULTS_LOG_FILE
   470  (time gpbackup --dbname scaletestdb --include-schema wide --backup-dir /data/gpdata/ --metadata-only --verbose) > $RESULTS_LOG_FILE 2>&1
   471  timestamp=\$(head -10 "\$RESULTS_LOG_FILE" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   472  echo "gpb_scale_metadata timestamp backed up: \$timestamp"
   473  test_metadata=\$(find /data/gpdata/ -name *\$timestamp*_metadata.sql)
   474  
   475  METADATA_DIFF=\$(diff -w /home/gpadmin/valid_metadata.sql \$test_metadata)
   476  echo "got past metadata diff"
   477  if [ "\$METADATA_DIFF" != "" ] 
   478  then
   479    echo "Failed result from gpb_scale_metadata -- mismatched metadata output.  Exiting early with failure code."
   480    fusermount -u /home/gpadmin/bucket
   481    exit 1
   482  fi
   483  
   484  # conduct runtime analysis
   485  python /home/gpadmin/analyze_run.py gpb_scale_metadata
   486  #####################################################################
   487  
   488  #####################################################################
   489  echo "## Performing restore on metadata-only ##"
   490  # RESTORE
   491  rm -f $RESULTS_LOG_FILE
   492  dropdb scaletestdb
   493  (time gprestore --timestamp "\$timestamp" --include-schema wide --backup-dir /data/gpdata/ --create-db --redirect-db scaletestdb) > $RESULTS_LOG_FILE 2>&1
   494  echo "gpr_scale_metadata timestamp restored: \$timestamp"
   495  
   496  echo "## Performing second backup with metadata-only ##"
   497  rm -f $RESULTS_LOG_FILE
   498  (time gpbackup --dbname scaletestdb --include-schema wide --backup-dir /data/gpdata/ --metadata-only --verbose) > $RESULTS_LOG_FILE 2>&1
   499  timestamp=\$(head -10 "\$RESULTS_LOG_FILE" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   500  test_metadata=\$(find /data/gpdata/ -name *\$timestamp*_metadata.sql)
   501  
   502  METADATA_DIFF=\$(diff -w /home/gpadmin/valid_metadata.sql \$test_metadata)
   503  if [ "\$METADATA_DIFF" != "" ] 
   504  then
   505    echo "Failed result from gpr_scale_metadata -- mismatched metadata output.  Exiting early with failure code."
   506    fusermount -u /home/gpadmin/bucket
   507    exit 1
   508  fi
   509  
   510  # conduct runtime analysis
   511  python /home/gpadmin/analyze_run.py gpr_scale_metadata
   512  
   513  # clean out redirected database before proceeding further
   514  yes y | gpbackup_manager delete-backup "\$timestamp"
   515  #####################################################################
   516  #####################################################################
   517  
   518  
   519  # if successful run, unmount bucket before exiting
   520  # fusermount -u /home/gpadmin/bucket
   521  SCRIPT
   522  
   523  chmod +x /tmp/run_tests.bash
   524  scp /tmp/run_tests.bash cdw:/home/gpadmin/run_tests.bash
   525  ssh -t cdw "/home/gpadmin/run_tests.bash"