github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/ci/scripts/scale-tests.bash (about)

     1  #!/bin/bash
     2  
     3  set -ex
     4  
     5  # setup cluster and install gpbackup tools using gppkg
     6  ccp_src/scripts/setup_ssh_to_cluster.sh
     7  out=$(ssh -t cdw 'source env.sh && psql postgres -c "select version();"')
     8  GPDB_VERSION=$(echo ${out} | sed -n 's/.*Greenplum Database \([0-9]\).*/\1/p')
     9  mkdir -p /tmp/untarred
    10  tar -xzf gppkgs/gpbackup-gppkgs.tar.gz -C /tmp/untarred
    11  scp /tmp/untarred/gpbackup_tools*gp${GPDB_VERSION}*${OS}*.gppkg cdw:/home/gpadmin
    12  
    13  scp cluster_env_files/hostfile_all cdw:/tmp
    14  
    15  cat <<SCRIPT > /tmp/run_tests.bash
    16  #!/bin/bash
    17  
    18  source env.sh
    19  
    20  # Double the vmem protect limit default on the coordinator segment to
    21  # prevent query cancels on large table creations (e.g. scale_db1.sql)
    22  gpconfig -c gp_vmem_protect_limit -v 16384 --masteronly
    23  gpstop -air
    24  
    25  # only install if not installed already
    26  is_installed_output=\$(source env.sh; gppkg -q gpbackup*gp*.gppkg)
    27  set +e
    28  echo \$is_installed_output | grep 'is installed'
    29  if [ \$? -ne 0 ] ; then
    30    set -e
    31    gppkg -i gpbackup*gp*.gppkg
    32  fi
    33  set -e
    34  
    35  ### Data scale tests ###
    36  log_file=/tmp/gpbackup.log
    37  
    38  echo "## Populating database for copy queue test ##"
    39  createdb copyqueuedb
    40  for j in {1..20000}
    41  do
    42    psql -d copyqueuedb -q -c "CREATE TABLE tbl_1k_\$j(i int) DISTRIBUTED BY (i);"
    43    psql -d copyqueuedb -q -c "INSERT INTO tbl_1k_\$j SELECT generate_series(1,1000)"
    44  done
    45  
    46  echo "## Performing single-data-file, --no-compression, --copy-queue-size 2 backup for copy queue test ##"
    47  time gpbackup --dbname copyqueuedb --backup-dir /data/gpdata/ --single-data-file --no-compression --copy-queue-size 2 | tee "\$log_file"
    48  timestamp=\$(head -10 "\$log_file" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
    49  gpbackup_manager display-report \$timestamp
    50  
    51  echo "## Performing single-data-file, --no-compression, --copy-queue-size 4 backup for copy queue test ##"
    52  time gpbackup --dbname copyqueuedb --backup-dir /data/gpdata/ --single-data-file --no-compression --copy-queue-size 4 | tee "\$log_file"
    53  timestamp=\$(head -10 "\$log_file" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
    54  gpbackup_manager display-report \$timestamp
    55  
    56  echo "## Performing single-data-file, --no-compression, --copy-queue-size 8 backup for copy queue test ##"
    57  time gpbackup --dbname copyqueuedb --backup-dir /data/gpdata/ --single-data-file --no-compression --copy-queue-size 8 | tee "\$log_file"
    58  timestamp=\$(head -10 "\$log_file" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
    59  gpbackup_manager display-report \$timestamp
    60  
    61  echo "## Performing single-data-file, --no-compression, --copy-queue-size 2 restore for copy queue test ##"
    62  time gprestore --timestamp "\$timestamp" --backup-dir /data/gpdata/ --create-db --redirect-db copyqueuerestore2 --copy-queue-size 2
    63  
    64  echo "## Performing single-data-file, --no-compression, --copy-queue-size 8 restore for copy queue test ##"
    65  time gprestore --timestamp "\$timestamp" --backup-dir /data/gpdata/ --create-db --redirect-db copyqueuerestore8 --copy-queue-size 8
    66  
    67  echo "## Populating database for data scale test ##"
    68  createdb datascaledb
    69  for j in {1..5000}
    70  do
    71    psql -d datascaledb -q -c "CREATE TABLE tbl_1k_\$j(i int) DISTRIBUTED BY (i);"
    72    psql -d datascaledb -q -c "INSERT INTO tbl_1k_\$j SELECT generate_series(1,1000)"
    73  done
    74  for j in {1..100}
    75  do
    76    psql -d datascaledb -q -c "CREATE TABLE tbl_1M_\$j(i int) DISTRIBUTED BY(i);"
    77    psql -d datascaledb -q -c "INSERT INTO tbl_1M_\$j SELECT generate_series(1,1000000)"
    78  done
    79  psql -d datascaledb -q -c "CREATE TABLE tbl_1B(i int) DISTRIBUTED BY(i);"
    80  for j in {1..1000}
    81  do
    82    psql -d datascaledb -q -c "INSERT INTO tbl_1B SELECT generate_series(1,1000000)"
    83  done
    84  
    85  echo "## Performing backup for data scale test ##"
    86  ### Multiple data file test ###
    87  time gpbackup --dbname datascaledb --backup-dir /data/gpdata/ | tee "\$log_file"
    88  timestamp=\$(head -10 "\$log_file" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
    89  dropdb datascaledb
    90  echo "## Performing restore for data scale test ##"
    91  time gprestore --timestamp "\$timestamp" --backup-dir /data/gpdata/ --create-db --jobs=4 --quiet
    92  rm "\$log_file"
    93  
    94  echo "## Performing backup for data scale test with zstd ##"
    95  ### Multiple data file test with zstd ###
    96  time gpbackup --dbname datascaledb --backup-dir /data/gpdata/ --compression-type zstd | tee "\$log_file"
    97  timestamp=\$(head -10 "\$log_file" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
    98  dropdb datascaledb
    99  echo "## Performing restore for data scale test with zstd ##"
   100  time gprestore --timestamp "\$timestamp" --backup-dir /data/gpdata/ --create-db --jobs=4 --quiet
   101  rm "\$log_file"
   102  
   103  echo "## Performing single-data-file backup for data scale test ##"
   104  ### Single data file test ###
   105  time gpbackup --dbname datascaledb --backup-dir /data/gpdata/ --single-data-file | tee "\$log_file"
   106  timestamp=\$(head -10 "\$log_file" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   107  dropdb datascaledb
   108  echo "## Performing single-data-file restore for data scale test ##"
   109  time gprestore --timestamp "\$timestamp" --backup-dir /data/gpdata/  --create-db --quiet
   110  rm "\$log_file"
   111  
   112  echo "## Performing single-data-file backup for data scale test with zstd ##"
   113  ### Single data file test with zstd ###
   114  time gpbackup --dbname datascaledb --backup-dir /data/gpdata/ --single-data-file --compression-type zstd | tee "\$log_file"
   115  timestamp=\$(head -10 "\$log_file" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   116  dropdb datascaledb
   117  echo "## Performing single-data-file restore for data scale test with zstd ##"
   118  time gprestore --timestamp "\$timestamp" --backup-dir /data/gpdata/  --create-db --quiet
   119  dropdb datascaledb
   120  rm "\$log_file"
   121  
   122  ### Metadata scale test ###
   123  echo "## Populating database for metadata scale test ##"
   124  tar -xvf scale_db1.tgz
   125  createdb metadatascaledb -T template0
   126  
   127  psql -f scale_db1.sql -d metadatascaledb -v client_min_messages=error -q
   128  
   129  echo "## Performing pg_dump with metadata-only ##"
   130  time pg_dump -s metadatascaledb > /data/gpdata/pg_dump.sql
   131  echo "## Performing gpbackup with metadata-only ##"
   132  time gpbackup --dbname metadatascaledb --backup-dir /data/gpdata/ --metadata-only --verbose | tee "\$log_file"
   133  
   134  timestamp=\$(head -10 "\$log_file" | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}")
   135  echo "## Performing gprestore with metadata-only ##"
   136  time gprestore --timestamp "\$timestamp" --backup-dir /data/gpdata/ --redirect-db=metadatascaledb_res --jobs=4 --create-db
   137  
   138  SCRIPT
   139  
   140  chmod +x /tmp/run_tests.bash
   141  scp /tmp/run_tests.bash cdw:/home/gpadmin/run_tests.bash
   142  scp -r scale_schema/scale_db1.tgz cdw:/home/gpadmin/
   143  ssh -t cdw "/home/gpadmin/run_tests.bash"