github.com/greenplum-db/gpbackup@v0.0.0-20240517212602-89daab1885b3/ci/scripts/scale-perf-load.bash (about)

     1  #!/bin/bash
     2  
     3  set -ex
     4  
     5  # setup cluster and load necessary tools and data to it
     6  ccp_src/scripts/setup_ssh_to_cluster.sh
     7  out=$(ssh -t cdw 'source env.sh && psql postgres -c "select version();"')
     8  GPDB_VERSION=$(echo ${out} | sed -n 's/.*Greenplum Database \([0-9]\).*/\1/p')
     9  mkdir -p /tmp/untarred
    10  tar -xzf gppkgs/gpbackup-gppkgs.tar.gz -C /tmp/untarred
    11  scp /tmp/untarred/gpbackup_tools*gp${GPDB_VERSION}*${OS}*.gppkg cdw:/home/gpadmin
    12  scp gpbackup/ci/scripts/analyze_run.py cdw:/home/gpadmin/analyze_run.py
    13  scp gpbackup/ci/scale/sql/scaletestdb_bigschema_ddl.sql cdw:/home/gpadmin/scaletestdb_bigschema_ddl.sql
    14  scp gpbackup/ci/scale/sql/scaletestdb_wideschema_ddl.sql cdw:/home/gpadmin/scaletestdb_wideschema_ddl.sql
    15  scp gpbackup/ci/scale/sql/etl_job.sql cdw:/home/gpadmin/etl_job.sql
    16  scp gpbackup/ci/scale/sql/pull_rowcount.sql cdw:/home/gpadmin/pull_rowcount.sql
    17  scp gpbackup/ci/scale/sql/valid_metadata.sql cdw:/home/gpadmin/valid_metadata.sql
    18  scp -r gpbackup/ci/scale/gpload_yaml cdw:/home/gpadmin/gpload_yaml
    19  
    20  set +x
    21  printf "%s" "${GOOGLE_CREDENTIALS}" > "/tmp/keyfile.json"
    22  set -x
    23  
    24  scp /tmp/keyfile.json cdw:/home/gpadmin/keyfile.json && rm -f /tmp/keyfile.json
    25  
    26  cat <<SCRIPT > /tmp/load_data.bash
    27  #!/bin/bash
    28  source env.sh
    29  
    30  # set GCS credentials and mount gcs bucket with gcsfuse
    31  export GOOGLE_APPLICATION_CREDENTIALS=/home/gpadmin/keyfile.json
    32  gcloud auth activate-service-account --key-file=/home/gpadmin/keyfile.json
    33  rm -rf /home/gpadmin/bucket && mkdir /home/gpadmin/bucket
    34  gcsfuse --implicit-dirs dp-gpbackup-scale-test-data /home/gpadmin/bucket
    35  
    36  # Double the vmem protect limit default on the coordinator segment to
    37  # prevent query cancels on large table creations (e.g. scale_db1.sql)
    38  gpconfig -c gp_vmem_protect_limit -v 16384 --masteronly
    39  gpconfig -c client_min_messages -v error
    40  gpstop -air
    41  
    42  # only install if not installed already
    43  is_installed_output=\$(source env.sh; gppkg -q gpbackup*gp*.gppkg)
    44  set +e
    45  echo \$is_installed_output | grep 'is installed'
    46  if [ \$? -ne 0 ] ; then
    47    set -e
    48    gppkg -i gpbackup*gp*.gppkg
    49  fi
    50  set -e
    51  
    52  ### Data scale tests ###
    53  echo "## Loading data into database for scale tests ##"
    54  createdb scaletestdb
    55  psql -d scaletestdb -q -f scaletestdb_bigschema_ddl.sql
    56  gpload -f /home/gpadmin/gpload_yaml/customer.yml
    57  gpload -f /home/gpadmin/gpload_yaml/lineitem.yml
    58  gpload -f /home/gpadmin/gpload_yaml/orders.yml
    59  gpload -f /home/gpadmin/gpload_yaml/orders_2.yml
    60  gpload -f /home/gpadmin/gpload_yaml/orders_3.yml
    61  gpload -f /home/gpadmin/gpload_yaml/nation.yml
    62  gpload -f /home/gpadmin/gpload_yaml/part.yml
    63  gpload -f /home/gpadmin/gpload_yaml/partsupp.yml
    64  gpload -f /home/gpadmin/gpload_yaml/region.yml
    65  gpload -f /home/gpadmin/gpload_yaml/supplier.yml
    66  
    67  # clean out credentials after data is loaded
    68  rm -f /home/gpadmin/keyfile.json
    69  
    70  # unmount bucket before exiting
    71  fusermount -u /home/gpadmin/bucket
    72  
    73  SCRIPT
    74  
    75  # tar up files for this cluster so that later jobs can connect to it
    76  tar -czf cluster-metadata/cluster-metadata.tar.gz cluster_env_files/
    77  
    78  chmod +x /tmp/load_data.bash
    79  scp /tmp/load_data.bash cdw:/home/gpadmin/load_data.bash
    80  ssh -t cdw "/home/gpadmin/load_data.bash"
    81