github.com/greenplum-db/gpbackup@v0.0.0-20240517212602-89daab1885b3/ci/scripts/setup-perf.bash (about)

     1  #!/bin/bash
     2  
     3  set -ex
     4  
     5  ccp_src/scripts/setup_ssh_to_cluster.sh
     6  ssh -t rocky@cdw "curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscli.zip" && \
     7      unzip -qq awscli.zip && \
     8      sudo ./aws/install"
     9  
    10  out=$(ssh -t cdw 'source env.sh && psql postgres -c "select version();"')
    11  GPDB_VERSION=$(echo ${out} | sed -n 's/.*Greenplum Database \([0-9]\).*/\1/p')
    12  mkdir -p /tmp/untarred
    13  tar -xzf gppkgs/gpbackup-gppkgs.tar.gz -C /tmp/untarred
    14  scp /tmp/untarred/gpbackup_tools*gp${GPDB_VERSION}*${OS}*.gppkg cdw:/home/gpadmin
    15  ssh -t cdw "source env.sh; gppkg -i gpbackup_tools*.gppkg"
    16  
    17  cat << EOF > lineitem.ddl
    18  CREATE TABLE lineitem (
    19      l_orderkey       INTEGER NOT NULL,
    20      l_partkey        INTEGER NOT NULL,
    21      l_suppkey        INTEGER NOT NULL,
    22      l_linenumber     INTEGER NOT NULL,
    23      l_quantity       DECIMAL(15,2) NOT NULL,
    24      l_extendedprice  DECIMAL(15,2) NOT NULL,
    25      l_discount       DECIMAL(15,2) NOT NULL,
    26      l_tax            DECIMAL(15,2) NOT NULL,
    27      l_returnflag     CHAR(1) NOT NULL,
    28      l_linestatus     CHAR(1) NOT NULL,
    29      l_shipdate       DATE NOT NULL,
    30      l_commitdate     DATE NOT NULL,
    31      l_receiptdate    DATE NOT NULL,
    32      l_shipinstruct   CHAR(25) NOT NULL,
    33      l_shipmode       CHAR(10) NOT NULL,
    34      l_comment        VARCHAR(44) NOT NULL
    35  )
    36  DISTRIBUTED BY (l_orderkey);
    37  EOF
    38  
    39  cat << EOF > gpload.yml
    40  ---
    41  VERSION: 1.0.0.1
    42  DATABASE: tpchdb
    43  USER: gpadmin
    44  HOST: localhost
    45  PORT: ${PG_PORT}
    46  GPLOAD:
    47     INPUT:
    48      - SOURCE:
    49           FILE:
    50             - /data/gpdata/lineitem_${SCALE_FACTOR}.tbl
    51      - FORMAT: text
    52      - DELIMITER: '|'
    53      - HEADER: false
    54     OUTPUT:
    55      - TABLE: lineitem
    56      - MODE: insert
    57      - UPDATE_CONDITION: 'boolean_condition'
    58     PRELOAD:
    59      - TRUNCATE: true
    60      - REUSE_TABLES: false
    61  EOF
    62  
    63  cat <<SCRIPT > /tmp/setup_perf.bash
    64  #!/bin/bash
    65  
    66  set -ex
    67  source env.sh
    68  TIMEFORMAT=%R
    69  
    70  function print_header() {
    71      header="### \$1 ###"
    72      len=\$(echo \$header | awk '{print length}')
    73      printf "%0.s#" \$(seq 1 \$len) && echo
    74      echo -e "\$header"
    75      printf "%0.s#" \$(seq 1 \$len) && echo
    76  }
    77  
    78  mkdir -p /home/gpadmin/.aws
    79  cat << CRED > \${HOME}/.aws/credentials
    80  [default]
    81  aws_access_key_id=${AWS_ACCESS_KEY_ID}
    82  aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}
    83  CRED
    84  chmod 400 \${HOME}/.aws/credentials
    85  aws s3 cp s3://${BUCKET}/benchmark/tpch/lineitem/${SCALE_FACTOR}/lineitem.tbl /data/gpdata/lineitem_${SCALE_FACTOR}.tbl
    86  
    87  # Create tpch dataset
    88  createdb tpchdb
    89  
    90  # install pgcrypto; works for GPDB 5.22+ and 6+
    91  psql -d postgres -c "CREATE EXTENSION pgcrypto"
    92  psql -d tpchdb -c "CREATE EXTENSION pgcrypto"
    93  
    94  psql -d tpchdb -a -f lineitem.ddl
    95  gpload -f gpload.yml
    96  
    97  set +x
    98  print_header "CREATE tpchdb with 150 lineitem tables each with ${SCALE_FACTOR} GB"
    99  count=0
   100  for i in {1..5}
   101  do
   102    pids=""
   103    for j in {1..30}
   104    do
   105      psql -d tpchdb -c "CREATE TABLE lineitem_\$count AS SELECT * FROM lineitem DISTRIBUTED BY (l_orderkey)" &
   106      pids+=" $!"
   107      ((count=count+1))
   108    done
   109    wait $pids || { echo "errors" >&2; exit 1; }
   110  done
   111  
   112  set -x
   113  
   114  SCRIPT
   115  
   116  chmod +x /tmp/setup_perf.bash
   117  scp lineitem.ddl gpload.yml /tmp/setup_perf.bash cdw:
   118  ssh -t cdw "/home/gpadmin/setup_perf.bash"