github.com/greenplum-db/gpbackup@v0.0.0-20240517212602-89daab1885b3/ci/scripts/s3-plugin-tests.bash (about)

     1  #!/bin/bash
     2  
     3  set -ex
     4  
     5  # setup cluster and install gpbackup tools using gppkg
     6  ccp_src/scripts/setup_ssh_to_cluster.sh
     7  out=$(ssh -t cdw 'source env.sh && psql postgres -c "select version();"')
     8  GPDB_VERSION=$(echo ${out} | sed -n 's/.*Greenplum Database \([0-9]\).*/\1/p')
     9  mkdir -p /tmp/untarred
    10  tar -xzf gppkgs/gpbackup-gppkgs.tar.gz -C /tmp/untarred
    11  scp /tmp/untarred/gpbackup_tools*gp${GPDB_VERSION}*${OS}*.gppkg cdw:/home/gpadmin
    12  ssh -t cdw "source env.sh; gppkg -i gpbackup_tools*.gppkg"
    13  
    14  cat <<SCRIPT > /tmp/setup_minio.bash
    15  # Install Minio
    16  wget https://dl.min.io/server/minio/release/linux-amd64/minio
    17  sudo mv minio /usr/local/bin/
    18  sudo chmod 777 /usr/local/bin/minio
    19  sudo chown rocky:rocky /usr/local/bin/minio
    20  
    21  # Setup Minio server
    22  mkdir -p /tmp/minio_root
    23  mkdir -p /tmp/.minio/logs
    24  touch /tmp/.minio/logs/minio_log
    25  chmod 777 /tmp/minio_root
    26  chmod 777 /tmp/.minio/logs
    27  chown -R rocky:rocky /tmp/minio_root
    28  chown -R rocky:rocky /tmp/.minio
    29  nohup minio server --console-address ":9001" /tmp/minio_root > /tmp/.minio/logs/minio_log 2>&1 &
    30  
    31  curl https://dl.min.io/client/mc/release/linux-amd64/mc \
    32    --create-dirs \
    33    -o \$HOME/minio-binaries/mc
    34  
    35  chmod +x \$HOME/minio-binaries/mc
    36  export PATH=\$PATH:\$HOME/minio-binaries/
    37  which mc
    38  
    39  #create test bucket
    40  mc mb /tmp/minio_root/minio-test-bucket
    41  
    42  SCRIPT
    43  
    44  chmod +x /tmp/setup_minio.bash
    45  scp /tmp/setup_minio.bash rocky@cdw:/home/rocky/setup_minio.bash
    46  ssh -t rocky@cdw "/home/rocky/setup_minio.bash"
    47  
    48  ssh -t rocky@cdw "ps aux | grep minio"
    49  
    50  cat <<SCRIPT > /tmp/minio_tests.bash
    51    #!/bin/bash
    52  
    53    set -ex
    54    source env.sh
    55  
    56    # config to test against s3-compliant storage minio
    57    cat << MINIO_CONFIG > \${HOME}/minio_config.yaml
    58  executablepath: \${GPHOME}/bin/gpbackup_s3_plugin
    59  options:
    60    endpoint: http://\$(hostname -I | awk '{print \$1}'):9000/
    61    aws_access_key_id: minioadmin
    62    aws_secret_access_key: minioadmin
    63    bucket: minio-test-bucket
    64    folder: minio-folder/test-cluster
    65    backup_max_concurrent_requests: 2
    66    backup_multipart_chunksize: 5MB
    67    restore_max_concurrent_requests: 2
    68    restore_multipart_chunksize: 5MB
    69  MINIO_CONFIG
    70  
    71    pushd ~/go/src/github.com/greenplum-db/gpbackup/plugins
    72      ./plugin_test.sh \${GPHOME}/bin/gpbackup_s3_plugin \${HOME}/minio_config.yaml
    73    popd
    74  SCRIPT
    75  
    76  chmod +x /tmp/minio_tests.bash
    77  scp /tmp/minio_tests.bash cdw:/home/gpadmin/minio_tests.bash
    78  ssh -t cdw "/home/gpadmin/minio_tests.bash"
    79  
    80  cat <<SCRIPT > /tmp/s3_tests.bash
    81    #!/bin/bash
    82  
    83    set -ex
    84    source env.sh
    85  
    86    cat << S3_CONFIG > \${HOME}/s3_config.yaml
    87  executablepath: \${GPHOME}/bin/gpbackup_s3_plugin
    88  options:
    89    region: ${REGION}
    90    aws_access_key_id: ${AWS_ACCESS_KEY_ID}
    91    aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY}
    92    bucket: ${BUCKET}
    93    folder: test/backup
    94    backup_multipart_chunksize: 100MB
    95    restore_multipart_chunksize: 100MB
    96  S3_CONFIG
    97  
    98    pushd ~/go/src/github.com/greenplum-db/gpbackup/plugins
    99      ./plugin_test.sh \${GPHOME}/bin/gpbackup_s3_plugin \${HOME}/s3_config.yaml
   100    popd
   101  SCRIPT
   102  
   103  chmod +x /tmp/s3_tests.bash
   104  scp /tmp/s3_tests.bash cdw:/home/gpadmin/s3_tests.bash
   105  ssh -t cdw "/home/gpadmin/s3_tests.bash"