github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/build/teamcity-nightly-roachtest.sh (about)

     1  #!/usr/bin/env bash
     2  set -euxo pipefail
     3  
     4  # Entry point for the nightly roachtests. These are run from CI and require
     5  # appropriate secrets for the ${CLOUD} parameter (along with other things,
     6  # apologies, you're going to have to dig around for them below or even better
     7  # yet, look at the job).
     8  
     9  # Note that when this script is called, the cockroach binary to be tested
    10  # already exists in the current directory.
    11  COCKROACH_BINARY="${PWD}/cockroach.linux-2.6.32-gnu-amd64"
    12  chmod +x "${COCKROACH_BINARY}"
    13  
    14  if [[ ! -f ~/.ssh/id_rsa.pub ]]; then
    15    ssh-keygen -q -C "roachtest-nightly $(date)" -N "" -f ~/.ssh/id_rsa
    16  fi
    17  
    18  # The artifacts dir should match up with that supplied by TC.
    19  artifacts=$PWD/artifacts
    20  mkdir -p "${artifacts}"
    21  chmod o+rwx "${artifacts}"
    22  
    23  # Disable global -json flag.
    24  export PATH=$PATH:$(GOFLAGS=; go env GOPATH)/bin
    25  
    26  make bin/workload bin/roachtest bin/roachprod > "${artifacts}/build.txt" 2>&1 || cat "${artifacts}/build.txt"
    27  
    28  # Set up Google credentials. Note that we need this for all clouds since we upload
    29  # perf artifacts to Google Storage at the end.
    30  if [[ "$GOOGLE_EPHEMERAL_CREDENTIALS" ]]; then
    31    echo "$GOOGLE_EPHEMERAL_CREDENTIALS" > creds.json
    32    gcloud auth activate-service-account --key-file=creds.json
    33    export ROACHPROD_USER=teamcity
    34  else
    35    echo 'warning: GOOGLE_EPHEMERAL_CREDENTIALS not set' >&2
    36    echo "Assuming that you've run \`gcloud auth login\` from inside the builder." >&2
    37  fi
    38  
    39  # Early bind the stats dir. Roachtest invocations can take ages, and we want the
    40  # date at the time of the start of the run (which identifies the version of the
    41  # code run best).
    42  stats_dir="$(date +"%Y%m%d")-${TC_BUILD_ID}"
    43  
    44  # Set up a function we'll invoke at the end.
    45  function upload_stats {
    46   if [[ "${TC_BUILD_BRANCH}" == "master" ]]; then
    47        bucket="cockroach-nightly-${CLOUD}"
    48        if [[ "${CLOUD}" == "gce" ]]; then
    49  	  # GCE, having been there first, gets an exemption.
    50            bucket="cockroach-nightly"
    51        fi
    52        # The stats.json files need some path translation:
    53        #     ${artifacts}/path/to/test/stats.json
    54        # to
    55        #     gs://${bucket}/artifacts/${stats_dir}/path/to/test/stats.json
    56        #
    57        # `find` below will expand "{}" as ./path/to/test/stats.json. We need
    58        # to bend over backwards to remove the `./` prefix or gsutil will have
    59        # a `.` folder in ${stats_dir}, which we don't want.
    60        (cd "${artifacts}" && \
    61          while IFS= read -r f; do
    62            gsutil cp "${f}" "gs://${bucket}/artifacts/${stats_dir}/${f}"
    63          done <<< "$(find . -name stats.json | sed 's/^\.\///')")
    64    fi
    65  }
    66  
    67  # Upload any stats.json we can find, no matter what happens.
    68  trap upload_stats EXIT
    69  
    70  # Set up the parameters for the roachtest invocation.
    71  
    72  ARTIFACTS="${artifacts}"
    73  PARALLELISM=16
    74  CPUQUOTA=1024
    75  ZONES=""
    76  case "${CLOUD}" in
    77    gce)
    78      # We specify --zones below so that nodes are created in us-central1-b by
    79      # default. This reserves us-east1-b (the roachprod default zone) for use by
    80      # manually created clusters.
    81      ZONES="us-central1-b,us-west1-b,europe-west2-b"
    82      ;;
    83    aws)
    84      PARALLELISM=3
    85      CPUQUOTA=384
    86      if [ -z "${TESTS}" ]; then
    87        TESTS="kv(0|95)|ycsb|tpcc/(headroom/n4cpu16)|tpccbench/(nodes=3/cpu=16)"
    88      fi
    89      ;;
    90    *)
    91      echo "unknown cloud ${CLOUD}"
    92      exit 1
    93      ;;
    94  esac
    95  
    96  export \
    97  CLOUD="${CLOUD}" \
    98  ARTIFACTS="${ARTIFACTS}" \
    99  PARALLELISM="${PARALLELISM}" \
   100  CPUQUOTA="${CPUQUOTA}" \
   101  ZONES="${ZONES}" \
   102  COUNT="${COUNT-1}" \
   103  DEBUG="${DEBUG-false}" \
   104  BUILD_TAG="${BUILD_TAG}" \
   105  COCKROACH_BINARY="${COCKROACH_BINARY}" \
   106  SLACK_TOKEN="${SLACK_TOKEN}" \
   107  TC_BUILD_ID="${TC_BUILD_ID}" \
   108  TESTS="${TESTS}"
   109  
   110  # Teamcity has a 1300 minute timeout that, when reached, kills the process
   111  # without a stack trace (probably SIGKILL).  We'd love to see a stack trace
   112  # though, so after 1200 minutes, kill with SIGINT which will allow roachtest to
   113  # fail tests and cleanup.
   114  timeout -s INT $((1200*60)) "build/teamcity-nightly-roachtest-invoke.sh"