github.com/spg/deis@v1.7.3/tests/bin/test-setup.sh (about)

     1  #!/bin/bash
     2  #
     3  # Prepares the process environment to run a test
     4  
     5  function log_phase {
     6      echo
     7      echo ">>> $1 <<<"
     8      echo
     9  }
    10  
    11  log_phase "Preparing test environment"
    12  
    13  # use GOPATH to determine project root
    14  export DEIS_ROOT=${GOPATH?}/src/github.com/deis/deis
    15  echo "DEIS_ROOT=$DEIS_ROOT"
    16  
    17  # prepend GOPATH/bin to PATH
    18  export PATH=${GOPATH}/bin:$PATH
    19  
    20  # the application under test
    21  export DEIS_TEST_APP=${DEIS_TEST_APP:-example-dockerfile-http}
    22  echo "DEIS_TEST_APP=$DEIS_TEST_APP"
    23  
    24  # SSH key name used for testing
    25  export DEIS_TEST_AUTH_KEY=${DEIS_TEST_AUTH_KEY:-deis-test}
    26  echo "DEIS_TEST_AUTH_KEY=$DEIS_TEST_AUTH_KEY"
    27  
    28  # SSH key used for deisctl tunneling
    29  export DEIS_TEST_SSH_KEY=${DEIS_TEST_SSH_KEY:-~/.vagrant.d/insecure_private_key}
    30  echo "DEIS_TEST_SSH_KEY=$DEIS_TEST_SSH_KEY"
    31  
    32  # domain used for wildcard DNS
    33  export DEIS_TEST_DOMAIN=${DEIS_TEST_DOMAIN:-local3.deisapp.com}
    34  echo "DEIS_TEST_DOMAIN=$DEIS_TEST_DOMAIN"
    35  
    36  # SSH tunnel used by deisctl
    37  export DEISCTL_TUNNEL=${DEISCTL_TUNNEL:-127.0.0.1:2222}
    38  echo "DEISCTL_TUNNEL=$DEISCTL_TUNNEL"
    39  
    40  # set units used by deisctl
    41  export DEISCTL_UNITS=${DEISCTL_UNITS:-$DEIS_ROOT/deisctl/units}
    42  echo "DEISCTL_UNITS=$DEISCTL_UNITS"
    43  
    44  # ip address for docker containers to communicate in functional tests
    45  export HOST_IPADDR=${HOST_IPADDR?}
    46  echo "HOST_IPADDR=$HOST_IPADDR"
    47  
    48  # the registry used to host dev-release images
    49  # must be accessible to local Docker engine and Deis cluster
    50  export DEV_REGISTRY=${DEV_REGISTRY?}
    51  echo "DEV_REGISTRY=$DEV_REGISTRY"
    52  
    53  # random 10-char (5-byte) hex string to identify a test run
    54  export DEIS_TEST_ID=${DEIS_TEST_ID:-$(openssl rand -hex 5)}
    55  echo "DEIS_TEST_ID=$DEIS_TEST_ID"
    56  
    57  # give this session a unique ~/.deis/<client>.json file
    58  export DEIS_PROFILE=test-$DEIS_TEST_ID
    59  rm -f $HOME/.deis/test-$DEIS_TEST_ID.json
    60  
    61  # bail if registry is not accessible
    62  if ! curl -s $DEV_REGISTRY; then
    63    echo "DEV_REGISTRY is not accessible, exiting..."
    64    exit 1
    65  fi
    66  echo ; echo
    67  
    68  # disable git+ssh host key checking
    69  export GIT_SSH=$DEIS_ROOT/tests/bin/git-ssh-nokeycheck.sh
    70  
    71  # install required go dependencies
    72  go get -v github.com/golang/lint/golint
    73  go get -v github.com/tools/godep
    74  
    75  # cleanup any stale example applications
    76  rm -rf $DEIS_ROOT/tests/example-*
    77  
    78  # generate ssh keys if they don't already exist
    79  test -e ~/.ssh/$DEIS_TEST_AUTH_KEY || ssh-keygen -t rsa -f ~/.ssh/$DEIS_TEST_AUTH_KEY -N ''
    80  # TODO: parameterize this key required for keys_test.go?
    81  test -e ~/.ssh/deiskey || ssh-keygen -q -t rsa -f ~/.ssh/deiskey -N '' -C deiskey
    82  
    83  # prepare the SSH agent
    84  ssh-add -D || eval $(ssh-agent) && ssh-add -D
    85  ssh-add ~/.ssh/$DEIS_TEST_AUTH_KEY
    86  ssh-add $DEIS_TEST_SSH_KEY
    87  
    88  # wipe out all vagrants & deis virtualboxen
    89  function cleanup {
    90      if [ "$SKIP_CLEANUP" != true ]; then
    91          log_phase "Cleaning up"
    92          set +e
    93          ${GOPATH}/src/github.com/deis/deis/tests/bin/destroy-all-vagrants.sh
    94          VBoxManage list vms | grep deis | sed -n -e 's/^.* {\(.*\)}/\1/p' | xargs -L1 -I {} VBoxManage unregistervm {} --delete
    95          vagrant global-status --prune
    96          docker rm -f -v `docker ps | grep deis- | awk '{print $1}'` 2>/dev/null
    97          log_phase "Test run complete"
    98      fi
    99  }
   100  
   101  function dump_logs {
   102    log_phase "Error detected, dumping logs"
   103    TIMESTAMP=`date +%Y-%m-%d-%H%M%S`
   104    FAILED_LOGS_DIR=$HOME/deis-test-failure-$TIMESTAMP
   105    mkdir -p $FAILED_LOGS_DIR
   106    set +e
   107    export FLEETCTL_TUNNEL=$DEISCTL_TUNNEL
   108    fleetctl -strict-host-key-checking=false list-units
   109    # application unit logs
   110    for APP in `fleetctl -strict-host-key-checking=false list-units --no-legend --fields=unit | grep -v "deis-"`;do
   111      CURRENT_APP=$(echo $APP | sed "s/\.service//")
   112      #echo "$CURRENT_APP"
   113      get_journal_logs $CURRENT_APP
   114    done
   115    # etcd keyspace
   116    get_logs deis-controller "etcdctl ls / --recursive" etcdctl-dump
   117    # component logs
   118    get_logs deis-builder
   119    get_logs deis-controller
   120    get_logs deis-database
   121    get_logs deis-logger
   122    get_logs deis-registry@1 deis-registry deis-registry-1
   123    get_logs deis-router@1 deis-router deis-router-1
   124    get_logs deis-router@2 deis-router deis-router-2
   125    get_logs deis-router@3 deis-router deis-router-3
   126    # deis-store logs
   127    get_logs deis-router@1 deis-store-monitor deis-store-monitor-1
   128    get_logs deis-router@1 deis-store-daemon deis-store-daemon-1
   129    get_logs deis-router@1 deis-store-metadata deis-store-metadata-1
   130    get_logs deis-router@1 deis-store-volume deis-store-volume-1
   131    get_logs deis-router@2 deis-store-monitor deis-store-monitor-2
   132    get_logs deis-router@2 deis-store-daemon deis-store-daemon-2
   133    get_logs deis-router@2 deis-store-metadata deis-store-metadata-2
   134    get_logs deis-router@2 deis-store-volume deis-store-volume-2
   135    get_logs deis-router@3 deis-store-monitor deis-store-monitor-3
   136    get_logs deis-router@3 deis-store-daemon deis-store-daemon-3
   137    get_logs deis-router@3 deis-store-metadata deis-store-metadata-3
   138    get_logs deis-router@3 deis-store-volume deis-store-volume-3
   139    get_logs deis-store-gateway
   140  
   141    # get debug-etcd logs
   142    fleetctl -strict-host-key-checking=false ssh deis-router@1 journalctl --no-pager -u etcd \
   143      > $FAILED_LOGS_DIR/debug-etcd-1.log
   144    fleetctl -strict-host-key-checking=false ssh deis-router@2 journalctl --no-pager -u etcd \
   145      > $FAILED_LOGS_DIR/debug-etcd-2.log
   146    fleetctl -strict-host-key-checking=false ssh deis-router@3 journalctl --no-pager -u etcd \
   147      > $FAILED_LOGS_DIR/debug-etcd-3.log
   148  
   149    # tarball logs
   150    BUCKET=jenkins-failure-logs
   151    FILENAME=deis-test-failure-$TIMESTAMP.tar.gz
   152    cd $FAILED_LOGS_DIR && tar -czf $FILENAME *.log && mv $FILENAME .. && cd ..
   153    rm -rf $FAILED_LOGS_DIR
   154    if [ `which s3cmd` ] && [ -f $HOME/.s3cfg ]; then
   155      echo "configured s3cmd found in path. Attempting to upload logs to S3"
   156      s3cmd put $HOME/$FILENAME s3://$BUCKET
   157      rm $HOME/$FILENAME
   158      echo "Logs are accessible at https://s3.amazonaws.com/$BUCKET/$FILENAME"
   159    else
   160      echo "Logs are accessible at $HOME/$FILENAME"
   161    fi
   162    exit 1
   163  }
   164  
   165  function get_logs {
   166    TARGET="$1"
   167    CONTAINER="$2"
   168    FILENAME="$3"
   169    if [ -z "$CONTAINER" ]; then
   170      CONTAINER=$TARGET
   171    fi
   172    if [ -z "$FILENAME" ]; then
   173      FILENAME=$TARGET
   174    fi
   175    fleetctl -strict-host-key-checking=false ssh "$TARGET" docker logs "$CONTAINER" > $FAILED_LOGS_DIR/$FILENAME.log
   176  }
   177  
   178  function get_journal_logs {
   179    TARGET="$1"
   180    fleetctl -strict-host-key-checking=false journal --lines=1000 "$TARGET" > $FAILED_LOGS_DIR/$TARGET.log
   181  }