github.com/verrazzano/verrazzano@v1.7.0/ci/rolebased/Jenkinsfile (about)

     1  // Copyright (c) 2020, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def SKIP_ACCEPTANCE_TESTS = false
     6  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     7  
     8  pipeline {
     9      options {
    10          skipDefaultCheckout true
    11      }
    12  
    13      agent {
    14         docker {
    15              image "${RUNNER_DOCKER_IMAGE}"
    16              args "${RUNNER_DOCKER_ARGS}"
    17              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    18              registryCredentialsId 'ocir-pull-and-push-account'
    19              label "${agentLabel}"
    20          }
    21      }
    22  
    23      parameters {
    24          booleanParam (description: 'Whether to kick off acceptance test run at the end of this build', name: 'RUN_ACCEPTANCE_TESTS', defaultValue: true)
    25          booleanParam (description: 'Whether to run example tests', name: 'RUN_EXAMPLE_TESTS', defaultValue: true)
    26          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    27                  defaultValue: 'NONE',
    28                  description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator',
    29                  trim: true)
    30          choice (name: 'WILDCARD_DNS_DOMAIN',
    31                  description: 'This is the wildcard DNS domain',
    32                  // 1st choice is the default value
    33                  choices: [ "nip.io", "sslip.io"])
    34          choice (name: 'CRD_API_VERSION',
    35                  description: 'This is the API crd version.',
    36                  // 1st choice is the default value
    37                  choices: [ "v1beta1", "v1alpha1"])
    38          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    39          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    40      }
    41  
    42      environment {
    43          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    44          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    45          DOCKER_PLATFORM_IMAGE_NAME = "${env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' ? env.DOCKER_PLATFORM_PUBLISH_IMAGE_NAME : env.DOCKER_PLATFORM_CI_IMAGE_NAME}"
    46          DOCKER_OAM_CI_IMAGE_NAME = 'verrazzano-application-operator-jenkins'
    47          DOCKER_OAM_PUBLISH_IMAGE_NAME = 'verrazzano-application-operator'
    48          DOCKER_OAM_IMAGE_NAME = "${env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' ? env.DOCKER_OAM_PUBLISH_IMAGE_NAME : env.DOCKER_OAM_CI_IMAGE_NAME}"
    49          CREATE_LATEST_TAG = "${env.BRANCH_NAME == 'master' ? '1' : '0'}"
    50          GOPATH = '/home/opc/go'
    51          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    52          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    53          DOCKER_EMAIL = credentials('github-packages-email')
    54          DOCKER_REPO = 'ghcr.io'
    55          DOCKER_NAMESPACE = 'verrazzano'
    56          NETRC_FILE = credentials('netrc')
    57          SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY')
    58  
    59          CLUSTER_NAME = 'verrazzano'
    60          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    61          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    62          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    63          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    64  
    65          // Environment variable for Verrazzano CLI executable
    66          VZ_COMMAND="${GO_REPO_PATH}/vz"
    67  
    68          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    69          OCR_CREDS = credentials('ocr-pull-and-push-account')
    70          OCR_REPO = 'container-registry.oracle.com'
    71          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    72          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
    73          INSTALL_PROFILE = "dev"
    74          VZ_ENVIRONMENT_NAME = "default"
    75          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    76  
    77          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test
    78          DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test
    79  
    80          // used to emit metrics
    81          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
    82          TEST_ENV = "KIND"
    83          SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
    84          SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
    85          SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
    86      }
    87  
    88      stages {
    89          stage('Clean workspace and checkout') {
    90              environment {
    91                  OCI_CLI_AUTH="instance_principal"
    92                  OCI_OS_NAMESPACE = credentials('oci-os-namespace')
    93                  OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
    94                  VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
    95              }
    96              steps {
    97                  sh """
    98                      echo "${NODE_LABELS}"
    99                  """
   100  
   101                  script {
   102                      checkout scm
   103                  }
   104                  sh """
   105                      cp -f "${NETRC_FILE}" $HOME/.netrc
   106                      chmod 600 $HOME/.netrc
   107                  """
   108  
   109                  script {
   110                      try {
   111                          sh """
   112                              echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   113                          """
   114                      } catch(error) {
   115                          echo "docker login failed, retrying after sleep"
   116                          retry(4) {
   117                              sleep(30)
   118                              sh """
   119                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   120                              """
   121                          }
   122                      }
   123                  }
   124                  sh """
   125                      rm -rf ${GO_REPO_PATH}/verrazzano
   126                      mkdir -p ${GO_REPO_PATH}/verrazzano
   127                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   128                  """
   129  
   130                  script {
   131                      def props = readProperties file: '.verrazzano-development-version'
   132                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   133                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   134                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   135                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   136                  }
   137                  script {
   138                      sh """
   139                          echo "Downloading VZ CLI from object storage"
   140                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   141                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   142                          ${GO_REPO_PATH}/vz version
   143                      """
   144                  }
   145              }
   146          }
   147  
   148          stage('Acceptance Tests') {
   149  
   150              stages {
   151  
   152                  stage('Prepare AT environment') {
   153                      environment {
   154                          VERRAZZANO_OPERATOR_IMAGE="NONE"
   155                          KIND_KUBERNETES_CLUSTER_VERSION="1.24"
   156                          OCI_CLI_AUTH="instance_principal"
   157                          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   158                          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   159                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   160                      }
   161                      steps {
   162                          sh """
   163                              cd ${GO_REPO_PATH}/verrazzano
   164                              ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
   165                          """
   166                      }
   167                      post {
   168                          always {
   169                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true
   170                          }
   171                          failure {
   172                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   173                          }
   174                      }
   175                  }
   176  
   177                  stage('Run Acceptance Tests') {
   178                      environment {
   179                          PROJECT_ADMIN_ROLE = "verrazzano-project-admin"
   180                          TEST_ROLE = "verrazzano-test-role"
   181                      }
   182                      stages {
   183                          stage ('Create test role') {
   184                              steps {
   185                                  script {
   186                                      sh """
   187                                          cd ${GO_REPO_PATH}/verrazzano
   188                                          kubectl apply -f ./tools/scripts/testrole.yaml || true
   189                                      """
   190                                  }
   191                              }
   192                          }
   193                          stage ('Run Tests') {
   194                              parallel {
   195                                  stage('verify-install istio') {
   196                                      environment {
   197                                          TEST_ID = UUID.randomUUID().toString()
   198                                          TEST_NAMESPACE = "installistio"
   199                                          TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}"
   200                                      }
   201                                      steps {
   202                                          runGinkgoWithKubeClient('verify-install/istio')
   203                                      }
   204                                      post {
   205                                          always {
   206                                              sh """
   207                                                  cd ${GO_REPO_PATH}/verrazzano
   208                                                  ./tools/scripts/remove-test-kube-client.sh
   209                                              """
   210                                          }
   211                                      }
   212                                  }
   213                                  stage('verify-install web') {
   214                                      environment {
   215                                          TEST_ID = UUID.randomUUID().toString()
   216                                          TEST_NAMESPACE = "installweb"
   217                                          TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}"
   218                                      }
   219                                      steps {
   220                                          runGinkgoWithKubeClient('verify-install/web')
   221                                      }
   222                                      post {
   223                                          always {
   224                                              sh """
   225                                                  cd ${GO_REPO_PATH}/verrazzano
   226                                                  ./tools/scripts/remove-test-kube-client.sh
   227                                              """
   228                                          }
   229                                      }
   230                                  }
   231                                  stage('verify-install verrazzano') {
   232                                      steps {
   233                                          runGinkgoRandomize('verify-install/verrazzano')
   234                                      }
   235                                  }
   236                                  stage('verify-install kubernetes') {
   237                                      steps {
   238                                          runGinkgoRandomize('verify-install/kubernetes')
   239                                      }
   240                                  }
   241                                  stage('verify-infra restapi') {
   242                                      environment {
   243                                          TEST_ID = UUID.randomUUID().toString()
   244                                          TEST_NAMESPACE = "infrarestapi"
   245                                          TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}"
   246                                      }
   247                                      steps {
   248                                          runGinkgoWithKubeClient('verify-infra/restapi')
   249                                      }
   250                                      post {
   251                                          always {
   252                                              sh """
   253                                                  cd ${GO_REPO_PATH}/verrazzano
   254                                                  ./tools/scripts/remove-test-kube-client.sh
   255                                              """
   256                                          }
   257                                      }
   258                                  }
   259                                  stage('verify-infra oam') {
   260                                      environment {
   261                                          TEST_ID = UUID.randomUUID().toString()
   262                                          TEST_NAMESPACE = "infraoam"
   263                                          TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}"
   264                                      }
   265                                      steps {
   266                                          runGinkgoWithKubeClient('verify-infra/oam')
   267                                      }
   268                                      post {
   269                                          always {
   270                                              sh """
   271                                                  cd ${GO_REPO_PATH}/verrazzano
   272                                                  ./tools/scripts/remove-test-kube-client.sh
   273                                              """
   274                                          }
   275                                      }
   276                                  }
   277                                  stage('verify-infra vmi') {
   278                                      steps {
   279                                          runGinkgoRandomize('verify-infra/vmi')
   280                                      }
   281                                  }
   282                                  stage('examples todo') {
   283                                      when {
   284                                          expression {params.RUN_EXAMPLE_TESTS == true}
   285                                      }
   286                                      steps {
   287                                          runGinkgo('examples/todo')
   288                                      }
   289                                  }
   290                                  stage('examples socks') {
   291                                      environment {
   292                                          TEST_ID = UUID.randomUUID().toString()
   293                                          TEST_NAMESPACE = "sockshop"
   294                                          TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}"
   295                                      }
   296                                      when {
   297                                          expression {params.RUN_EXAMPLE_TESTS == true}
   298                                      }
   299                                      steps {
   300                                          runGinkgoWithKubeClient('examples/socks')
   301                                      }
   302                                      post {
   303                                          always {
   304                                              sh """
   305                                                  cd ${GO_REPO_PATH}/verrazzano
   306                                                  ./tools/scripts/remove-test-kube-client.sh
   307                                              """
   308                                          }
   309                                      }
   310                                  }
   311                                  stage('examples springboot') {
   312                                      environment {
   313                                          TEST_ID = UUID.randomUUID().toString()
   314                                          TEST_NAMESPACE = "springboot"
   315                                          TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}"
   316                                      }
   317                                      when {
   318                                          expression {params.RUN_EXAMPLE_TESTS == true}
   319                                      }
   320                                      steps {
   321                                          runGinkgoWithKubeClient('examples/springboot')
   322                                      }
   323                                      post {
   324                                          always {
   325                                              sh """
   326                                                  cd ${GO_REPO_PATH}/verrazzano
   327                                                  ./tools/scripts/remove-test-kube-client.sh
   328                                              """
   329                                          }
   330                                      }
   331                                  }
   332                                  stage('examples helidon') {
   333                                      environment {
   334                                          TEST_ID = UUID.randomUUID().toString()
   335                                          TEST_NAMESPACE = "hello-helidon"
   336                                          TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}"
   337                                      }
   338                                      when {
   339                                          expression {params.RUN_EXAMPLE_TESTS == true}
   340                                      }
   341                                      steps {
   342                                          runGinkgoWithKubeClient('examples/helidon')
   343                                      }
   344                                      post {
   345                                          always {
   346                                              sh """
   347                                                  cd ${GO_REPO_PATH}/verrazzano
   348                                                  ./tools/scripts/remove-test-kube-client.sh
   349                                              """
   350                                          }
   351                                      }
   352                                  }
   353                                  stage('examples bobs books') {
   354                                      when {
   355                                          expression {params.RUN_EXAMPLE_TESTS == true}
   356                                      }
   357                                      steps {
   358                                          runGinkgo('examples/bobsbooks')
   359                                      }
   360                                  }
   361                              }
   362                          }
   363                      }
   364                      post {
   365                          always {
   366                              sh """
   367                                  cd ${GO_REPO_PATH}/verrazzano
   368                                  kubectl delete -f ./tools/scripts/testrole.yaml || true
   369                              """
   370  
   371                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true
   372                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   373                          }
   374                      }
   375                  }
   376              }
   377  
   378              post {
   379                  failure {
   380                      script {
   381                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   382                              dumpK8sCluster('new-acceptance-tests-cluster-snapshot')
   383                          }
   384                      }
   385                  }
   386                  success {
   387                      script {
   388                          if (params.DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   389                              dumpK8sCluster('new-acceptance-tests-cluster-snapshot')
   390                          }
   391                      }
   392                  }
   393              }
   394          }
   395      }
   396  
   397      post {
   398          always {
   399              script {
   400                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   401                      dumpVerrazzanoSystemPods()
   402                      dumpCattleSystemPods()
   403                      dumpNginxIngressControllerLogs()
   404                      dumpVerrazzanoPlatformOperatorLogs()
   405                      dumpVerrazzanoApplicationOperatorLogs()
   406                      dumpOamKubernetesRuntimeLogs()
   407                      dumpVerrazzanoApiLogs()
   408                  }
   409              }
   410              archiveArtifacts artifacts: '**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**', allowEmptyArchive: true
   411              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   412  
   413              sh """
   414                  cd ${GO_REPO_PATH}/verrazzano/platform-operator
   415                  make delete-cluster
   416                  if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   417                    echo "Failures seen during dumping of artifacts, treat post as failed"
   418                    exit 1
   419                  fi
   420              """
   421              deleteDir()
   422          }
   423      }
   424  }
   425  
   426  def runGinkgoRandomize(testSuitePath) {
   427      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   428          sh """
   429              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   430              ginkgo -p --randomize-all -v --keep-going --no-color ${testSuitePath}/...
   431          """
   432      }
   433  }
   434  
   435  def runGinkgo(testSuitePath) {
   436      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   437          sh """
   438              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   439              ginkgo -v --keep-going --no-color ${testSuitePath}/...
   440          """
   441      }
   442  }
   443  
   444  def dumpK8sCluster(dumpDirectory) {
   445      sh """
   446          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   447      """
   448  }
   449  
   450  def dumpVerrazzanoSystemPods() {
   451      sh """
   452          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   453          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log"
   454          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   455          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log"
   456          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   457          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log"
   458          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   459          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log"
   460          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   461      """
   462  }
   463  
   464  def dumpCattleSystemPods() {
   465      sh """
   466          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   467          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/cattle-system-pods.log"
   468          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   469          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/rancher.log"
   470          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   471      """
   472  }
   473  
   474  def dumpNginxIngressControllerLogs() {
   475      sh """
   476          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   477          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log"
   478          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   479      """
   480  }
   481  
   482  def dumpVerrazzanoPlatformOperatorLogs() {
   483      sh """
   484          ## dump out verrazzano-platform-operator logs
   485          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   486          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   487          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   488          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   489          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   490          echo "------------------------------------------"
   491      """
   492  }
   493  
   494  def dumpVerrazzanoApplicationOperatorLogs() {
   495      sh """
   496          ## dump out verrazzano-application-operator logs
   497          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   498          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   499          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   500          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   501          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   502          echo "------------------------------------------"
   503      """
   504  }
   505  
   506  def dumpOamKubernetesRuntimeLogs() {
   507      sh """
   508          ## dump out oam-kubernetes-runtime logs
   509          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   510          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   511          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   512          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   513          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   514          echo "------------------------------------------"
   515      """
   516  }
   517  
   518  def dumpVerrazzanoApiLogs() {
   519      sh """
   520          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   521          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log"
   522          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   523      """
   524  }
   525  
   526  def runGinkgoWithKubeClient(testSuitePath) {
   527      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   528          sh """
   529              cd ${GO_REPO_PATH}/verrazzano
   530              ./tools/scripts/create-test-kube-client.sh
   531              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   532              ginkgo -v --keep-going --no-color ${testSuitePath}/...
   533          """
   534      }
   535  }