github.com/verrazzano/verrazzano@v1.7.1/ci/examples/Jenkinsfile (about)

     1  // Copyright (c) 2020, 2024, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          skipDefaultCheckout true
    11          copyArtifactPermission('*');
    12          timestamps ()
    13      }
    14  
    15      agent {
    16         docker {
    17              image "${RUNNER_DOCKER_IMAGE}"
    18              args "${RUNNER_DOCKER_ARGS}"
    19              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    20              registryCredentialsId 'ocir-pull-and-push-account'
    21              label "${agentLabel}"
    22          }
    23      }
    24  
    25      parameters {
    26          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    27                  description: 'Kubernetes Version for KinD Cluster',
    28                  // 1st choice is the default value
    29                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    30          string (name: 'GIT_COMMIT_TO_USE',
    31                          defaultValue: 'NONE',
    32                          description: 'This is the full git commit hash from the source build to be used for all jobs',
    33                          trim: true)
    34          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    35                          defaultValue: 'NONE',
    36                          description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    37                          trim: true)
    38          choice (name: 'WILDCARD_DNS_DOMAIN',
    39                  description: 'This is the wildcard DNS domain',
    40                  // 1st choice is the default value
    41                  choices: [ "nip.io", "sslip.io"])
    42          choice (name: 'CRD_API_VERSION',
    43                  description: 'This is the API crd version.',
    44                  // 1st choice is the default value
    45                  choices: [ "v1beta1", "v1alpha1"])
    46          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    47          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    48          booleanParam (description: 'Whether to include the slow tests in the acceptance tests', name: 'RUN_SLOW_TESTS', defaultValue: true)
    49          string (name: 'CONSOLE_REPO_BRANCH',
    50                  defaultValue: '',
    51                  description: 'The branch to check out after cloning the console repository.',
    52                  trim: true)
    53          booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true)
    54          string (name: 'TAGGED_TESTS',
    55                  defaultValue: '',
    56                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    57                  trim: true)
    58          string (name: 'INCLUDED_TESTS',
    59                  defaultValue: '.*',
    60                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    61                  trim: true)
    62          string (name: 'EXCLUDED_TESTS',
    63                  defaultValue: '_excluded_test',
    64                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    65                  trim: true)
    66          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    67      }
    68  
    69      environment {
    70          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    71          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    72          GOPATH = '/home/opc/go'
    73          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    74          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    75          DOCKER_EMAIL = credentials('github-packages-email')
    76          DOCKER_REPO = 'ghcr.io'
    77          DOCKER_NAMESPACE = 'verrazzano'
    78          NETRC_FILE = credentials('netrc')
    79          CLUSTER_NAME = 'verrazzano'
    80          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    81          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    82          IMG_LIST_FILE = "${WORKSPACE}/verrazzano_images.txt"
    83          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    84          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    85          OCR_CREDS = credentials('ocr-pull-and-push-account')
    86          OCR_REPO = 'container-registry.oracle.com'
    87          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    88          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
    89          INSTALL_PROFILE = "dev"
    90          VZ_ENVIRONMENT_NAME = "default"
    91          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    92          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    93  
    94          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test
    95          DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test
    96  
    97          // Environment variables required to capture cluster snapshot and bug report on test failure
    98          DUMP_KUBECONFIG="${KUBECONFIG}"
    99          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   100          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
   101          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   102  
   103          // Environment variable for Verrazzano CLI executable
   104          VZ_COMMAND="${GO_REPO_PATH}/vz"
   105  
   106          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   107          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   108  
   109          // used for console artifact capture on failure
   110          JENKINS_READ = credentials('jenkins-auditor')
   111          OCI_CLI_AUTH="instance_principal"
   112          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   113          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   114          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   115  
   116          // used to emit metrics
   117          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   118          TEST_ENV_LABEL = "kind"
   119          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   120          TEST_ENV = "KIND"
   121  
   122          // used to generate Ginkgo test reports
   123          TEST_REPORT = "test-report.xml"
   124          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   125          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   126      }
   127  
   128      stages {
   129          stage('Clean workspace and checkout') {
   130              steps {
   131                  sh """
   132                      echo "${NODE_LABELS}"
   133                  """
   134  
   135                  script {
   136                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   137                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   138                          echo "Specific GIT commit was not specified, use current head"
   139                          def scmInfo = checkout scm
   140                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   141                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   142                      } else {
   143                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   144                          def scmInfo = checkout([
   145                              $class: 'GitSCM',
   146                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   147                              doGenerateSubmoduleConfigurations: false,
   148                              extensions: [],
   149                              submoduleCfg: [],
   150                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   151                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   152                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   153                          // If the commit we were handed is not what the SCM says we are using, fail
   154                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   155                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   156                              exit 1
   157                          }
   158                      }
   159                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   160                  }
   161  
   162                  sh """
   163                      cp -f "${NETRC_FILE}" $HOME/.netrc
   164                      chmod 600 $HOME/.netrc
   165                  """
   166  
   167                  script {
   168                      try {
   169                      sh """
   170                          echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   171                      """
   172                      } catch(error) {
   173                          echo "docker login failed, retrying after sleep"
   174                          retry(4) {
   175                              sleep(30)
   176                              sh """
   177                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   178                              """
   179                          }
   180                      }
   181                  }
   182                  sh """
   183                      rm -rf ${GO_REPO_PATH}/verrazzano
   184                      mkdir -p ${GO_REPO_PATH}/verrazzano
   185                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   186                  """
   187  
   188                  script {
   189                      def props = readProperties file: '.verrazzano-development-version'
   190                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   191                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   192                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   193                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   194                      // update the description with some meaningful info
   195                      setDisplayName()
   196                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   197                  }
   198                  script {
   199                      sh """
   200                          echo "Downloading VZ CLI from object storage"
   201                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   202                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   203                          ${GO_REPO_PATH}/vz version
   204                      """
   205                  }
   206              }
   207          }
   208  
   209          stage('Acceptance Tests') {
   210  
   211              stages {
   212                  stage('Prepare AT environment') {
   213                      environment {
   214                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   215                          OCI_CLI_AUTH="instance_principal"
   216                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   217                      }
   218                      steps {
   219                          sh """
   220                              cd ${GO_REPO_PATH}/verrazzano
   221                              ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
   222                          """
   223                      }
   224                      post {
   225                          failure {
   226                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   227                          }
   228                          success {
   229                              sh """
   230                                  oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${env.BRANCH_NAME}/generated-verrazzano-bom.json --file ${WORKSPACE}/downloaded-verrazzano-bom.json
   231                                  ${GO_REPO_PATH}/verrazzano/tools/scripts/generate_image_list.sh $WORKSPACE/downloaded-verrazzano-bom.json $WORKSPACE/generated_verrazzano_images.txt
   232                                  cat $WORKSPACE/generated_verrazzano_images.txt | cut -d/ -f2- | sort -u > $WORKSPACE/verrazzano_images.txt
   233                              """
   234                          }
   235                          always {
   236                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true
   237                              // enable debug logging of Verrazzano api istio proxy
   238                              script {
   239                                  if (params.ENABLE_API_ENVOY_LOGGING) {
   240                                      sh '''
   241                                          vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\")
   242                                          if [ -z "\$vz_api_pod" ]; then
   243                                            echo "Could not find verrazzano-authproxy pod, not enabling debug logging"
   244                                          else
   245                                            kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug
   246                                          fi
   247                                          nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\")
   248                                          if [ -z "\$nginx_ing_pod" ]; then
   249                                            echo "Could not find nginx ingress controller pod, not enabling debug logging"
   250                                          else
   251                                            kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug
   252                                          fi
   253                                      '''
   254                                  }
   255                              }
   256                          }
   257                      }
   258                  }
   259  
   260                  stage('Verify Install') {
   261                      environment {
   262                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install"
   263                      }
   264                      steps {
   265                          runGinkgoRandomize('verify-install')
   266                      }
   267                      post {
   268                          always {
   269                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   270                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   271                          }
   272                      }
   273                  }
   274  
   275                  stage('Run Acceptance Tests Deployments') {
   276                      parallel {
   277                          stage('examples todo') {
   278                              environment {
   279                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-todo"
   280                              }
   281                              steps {
   282                                  runGinkgo('examples/todo')
   283                              }
   284                          }
   285                          stage('examples socks') {
   286                              environment {
   287                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-socks"
   288                              }
   289                              steps {
   290                                  runSocksVariant('helidon')
   291                                  runSocksVariant('micronaut')
   292                                  runSocksVariant('spring')
   293                              }
   294                          }
   295                          stage('examples springboot') {
   296                              environment {
   297                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-spring"
   298                              }
   299                              steps {
   300                                  runGinkgo('examples/springboot')
   301                              }
   302                          }
   303                          stage('examples helidon 1') {
   304                              environment {
   305                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-1"
   306                              }
   307                              steps {
   308                                  runGinkgo('examples/helidon')
   309                              }
   310                          }
   311                          stage('examples helidon 2') {
   312                              environment {
   313                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-2"
   314                              }
   315                              steps {
   316                                  runGinkgo('examples/helidon')
   317                              }
   318                          }
   319                          stage('examples helidon-config') {
   320                              environment {
   321                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-config"
   322                              }
   323                              steps {
   324                                  runGinkgo('examples/helidonconfig')
   325                              }
   326                          }
   327                          stage('examples bobs books') {
   328                              environment {
   329                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-bobs"
   330                              }
   331                              steps {
   332                                  runGinkgo('examples/bobsbooks')
   333                              }
   334                          }
   335                          stage('examples weblogic cluster') {
   336                              environment {
   337                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-cluster"
   338                              }
   339                              steps {
   340                                  runGinkgo('workloads/weblogic-cluster')
   341                              }
   342                          }
   343                      }
   344                  }
   345              }
   346  
   347              post {
   348                  failure {
   349                      script {
   350                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   351                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   352                          }
   353                      }
   354                  }
   355                  success {
   356                      script {
   357                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   358                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   359                          }
   360                      }
   361                  }
   362              }
   363          }
   364      }
   365  
   366      post {
   367          always {
   368              script {
   369                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   370                      dumpVerrazzanoSystemPods()
   371                      dumpCattleSystemPods()
   372                      dumpNginxIngressControllerLogs()
   373                      dumpVerrazzanoPlatformOperatorLogs()
   374                      dumpVerrazzanoApplicationOperatorLogs()
   375                      dumpOamKubernetesRuntimeLogs()
   376                      dumpVerrazzanoApiLogs()
   377                  }
   378              }
   379  
   380              sh """
   381                  # Copy the generated test reports to WORKSPACE to archive them
   382                  mkdir -p ${TEST_REPORT_DIR}
   383                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   384                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   385  
   386                  # Append the images for the examples to verrazzano_images.txt
   387                  ${WORKSPACE}/tests/e2e/config/scripts/get_examples_image.sh ${WORKSPACE}/examples/todo-list/todo-list-components.yaml ${IMG_LIST_FILE}
   388                  ${WORKSPACE}/tests/e2e/config/scripts/get_examples_image.sh ${WORKSPACE}/examples/springboot-app/springboot-comp.yaml ${IMG_LIST_FILE}
   389                  ${WORKSPACE}/tests/e2e/config/scripts/get_examples_image.sh ${WORKSPACE}/examples/hello-helidon/hello-helidon-comp.yaml ${IMG_LIST_FILE}
   390                  ${WORKSPACE}/tests/e2e/config/scripts/get_examples_image.sh ${WORKSPACE}/examples/bobs-books/bobs-books-comp.yaml ${IMG_LIST_FILE}
   391  
   392                  # Rename example-bobs-books-order-manager to example-bobs-bookstore-order-manager to match existing image repository
   393                  sed -i 's/example-bobs-books-order-manager/example-bobs-bookstore-order-manager/' ${IMG_LIST_FILE}
   394  
   395                  echo "sorting the images file prior to archiving"
   396                  if [ -f ${IMG_LIST_FILE} ];
   397                  then
   398                   sort -u -o ${IMG_LIST_FILE} ${IMG_LIST_FILE}
   399                  fi
   400              """
   401              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/Screenshot*.png,**/${TEST_REPORT}", allowEmptyArchive: true
   402              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   403              deleteCluster()
   404          }
   405          failure {
   406              sh """
   407                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   408              """
   409              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   410              sh """
   411                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   412                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   413                  rm archive.zip
   414              """
   415          }
   416          cleanup {
   417              deleteDir()
   418          }
   419      }
   420  }
   421  
   422  def runGinkgoRandomize(testSuitePath, kubeConfig = '') {
   423      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   424          sh """
   425              if [ ! -z "${kubeConfig}" ]; then
   426                  export KUBECONFIG="${kubeConfig}"
   427              fi
   428              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   429              ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   430          """
   431      }
   432  }
   433  
   434  def runSocksVariant(variant) {
   435      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   436              sh """
   437                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   438                  SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}"  examples/socks/...
   439              """
   440          }
   441  }
   442  
   443  def runGinkgo(testSuitePath) {
   444      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   445          sh """
   446              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   447              ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   448          """
   449      }
   450  }
   451  
   452  def dumpK8sCluster(dumpDirectory) {
   453      sh """
   454          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   455      """
   456  }
   457  
   458  def dumpVerrazzanoSystemPods() {
   459      sh """
   460          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   461          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   462          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   463          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   464          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   465          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   466          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   467          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   468          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   469      """
   470  }
   471  
   472  def dumpCattleSystemPods() {
   473      sh """
   474          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   475          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   476          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   477          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   478          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   479      """
   480  }
   481  
   482  def dumpNginxIngressControllerLogs() {
   483      sh """
   484          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   485          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   486          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   487      """
   488  }
   489  
   490  def dumpVerrazzanoPlatformOperatorLogs() {
   491      sh """
   492          ## dump out verrazzano-platform-operator logs
   493          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   494          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   495          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   496          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   497          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   498          echo "------------------------------------------"
   499      """
   500  }
   501  
   502  def dumpVerrazzanoApplicationOperatorLogs() {
   503      sh """
   504          ## dump out verrazzano-application-operator logs
   505          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   506          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   507          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   508          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   509          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   510          echo "------------------------------------------"
   511      """
   512  }
   513  
   514  def dumpOamKubernetesRuntimeLogs() {
   515      sh """
   516          ## dump out oam-kubernetes-runtime logs
   517          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   518          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   519          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   520          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   521          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   522          echo "------------------------------------------"
   523      """
   524  }
   525  
   526  def dumpVerrazzanoApiLogs() {
   527      sh """
   528          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   529          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log"
   530          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   531      """
   532  }
   533  
   534  def getEffectiveDumpOnSuccess() {
   535      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   536      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   537          effectiveValue = true
   538          echo "Forcing dump on success based on global override setting"
   539      }
   540      return effectiveValue
   541  }
   542  
   543  def deleteCluster() {
   544      sh """
   545          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   546          make delete-cluster
   547          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   548            echo "Failures seen during dumping of artifacts, treat post as failed"
   549            exit 1
   550          fi
   551      """
   552  }
   553  
   554  def setDisplayName() {
   555      echo "Start setDisplayName"
   556      def causes = currentBuild.getBuildCauses()
   557      echo "causes: " + causes.toString()
   558      for (cause in causes) {
   559          def causeString = cause.toString()
   560          echo "current cause: " + causeString
   561          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   562               echo "This job was caused by " + causeString
   563               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   564                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   565               } else if (causeString.contains("verrazzano-flaky-tests")) {
   566                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   567               }
   568           }
   569      }
   570      echo "End setDisplayName"
   571  }