github.com/verrazzano/verrazzano@v1.7.1/ci/vz-analyze/Jenkinsfile (about)

     1  // Copyright (c) 2020, 2024, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          timeout(time: 1, unit: 'HOURS')
    11          skipDefaultCheckout true
    12          timestamps ()
    13      }
    14  
    15      agent {
    16         docker {
    17              image "${RUNNER_DOCKER_IMAGE}"
    18              args "${RUNNER_DOCKER_ARGS}"
    19              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    20              registryCredentialsId 'ocir-pull-and-push-account'
    21              label "${agentLabel}"
    22          }
    23      }
    24  
    25      parameters {
    26          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    27                  description: 'Kubernetes Version for KinD Cluster',
    28                  // 1st choice is the default value
    29                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    30          string (name: 'GIT_COMMIT_TO_USE',
    31                          defaultValue: 'NONE',
    32                          description: 'This is the full git commit hash from the source build to be used for all jobs',
    33                          trim: true)
    34          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    35                          defaultValue: 'NONE',
    36                          description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    37                          trim: true)
    38          choice (name: 'WILDCARD_DNS_DOMAIN',
    39                  description: 'This is the wildcard DNS domain',
    40                  // 1st choice is the default value
    41                  choices: [ "nip.io", "sslip.io"])
    42          choice (name: 'CRD_API_VERSION',
    43                  description: 'This is the API crd version.',
    44                  // 1st choice is the default value
    45                  choices: [ "v1beta1", "v1alpha1"])
    46          booleanParam (description: 'Whether to install Verrazzano using examples/ha/ha.yaml and 3 kind nodes (defaults to false)', name: 'INSTALL_HA', defaultValue: false)
    47          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    48          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    49          booleanParam (description: 'Whether to use a database for Grafana persistence', name: 'USE_DB_FOR_GRAFANA', defaultValue: false)
    50          string (name: 'CONSOLE_REPO_BRANCH',
    51                  defaultValue: '',
    52                  description: 'The branch to check out after cloning the console repository.',
    53                  trim: true)
    54          booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true)
    55          string (name: 'TAGGED_TESTS',
    56                  defaultValue: '',
    57                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    58                  trim: true)
    59          string (name: 'INCLUDED_TESTS',
    60                  defaultValue: '.*',
    61                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    62                  trim: true)
    63          string (name: 'EXCLUDED_TESTS',
    64                  defaultValue: '_excluded_test',
    65                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    66                  trim: true)
    67          booleanParam (description: 'Whether to run JWT tests', name: 'ENABLE_JWT_TESTING', defaultValue: false)
    68          booleanParam (description: 'Whether to run slow tests', name: 'RUN_SLOW_TESTS', defaultValue: false)
    69          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    70      }
    71  
    72      environment {
    73          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    74          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    75          GOPATH = '/home/opc/go'
    76          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    77          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    78          DOCKER_EMAIL = credentials('github-packages-email')
    79          DOCKER_REPO = 'ghcr.io'
    80          DOCKER_NAMESPACE = 'verrazzano'
    81          NETRC_FILE = credentials('netrc')
    82          CLUSTER_NAME = 'verrazzano'
    83          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    84          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    85          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    86          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    87          OCR_CREDS = credentials('ocr-pull-and-push-account')
    88          OCR_REPO = 'container-registry.oracle.com'
    89          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    90          INSTALL_CONFIG_FILE_KIND_TESTS = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
    91          INSTALL_CONFIG_FILE_KIND_HA = "./examples/ha/ha.yaml"
    92          INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/install-verrazzano.yaml"
    93          INSTALL_PROFILE = "${params.INSTALL_HA == true ? "prod" : "dev"}"
    94          KIND_NODE_COUNT = "${params.INSTALL_HA == true ? "3" : "1"}"
    95          VZ_ENVIRONMENT_NAME = "default"
    96          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    97          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    98  
    99          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test
   100          DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test
   101  
   102          // Environment variables required to capture cluster snapshot and bug report on test failure
   103          DUMP_KUBECONFIG="${KUBECONFIG}"
   104          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   105          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
   106          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   107  
   108          // Environment variable for Verrazzano CLI executable
   109          VZ_COMMAND="${GO_REPO_PATH}/vz"
   110  
   111          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   112          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   113  
   114          // used for console artifact capture on failure
   115          JENKINS_READ = credentials('jenkins-auditor')
   116          OCI_CLI_AUTH="instance_principal"
   117          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   118          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   119          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   120  
   121          // used to emit metrics from Ginkgo suites
   122          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   123          TEST_ENV_LABEL = "kind"
   124          TEST_ENV = "KIND"
   125          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   126  
   127          // used to generate Ginkgo test reports
   128          TEST_REPORT = "test-report.xml"
   129          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   130          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   131      }
   132  
   133      stages {
   134          stage('Clean workspace and checkout') {
   135              steps {
   136                  sh """
   137                      echo "${NODE_LABELS}"
   138                  """
   139  
   140                  script {
   141                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   142                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   143                          echo "Specific GIT commit was not specified, use current head"
   144                          def scmInfo = checkout scm
   145                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   146                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   147                      } else {
   148                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   149                          def scmInfo = checkout([
   150                              $class: 'GitSCM',
   151                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   152                              doGenerateSubmoduleConfigurations: false,
   153                              extensions: [],
   154                              submoduleCfg: [],
   155                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   156                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   157                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   158                          // If the commit we were handed is not what the SCM says we are using, fail
   159                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   160                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   161                              exit 1
   162                          }
   163                      }
   164                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   165                  }
   166  
   167                  sh """
   168                      cp -f "${NETRC_FILE}" $HOME/.netrc
   169                      chmod 600 $HOME/.netrc
   170                  """
   171  
   172                  script {
   173                      try {
   174                      sh """
   175                          echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   176                      """
   177                      } catch(error) {
   178                          echo "docker login failed, retrying after sleep"
   179                          retry(4) {
   180                              sleep(30)
   181                              sh """
   182                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   183                              """
   184                          }
   185                      }
   186                  }
   187                  sh """
   188                      rm -rf ${GO_REPO_PATH}/verrazzano
   189                      mkdir -p ${GO_REPO_PATH}/verrazzano
   190                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   191                  """
   192  
   193                  script {
   194                      def props = readProperties file: '.verrazzano-development-version'
   195                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   196                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   197                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   198                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   199                      // update the description with some meaningful info
   200                      setDisplayName()
   201                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   202                  }
   203                  script {
   204                      sh """
   205                          echo "Downloading VZ CLI from object storage"
   206                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   207                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   208                          ${GO_REPO_PATH}/vz version
   209                      """
   210                  }
   211              }
   212          }
   213  
   214          stage('Acceptance Tests') {
   215              stages {
   216                  stage('Prepare AT environment') {
   217                      environment {
   218                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   219                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   220                          REALM_USER_PASSWORD = credentials('todo-mysql-password')
   221                          REALM_NAME = "test-realm"
   222                      }
   223                      steps {
   224                          script {
   225                              if (params.INSTALL_HA) {
   226                                  sh """
   227                                      cp ${env.INSTALL_CONFIG_FILE_KIND_HA} ${env.INSTALL_CONFIG_FILE_KIND}
   228                                      cd ${GO_REPO_PATH}/verrazzano
   229                                      ci/scripts/prepare_ha_test_at_environment.sh ${env.INSTALL_CONFIG_FILE_KIND}
   230                                  """
   231                              } else {
   232                                  sh """
   233                                      cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND}
   234                                  """
   235                              }
   236                          }
   237                          sh """
   238                              cd ${GO_REPO_PATH}/verrazzano
   239                              ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA}
   240                          """
   241                          script {
   242                              if (params.ENABLE_JWT_TESTING) {
   243                                  sh """
   244                                      # setup test realm for JWT testing scenarios
   245                                      keycloakPassword=\$(kubectl get secret --namespace keycloak keycloak-http -o jsonpath={.data.password} | base64 --decode; echo)
   246                                      sed -i "s|##KEYCLOAK_PASSWORD##|\$keycloakPassword|g" ci/scripts/create_test_realm.sh
   247                                      sed -i "s|##REALM_USER_PASSWORD##|${env.REALM_USER_PASSWORD}|g" ci/scripts/create_test_realm.sh
   248                                      sed -i "s|##REALM_NAME##|${env.REALM_NAME}|g" ci/scripts/create_test_realm.sh
   249                                      kubectl exec keycloak-0 -n keycloak -- /bin/sh -c "`cat ci/scripts/create_test_realm.sh`"
   250                                      # setup request authentication policy
   251                                      keycloakURI=\$(kubectl get ingress -n keycloak keycloak -o jsonpath="{.spec.rules[0].host}")
   252                                      sed -i "s|##KEYCLOAK_URI##|\$keycloakURI|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml
   253                                      jwks=\$(curl -sk https://\$keycloakURI/auth/realms/test-realm/protocol/openid-connect/certs | jq -c 'del(.keys[] | select( .use == "enc"))')
   254                                      sed -i "s|##JWKS_KEY##|\$jwks|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml
   255                                      kubectl apply -f tests/testdata/jwt/helidon/test-realm-reqauth.yaml
   256                                  """
   257                              }
   258                          }
   259                      }
   260                      post {
   261                          failure {
   262                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   263                          }
   264                          always {
   265                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   266                              // enable debug logging of Verrazzano api istio proxy
   267                              script {
   268                                  if (params.ENABLE_API_ENVOY_LOGGING) {
   269                                      sh '''
   270                                          vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\")
   271                                          if [ -z "\$vz_api_pod" ]; then
   272                                            echo "Could not find verrazzano-authproxy pod, not enabling debug logging"
   273                                          else
   274                                            kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug
   275                                          fi
   276                                          nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\")
   277                                          if [ -z "\$nginx_ing_pod" ]; then
   278                                            echo "Could not find nginx ingress controller pod, not enabling debug logging"
   279                                          else
   280                                            kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug
   281                                          fi
   282                                      '''
   283                                  }
   284                              }
   285                          }
   286                      }
   287                  }
   288  
   289                  stage('Verify analyze-tool') {
   290                      environment {
   291                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-analyze-tool"
   292                      }
   293                      steps {
   294                          runGinkgo('verify-analyze-tool')
   295                      }
   296                      post {
   297                          always {
   298                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   299                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   300                          }
   301                      }
   302                  }
   303              }
   304  
   305              post {
   306                  aborted {
   307                      script {
   308                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   309                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   310                          }
   311                      }
   312                  }
   313                  failure {
   314                      script {
   315                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   316                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   317                          }
   318                      }
   319                  }
   320                  success {
   321                      script {
   322                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   323                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   324                          }
   325                      }
   326                  }
   327              }
   328          }
   329  
   330      }
   331  
   332      post {
   333          always {
   334              script {
   335                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   336                      dumpVerrazzanoSystemPods()
   337                      dumpCattleSystemPods()
   338                      dumpNginxIngressControllerLogs()
   339                      dumpVerrazzanoPlatformOperatorLogs()
   340                      dumpVerrazzanoApplicationOperatorLogs()
   341                      dumpOamKubernetesRuntimeLogs()
   342                      dumpVerrazzanoApiLogs()
   343                  }
   344              }
   345  
   346              sh """
   347                  # Copy the generated test reports to WORKSPACE to archive them
   348                  mkdir -p ${TEST_REPORT_DIR}
   349                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   350                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   351              """
   352              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   353              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   354              deleteCluster()
   355          }
   356          failure {
   357              sh """
   358                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   359              """
   360              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   361              sh """
   362                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   363                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   364                  rm archive.zip
   365              """
   366          }
   367          cleanup {
   368              deleteDir()
   369          }
   370      }
   371  }
   372  
   373  
   374  def runGinkgoRandomize(testSuitePath, kubeConfig = '') {
   375      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   376          sh """
   377              if [ ! -z "${kubeConfig}" ]; then
   378                  export KUBECONFIG="${kubeConfig}"
   379              fi
   380              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   381              if [ -d "${testSuitePath}" ]; then
   382                  ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   383              fi
   384          """
   385      }
   386  }
   387  
   388  def runSocksVariant(variant) {
   389      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   390              sh """
   391                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   392                  SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}"  examples/socks/...
   393              """
   394          }
   395  }
   396  
   397  def runGinkgo(testSuitePath) {
   398      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   399          sh """
   400              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   401              ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   402          """
   403      }
   404  }
   405  
   406  def dumpK8sCluster(dumpDirectory) {
   407      sh """
   408          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   409      """
   410  }
   411  
   412  def dumpVerrazzanoSystemPods() {
   413      sh """
   414          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   415          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   416          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   417          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   418          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   419          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   420          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   421          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   422          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   423      """
   424  }
   425  
   426  def dumpCattleSystemPods() {
   427      sh """
   428          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   429          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   430          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   431          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   432          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   433      """
   434  }
   435  
   436  def dumpNginxIngressControllerLogs() {
   437      sh """
   438          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   439          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   440          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   441      """
   442  }
   443  
   444  def dumpVerrazzanoPlatformOperatorLogs() {
   445      sh """
   446          ## dump out verrazzano-platform-operator logs
   447          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   448          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   449          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   450          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   451          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   452          echo "------------------------------------------"
   453      """
   454  }
   455  
   456  def dumpVerrazzanoApplicationOperatorLogs() {
   457      sh """
   458          ## dump out verrazzano-application-operator logs
   459          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   460          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   461          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   462          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   463          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   464          echo "------------------------------------------"
   465      """
   466  }
   467  
   468  def dumpOamKubernetesRuntimeLogs() {
   469      sh """
   470          ## dump out oam-kubernetes-runtime logs
   471          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   472          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   473          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   474          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   475          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   476          echo "------------------------------------------"
   477      """
   478  }
   479  
   480  def dumpVerrazzanoApiLogs() {
   481      sh """
   482          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   483          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log"
   484          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   485      """
   486  }
   487  
   488  def getEffectiveDumpOnSuccess() {
   489      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   490      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   491          effectiveValue = true
   492          echo "Forcing dump on success based on global override setting"
   493      }
   494      return effectiveValue
   495  }
   496  
   497  def deleteCluster() {
   498      sh """
   499          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   500          make delete-cluster
   501          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   502            echo "Failures seen during dumping of artifacts, treat post as failed"
   503            exit 1
   504          fi
   505      """
   506  }
   507  
   508  def setDisplayName() {
   509      echo "Start setDisplayName"
   510      def causes = currentBuild.getBuildCauses()
   511      echo "causes: " + causes.toString()
   512      for (cause in causes) {
   513          def causeString = cause.toString()
   514          echo "current cause: " + causeString
   515          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   516               echo "This job was caused by " + causeString
   517               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   518                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   519               } else if (causeString.contains("verrazzano-flaky-tests")) {
   520                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   521               }
   522           }
   523      }
   524      echo "End setDisplayName"
   525  }