github.com/verrazzano/verrazzano@v1.7.1/ci/multiplatform/Jenkinsfile (about)

     1  // Copyright (c) 2020, 2024, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def SKIP_ACCEPTANCE_TESTS = false
     6  def availableRegions = [  "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ]
     7  Collections.shuffle(availableRegions)
     8  def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','')
     9  def dns_zone_ocid = 'dummy'
    10  def OKE_CLUSTER_PREFIX = ""
    11  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
    12  
    13  pipeline {
    14      options {
    15          skipDefaultCheckout true
    16      }
    17  
    18      agent {
    19         docker {
    20              image "${RUNNER_DOCKER_IMAGE}"
    21              args "${RUNNER_DOCKER_ARGS}"
    22              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    23              registryCredentialsId 'ocir-pull-and-push-account'
    24              label "${agentLabel}"
    25          }
    26      }
    27  
    28      parameters {
    29          choice (description: 'Verrazzano Test Environment', name: 'TEST_ENV',
    30                  choices: ["kind", "magicdns_oke", "ocidns_oke"])
    31          choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION',
    32              // 1st choice is the default value
    33              choices: availableRegions )
    34          choice (description: 'OKE node pool configuration', name: 'OKE_NODE_POOL',
    35              // 1st choice is the default value
    36              choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2" ])
    37          choice (description: 'Kubernetes Version for OKE Cluster', name: 'OKE_CLUSTER_VERSION',
    38                  // 1st choice is the default value
    39                  choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ])
    40          string defaultValue: 'dev', description: 'Verrazzano install profile name', name: "INSTALL_PROFILE", trim: true
    41          choice (name: 'CRD_API_VERSION',
    42                  description: 'This is the API crd version.',
    43                  // 1st choice is the default value
    44                  choices: [ "v1beta1", "v1alpha1"])
    45          booleanParam (description: 'Whether to kick off acceptance test run at the end of this build', name: 'RUN_ACCEPTANCE_TESTS', defaultValue: true)
    46          booleanParam (description: 'Whether to run example tests', name: 'RUN_EXAMPLE_TESTS', defaultValue: true)
    47          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    48      }
    49  
    50      environment {
    51          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    52          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    53          DOCKER_PLATFORM_IMAGE_NAME = "${env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' ? env.DOCKER_PLATFORM_PUBLISH_IMAGE_NAME : env.DOCKER_PLATFORM_CI_IMAGE_NAME}"
    54          DOCKER_OAM_CI_IMAGE_NAME = 'verrazzano-application-operator-jenkins'
    55          DOCKER_OAM_PUBLISH_IMAGE_NAME = 'verrazzano-application-operator'
    56          DOCKER_OAM_IMAGE_NAME = "${env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' ? env.DOCKER_OAM_PUBLISH_IMAGE_NAME : env.DOCKER_OAM_CI_IMAGE_NAME}"
    57          CREATE_LATEST_TAG = "${env.BRANCH_NAME == 'master' ? '1' : '0'}"
    58          GOPATH = '/home/opc/go'
    59          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    60          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    61          DOCKER_EMAIL = credentials('github-packages-email')
    62          DOCKER_REPO = 'ghcr.io'
    63          DOCKER_NAMESPACE = 'verrazzano'
    64          NETRC_FILE = credentials('netrc')
    65          SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY')
    66  
    67          CLUSTER_NAME = 'verrazzano'
    68          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    69          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    70          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    71          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    72  
    73          // Environment variable for Verrazzano CLI executable
    74          VZ_COMMAND="${GO_REPO_PATH}/vz"
    75  
    76          OCR_CREDS = credentials('ocr-pull-and-push-account')
    77          OCR_REPO = 'container-registry.oracle.com'
    78          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    79          INSTALL_CONFIG_FILE_KIND = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
    80  
    81          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test
    82          DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test
    83  
    84          TEST_ENV = "${params.TEST_ENV}"
    85          INSTALL_PROFILE = "${params.INSTALL_PROFILE}"
    86  
    87          TF_VAR_tenancy_id = credentials('oci-tenancy')
    88          TF_VAR_user_id = credentials('oci-user-ocid')
    89          TF_VAR_region = "${params.OKE_CLUSTER_REGION}"
    90          TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}"
    91          TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}"
    92          TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint')
    93          TF_VAR_api_private_key_path = credentials('oci-api-key')
    94          TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key')
    95          TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid')
    96  
    97          OCI_CLI_TENANCY = credentials('oci-tenancy')
    98          OCI_CLI_USER = credentials('oci-user-ocid')
    99          OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint')
   100          OCI_CLI_KEY_FILE = credentials('oci-api-key')
   101          OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}"
   102          OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True'
   103          INSTALL_CONFIG_FILE_OCIDNS = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-ocidns.yaml"
   104          INSTALL_CONFIG_FILE_NIPIO = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-nipio.yaml"
   105          OCI_DNS_ZONE_SUFFIX=credentials('oci-dns-zone')
   106          OCI_DNS_ZONE_NAME="z${zoneId}." + "${OCI_DNS_ZONE_SUFFIX}"
   107          VZ_ENVIRONMENT_NAME = "${params.TEST_ENV == 'ocidns_oke' ? 'b' + env.BUILD_NUMBER : 'default'}"
   108          TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   109          SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim()
   110  
   111          // used to emit metrics
   112          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   113      }
   114  
   115      stages {
   116          stage('Clean workspace and checkout') {
   117              environment {
   118                  OCI_CLI_AUTH="instance_principal"
   119                  OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   120                  VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   121              }
   122              steps {
   123                  sh """
   124                      echo "${NODE_LABELS}"
   125                  """
   126  
   127                  script {
   128                      checkout scm
   129                  }
   130                  sh """
   131                      cp -f "${NETRC_FILE}" $HOME/.netrc
   132                      chmod 600 $HOME/.netrc
   133                  """
   134  
   135                  script {
   136                      try {
   137                          sh """
   138                      echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   139                          """
   140                      } catch(error) {
   141                          echo "docker login failed, retrying after sleep"
   142                          retry(4) {
   143                              sleep(30)
   144                              sh """
   145                      echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   146                              """
   147                          }
   148                      }
   149                  }
   150                  sh """
   151                      rm -rf ${GO_REPO_PATH}/verrazzano
   152                      mkdir -p ${GO_REPO_PATH}/verrazzano
   153                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   154                  """
   155  
   156                  script {
   157                      def props = readProperties file: '.verrazzano-development-version'
   158                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   159                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   160                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   161                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   162  
   163                      if (params.TEST_ENV != "kind") {
   164                          // derive the prefix for the OKE cluster
   165                          OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${GO_REPO_PATH}/verrazzano/ci/scripts/derive_oke_cluster_name.sh").trim()
   166                      }
   167                  }
   168                  script {
   169                      sh """
   170                          echo "Downloading VZ CLI from object storage"
   171                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   172                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   173                          ${GO_REPO_PATH}/vz version
   174                      """
   175                  }
   176              }
   177          }
   178  
   179          stage('Acceptance Tests') {
   180  
   181              stages {
   182  
   183                  stage('Prepare AT environment') {
   184                      stages {
   185                          stage('Create Cluster') {
   186                              steps {
   187                                  script {
   188                                      sh """
   189                                          echo "tests will execute" > ${TESTS_EXECUTED_FILE}
   190                                          if [ "${env.TEST_ENV}" == "kind" ]
   191                                          then
   192                                              echo "Create Kind cluster"
   193                                              cd ${GO_REPO_PATH}/verrazzano/platform-operator
   194                                              make create-cluster
   195  
   196                                              echo "Install metallb"
   197                                              cd ${GO_REPO_PATH}/verrazzano
   198                                              ./tests/e2e/config/scripts/install-metallb.sh
   199                                          else
   200                                              echo "Create OKE cluster"
   201                                              cd ${GO_REPO_PATH}/verrazzano
   202                                              TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} ./tests/e2e/config/scripts/create_oke_cluster.sh
   203                                          fi
   204                                      """
   205                                      }
   206                                  }
   207                          }
   208  
   209                          stage('Configure Verrazzano prerequisites') {
   210                              environment {
   211                                  OCI_CLI_AUTH="instance_principal"
   212                                  OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   213                              }
   214                              parallel {
   215                                  stage ('Install platform operator') {
   216                                       steps {
   217                                          script {
   218                                              sh """
   219                                                  echo "Create Image Pull Secrets"
   220                                                  cd ${GO_REPO_PATH}/verrazzano
   221                                                  ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}"
   222                                                  ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}"
   223                                                  ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}"
   224  
   225                                                  # Install the verrazzano-platform-operator
   226                                                  echo "Install Platform Operator"
   227                                                  if [ "NONE" = "${VERRAZZANO_OPERATOR_IMAGE}" ]; then
   228                                                      echo "Using operator.yaml from object storage"
   229                                                      oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml
   230                                                      cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml
   231                                                  else
   232                                                      echo "Generating operator.yaml based on image name provided: ${VERRAZZANO_OPERATOR_IMAGE}"
   233                                                      env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/acceptance-test-operator.yaml
   234                                                  fi
   235                                                  kubectl apply -f ${WORKSPACE}/acceptance-test-operator.yaml
   236  
   237  
   238                                                  # make sure ns exists
   239                                                  ./tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install
   240  
   241                                                  # create secret in verrazzano-install ns
   242                                                  ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install"
   243                                              """
   244                                              }
   245                                          }
   246                                  }
   247  
   248                                  stage("Configure OCI DNS install prerequisites") {
   249                                      when { expression { return params.TEST_ENV == 'ocidns_oke' } }
   250                                      stages {
   251                                          stage('Create dns zone') {
   252                                              steps {
   253                                                  script {
   254                                                      dns_zone_ocid = sh(script: "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/oci_dns_ops.sh -o create -c ${TF_VAR_compartment_id} -s z${zoneId}", returnStdout: true)
   255                                                  }
   256                                              }
   257                                          }
   258                                          stage('Configure Install profile') {
   259                                                environment {
   260                                                  OCI_DNS_COMPARTMENT_OCID = credentials('oci-dns-compartment')
   261                                                  OCI_PRIVATE_KEY_FILE = credentials('oci-api-key')
   262                                                  OCI_DNS_ZONE_OCID = "${dns_zone_ocid}"
   263                                              }
   264                                              steps {
   265                                                  script {
   266                                                      sh """
   267                                                          export PATH=${HOME}/go/bin:${PATH}
   268                                                          cd ${GO_REPO_PATH}/verrazzano
   269                                                          ./tests/e2e/config/scripts/process_oci_dns_install_yaml.sh $INSTALL_CONFIG_FILE_OCIDNS
   270                                                          ./tests/e2e/config/scripts/create-test-oci-config-secret.sh
   271                                                      """
   272                                                  }
   273                                              }
   274                                          }
   275                                      }
   276                                  }
   277  
   278                                  stage("Configure Magic DNS install prerequisites") {
   279                                      when { expression { return params.TEST_ENV == 'magicdns_oke' } }
   280                                      steps {
   281                                          script {
   282                                              sh """
   283                                                  export PATH=${HOME}/go/bin:${PATH}
   284                                                  cd ${GO_REPO_PATH}/verrazzano
   285                                                  ./tests/e2e/config/scripts/process_nipio_install_yaml.sh $INSTALL_CONFIG_FILE_NIPIO
   286                                              """
   287                                          }
   288                                      }
   289                                  }
   290  
   291                                  stage("Configure Kind install prerequisites") {
   292                                      when { expression { return params.TEST_ENV == 'kind' } }
   293                                      steps {
   294                                          script {
   295                                              sh """
   296                                                  export PATH=${HOME}/go/bin:${PATH}
   297                                                  cd ${GO_REPO_PATH}/verrazzano
   298                                                  ./tests/e2e/config/scripts/process_kind_install_yaml.sh ${INSTALL_CONFIG_FILE_KIND}
   299                                              """
   300                                          }
   301                                      }
   302                                  }
   303                              }
   304                          }
   305  
   306                          stage ('Install Verrazzano') {
   307                              steps {
   308                                  script {
   309                                      sh """
   310                                          echo "Waiting for Operator to be ready"
   311                                          kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator
   312                                          echo "Installing Verrazzano on ${TEST_ENV}"
   313                                          # apply config to create cluster
   314                                          if [ "${TEST_ENV}" == "magicdns_oke" ]; then
   315                                          kubectl apply -f ${INSTALL_CONFIG_FILE_NIPIO}
   316                                          elif [ "${TEST_ENV}" == "ocidns_oke" ]; then
   317                                          kubectl apply -f ${INSTALL_CONFIG_FILE_OCIDNS}
   318                                          elif [ "${TEST_ENV}" == "kind" ]; then
   319                                          kubectl apply -f ${INSTALL_CONFIG_FILE_KIND}
   320                                          fi
   321  
   322                                          cd ${GO_REPO_PATH}/verrazzano
   323                                          ./tests/e2e/config/scripts/wait-for-verrazzano-install.sh
   324  
   325                                      """
   326                                  }
   327                              }
   328                          }
   329                      }
   330                      post {
   331                          always {
   332                              sh """
   333                                  ## dump out install logs
   334                                  mkdir -p ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs
   335                                  kubectl -n verrazzano-install logs --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-install.log --tail -1
   336                                  kubectl -n verrazzano-install describe pod --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-install-job-pod.out
   337                                  echo "Verrazzano Installation logs dumped to verrazzano-install.log"
   338                                  echo "Verrazzano Install pod description dumped to verrazzano-install-job-pod.out"
   339                                  echo "------------------------------------------"
   340                              """
   341                          }
   342                      }
   343                  }
   344  
   345                  stage('Run Acceptance Tests') {
   346  
   347                      parallel {
   348                          stage('verify-install') {
   349                              steps {
   350                                  runGinkgoRandomize('verify-install')
   351                              }
   352                          }
   353                          stage('verify-infra restapi') {
   354                              steps {
   355                                  runGinkgoRandomize('verify-infra/restapi')
   356                              }
   357                          }
   358                          stage('verify-infra oam') {
   359                              steps {
   360                                  runGinkgoRandomize('verify-infra/oam')
   361                              }
   362                          }
   363                          stage('verify-infra vmi') {
   364                              steps {
   365                                  runGinkgoRandomize('verify-infra/vmi')
   366                              }
   367                          }
   368                          // yes i know this is ugly - working on cleaning it up
   369                          stage('examples todo') {
   370                              when {
   371                                  expression {params.RUN_EXAMPLE_TESTS == true}
   372                              }
   373                              steps {
   374                                  runGinkgo('examples/todo-list')
   375                              }
   376                          }
   377                          stage('examples socks') {
   378                              when {
   379                                  expression {params.RUN_EXAMPLE_TESTS == true}
   380                              }
   381                              steps {
   382                                  runGinkgo('examples/sock-shop')
   383                              }
   384                          }
   385                          stage('examples springboot') {
   386                              when {
   387                                  expression {params.RUN_EXAMPLE_TESTS == true}
   388                              }
   389                              steps {
   390                                  runGinkgo('examples/springboot-app')
   391                              }
   392                          }
   393                          stage('examples helidon') {
   394                              when {
   395                                  expression {params.RUN_EXAMPLE_TESTS == true}
   396                              }
   397                              steps {
   398                                  runGinkgo('examples/hello-helidon')
   399                              }
   400                          }
   401                                      stage('examples bobs books') {
   402                              when {
   403                                  expression {params.RUN_EXAMPLE_TESTS == true}
   404                              }
   405                              steps {
   406                                  runGinkgo('examples/bobs-books')
   407                              }
   408                          }
   409                      }
   410                      post {
   411                          always {
   412                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true
   413                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   414                          }
   415                      }
   416                  }
   417              }
   418  
   419              post {
   420                  failure {
   421                      script {
   422                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   423                              dumpK8sCluster('new-acceptance-tests-cluster-snapshot')
   424                          }
   425                      }
   426                  }
   427                  success {
   428                      script {
   429                          if (params.DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   430                              dumpK8sCluster('new-acceptance-tests-cluster-snapshot')
   431                          }
   432                      }
   433                  }
   434              }
   435          }
   436      }
   437  
   438      post {
   439          always {
   440              script {
   441                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   442                      dumpVerrazzanoSystemPods()
   443                      dumpCattleSystemPods()
   444                      dumpNginxIngressControllerLogs()
   445                      dumpVerrazzanoPlatformOperatorLogs()
   446                      dumpVerrazzanoApplicationOperatorLogs()
   447                      dumpOamKubernetesRuntimeLogs()
   448                      dumpVerrazzanoApiLogs()
   449                  }
   450              }
   451              archiveArtifacts artifacts: '**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**', allowEmptyArchive: true
   452              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   453  
   454              sh """
   455                  if [ "${TEST_ENV}" == "ocidns_oke" ]; then
   456                    cd ${GO_REPO_PATH}/verrazzano
   457                    ./tests/e2e/config/scripts/oci_dns_ops.sh -o delete -s z${zoneId} || echo "Failed to delete DNS zone z${zoneId}"
   458                  fi
   459                  if [ "${TEST_ENV}" == "kind" ]; then
   460                    cd ${GO_REPO_PATH}/verrazzano/platform-operator
   461                    make delete-cluster
   462                  else
   463                    cd ${GO_REPO_PATH}/verrazzano
   464                    TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} ./tests/e2e/config/scripts/delete_oke_cluster.sh || true
   465                  fi
   466                  if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   467                    echo "Failures seen during dumping of artifacts, treat post as failed"
   468                    exit 1
   469                  fi
   470              """
   471              deleteDir()
   472          }
   473      }
   474  }
   475  
   476  def runGinkgoRandomize(testSuitePath) {
   477      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   478          sh """
   479              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   480              ginkgo -p --randomize-all -v --keep-going --no-color ${testSuitePath}/...
   481          """
   482      }
   483  }
   484  
   485  def runGinkgo(testSuitePath) {
   486      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   487          sh """
   488              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   489              ginkgo -v --keep=going --no-color ${testSuitePath}/...
   490          """
   491      }
   492  }
   493  
   494  def dumpK8sCluster(dumpDirectory) {
   495      sh """
   496          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   497      """
   498  }
   499  
   500  def dumpVerrazzanoSystemPods() {
   501      sh """
   502          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log"
   503          ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   504          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log"
   505          ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   506          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log"
   507          ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   508          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log"
   509          ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   510      """
   511  }
   512  
   513  def dumpCattleSystemPods() {
   514      sh """
   515          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/cattle-system-pods.log"
   516          ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   517          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/rancher.log"
   518          ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   519      """
   520  }
   521  
   522  def dumpNginxIngressControllerLogs() {
   523      sh """
   524          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log"
   525          ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   526      """
   527  }
   528  
   529  def dumpVerrazzanoPlatformOperatorLogs() {
   530      sh """
   531          ## dump out verrazzano-platform-operator logs
   532          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   533          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   534          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   535          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   536          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   537          echo "------------------------------------------"
   538      """
   539  }
   540  
   541  def dumpVerrazzanoApplicationOperatorLogs() {
   542      sh """
   543          ## dump out verrazzano-application-operator logs
   544          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   545          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   546          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   547          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   548          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   549          echo "------------------------------------------"
   550      """
   551  }
   552  
   553  def dumpOamKubernetesRuntimeLogs() {
   554      sh """
   555          ## dump out oam-kubernetes-runtime logs
   556          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   557          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   558          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   559          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   560          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   561          echo "------------------------------------------"
   562      """
   563  }
   564  
   565  def dumpVerrazzanoApiLogs() {
   566      sh """
   567          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log"
   568          ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   569      """
   570  }