github.com/verrazzano/verrazzano@v1.7.0/ci/no-injection/Jenkinsfile (about)

     1  // Copyright (c) 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          skipDefaultCheckout true
    11          timestamps ()
    12      }
    13  
    14      agent {
    15         docker {
    16              image "${RUNNER_DOCKER_IMAGE}"
    17              args "${RUNNER_DOCKER_ARGS}"
    18              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    19              registryCredentialsId 'ocir-pull-and-push-account'
    20              label "${agentLabel}"
    21          }
    22      }
    23  
    24      parameters {
    25          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    26                  description: 'Kubernetes Version for KinD Cluster',
    27                  // 1st choice is the default value
    28                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    29          string (name: 'GIT_COMMIT_TO_USE',
    30                          defaultValue: 'NONE',
    31                          description: 'This is the full git commit hash from the source build to be used for all jobs',
    32                          trim: true)
    33          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    34                          defaultValue: 'NONE',
    35                          description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    36                          trim: true)
    37          choice (name: 'WILDCARD_DNS_DOMAIN',
    38                  description: 'This is the wildcard DNS domain',
    39                  // 1st choice is the default value
    40                  choices: [ "nip.io", "sslip.io"])
    41          choice (name: 'CRD_API_VERSION',
    42                  description: 'This is the API crd version.',
    43                  // 1st choice is the default value
    44                  choices: [ "v1beta1", "v1alpha1"])
    45          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    46          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    47          booleanParam (name: 'RUN_COHERENCE_TESTS', description: 'Whether to run the Coherence Workload/Logging tests (defaults to true)',defaultValue: true)
    48          string (name: 'CONSOLE_REPO_BRANCH',
    49                  defaultValue: '',
    50                  description: 'The branch to check out after cloning the console repository.',
    51                  trim: true)
    52          string (name: 'TAGGED_TESTS',
    53                  defaultValue: '',
    54                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    55                  trim: true)
    56          string (name: 'INCLUDED_TESTS',
    57                  defaultValue: '.*',
    58                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    59                  trim: true)
    60          string (name: 'EXCLUDED_TESTS',
    61                  defaultValue: '_excluded_test',
    62                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    63                  trim: true)
    64          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    65      }
    66  
    67      environment {
    68          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    69          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    70          GOPATH = '/home/opc/go'
    71          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    72          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    73          DOCKER_EMAIL = credentials('github-packages-email')
    74          DOCKER_REPO = 'ghcr.io'
    75          DOCKER_NAMESPACE = 'verrazzano'
    76          NETRC_FILE = credentials('netrc')
    77          CLUSTER_NAME = 'verrazzano'
    78          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    79          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    80          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    81          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    82          OCR_CREDS = credentials('ocr-pull-and-push-account')
    83          OCR_REPO = 'container-registry.oracle.com'
    84          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    85          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-no-injection.yaml"
    86          INSTALL_PROFILE = "prod"
    87          VZ_ENVIRONMENT_NAME = "default"
    88          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    89          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    90  
    91          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test
    92          DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test
    93  
    94          // Environment variables required to capture cluster snapshot and bug report on test failure
    95          DUMP_KUBECONFIG="${KUBECONFIG}"
    96          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
    97          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
    98          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
    99  
   100          // Environment variable for Verrazzano CLI executable
   101          VZ_COMMAND="${GO_REPO_PATH}/vz"
   102  
   103          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   104          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   105  
   106          // used for console artifact capture on failure
   107          JENKINS_READ = credentials('jenkins-auditor')
   108          OCI_CLI_AUTH="instance_principal"
   109          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   110          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   111          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   112          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   113  
   114          // used to emit metrics
   115          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   116          TEST_ENV_LABEL = "kind"
   117          TEST_ENV = "KIND"
   118          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   119          SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
   120          SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
   121          SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
   122  
   123          // used to generate Ginkgo test reports
   124          TEST_REPORT = "test-report.xml"
   125          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   126          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   127      }
   128  
   129      stages {
   130          stage('Clean workspace and checkout') {
   131              steps {
   132                  sh """
   133                      echo "${NODE_LABELS}"
   134                  """
   135  
   136                  script {
   137                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   138                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   139                          echo "Specific GIT commit was not specified, use current head"
   140                          def scmInfo = checkout scm
   141                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   142                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   143                      } else {
   144                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   145                          def scmInfo = checkout([
   146                              $class: 'GitSCM',
   147                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   148                              doGenerateSubmoduleConfigurations: false,
   149                              extensions: [],
   150                              submoduleCfg: [],
   151                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   152                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   153                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   154                          // If the commit we were handed is not what the SCM says we are using, fail
   155                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   156                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   157                              exit 1
   158                          }
   159                      }
   160                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   161                  }
   162  
   163                  sh """
   164                      cp -f "${NETRC_FILE}" $HOME/.netrc
   165                      chmod 600 $HOME/.netrc
   166                  """
   167  
   168                  script {
   169                      try {
   170                      sh """
   171                          echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   172                      """
   173                      } catch(error) {
   174                          echo "docker login failed, retrying after sleep"
   175                          retry(4) {
   176                              sleep(30)
   177                              sh """
   178                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   179                              """
   180                          }
   181                      }
   182                  }
   183                  sh """
   184                      rm -rf ${GO_REPO_PATH}/verrazzano
   185                      mkdir -p ${GO_REPO_PATH}/verrazzano
   186                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   187                  """
   188  
   189                  script {
   190                      def props = readProperties file: '.verrazzano-development-version'
   191                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   192                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   193                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   194                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   195                      // update the description with some meaningful info
   196                      setDisplayName()
   197                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   198                  }
   199                  script {
   200                      sh """
   201                          echo "Downloading VZ CLI from object storage"
   202                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   203                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   204                          ${GO_REPO_PATH}/vz version
   205                      """
   206                  }
   207              }
   208          }
   209  
   210          stage('Acceptance Tests') {
   211              stages {
   212                  stage('Prepare AT environment') {
   213                      environment {
   214                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   215                          OCI_CLI_AUTH="instance_principal"
   216                          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   217                          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   218                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   219                      }
   220                      steps {
   221                          sh """
   222                              cd ${GO_REPO_PATH}/verrazzano
   223                              ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
   224                          """
   225                      }
   226                      post {
   227                          failure {
   228                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   229                          }
   230                          always {
   231                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true
   232                          }
   233                      }
   234                  }
   235  
   236                  stage('verify-install') {
   237                      environment {
   238                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install"
   239                      }
   240                      steps {
   241                          runGinkgoRandomize('verify-install')
   242                      }
   243                      post {
   244                          always {
   245                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   246                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   247                          }
   248                      }
   249                  }
   250  
   251                  stage('Run Acceptance Tests Infra') {
   252                      parallel {
   253                          stage('verify-scripts') {
   254                              steps {
   255                                  runGinkgoRandomize('scripts', "${KUBECONFIG}")
   256                              }
   257                          }
   258                          stage('verify-infra restapi') {
   259                              environment {
   260                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra-restapi"
   261                              }
   262                              steps {
   263                                  runGinkgoRandomize('verify-infra/restapi')
   264                              }
   265                          }
   266                          stage('verify-infra oam') {
   267                              environment {
   268                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra-oam"
   269                              }
   270                              steps {
   271                                  runGinkgoRandomize('verify-infra/oam')
   272                              }
   273                          }
   274                          stage('verify-infra vmi') {
   275                              environment {
   276                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra-vmi"
   277                              }
   278                              steps {
   279                                  runGinkgoRandomize('verify-infra/vmi')
   280                              }
   281                          }
   282                          stage('security rbac') {
   283                              environment {
   284                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/sec-role-based-access"
   285                              }
   286                              steps {
   287                                  runGinkgoNoFlags('security/rbac')
   288                              }
   289                          }
   290                          stage('system component metrics') {
   291                              environment {
   292                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/system-component-metrics"
   293                              }
   294                              steps {
   295                                  runGinkgo('metrics/syscomponents')
   296                              }
   297                          }
   298                          stage ('console') {
   299                              environment {
   300                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/console"
   301                              }
   302                              steps {
   303                                  catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
   304                                      sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh"
   305                                  }
   306                              }
   307                              post {
   308                                  failure {
   309                                      sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh"
   310                                  }
   311                              }
   312                          }
   313                      }
   314                      post {
   315                          always {
   316                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   317                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   318                          }
   319                      }
   320                  }
   321  
   322                  stage('Run Acceptance Tests Deployments') {
   323                      parallel {
   324                          stage('opensearch-topology') {
   325                              steps {
   326                                  runGinkgoRandomize('opensearch/topology')
   327                              }
   328                          }
   329  //                         stage('istio authorization policy') {
   330  //                             environment {
   331  //                                 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/istio-authz-policy"
   332  //                             }
   333  //                             steps {
   334  //                                 runGinkgo('istio/authz')
   335  //                             }
   336  //                         }
   337  // Following test is disabled for now while the test code is assessed for validity in the absence of istio side cars
   338  //                         stage('security network policies') {
   339  //                             environment {
   340  //                                 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/netpol"
   341  //                             }
   342  //                             steps {
   343  //                                 script {
   344  //                                     if (params.CREATE_CLUSTER_USE_CALICO == true) {
   345  //                                         runGinkgo('security/netpol')
   346  //                                     }
   347  //                                 }
   348  //                             }
   349  //                         }
   350                          stage('deployment metrics') {
   351                              environment {
   352                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/k8sdeploy-workload-metrics"
   353                              }
   354                              steps {
   355                                  runGinkgo('metrics/deploymetrics')
   356                              }
   357                          }
   358                          stage('system logging') {
   359                              environment {
   360                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/system-logging"
   361                              }
   362                              steps {
   363                                  runGinkgoNoFlags('logging/system')
   364                              }
   365                          }
   366                          stage('examples logging helidon') {
   367                              environment {
   368                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-logging-helidon"
   369                              }
   370                              steps {
   371                                  runGinkgo('logging/helidon')
   372                              }
   373                          }
   374                          stage('examples helidon') {
   375                              environment {
   376                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon"
   377                              }
   378                              steps {
   379                                  runGinkgo('examples/helidon')
   380                              }
   381                          }
   382                          stage('weblogic workload') {
   383                              environment {
   384                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-workload"
   385                              }
   386                              steps {
   387                                  runGinkgo('workloads/weblogic')
   388                              }
   389                          }
   390                          stage('coherence workload') {
   391                              when {
   392                                  expression {params.RUN_COHERENCE_TESTS}
   393                              }
   394                              environment {
   395                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/coherence-workload"
   396                              }
   397                              steps {
   398                                  runGinkgoNoFlags('workloads/coherence')
   399                              }
   400                          }
   401                          stage('examples helidon-metrics') {
   402                              environment {
   403                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/helidonmetrics"
   404                              }
   405                              when {
   406                                  expression {params.RUN_SLOW_TESTS == true}
   407                              }
   408                              steps {
   409                                  runGinkgo('examples/helidonmetrics')
   410                              }
   411                          }
   412                          stage('Ingress & logging trait WebLogic workload') {
   413                              environment {
   414                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/loggingtrait-weblogicworkload"
   415                              }
   416                              steps {
   417                                  runGinkgo('loggingtrait/weblogicworkload')
   418                              }
   419                          }
   420                          stage('logging trait Helidon workload') {
   421                              environment {
   422                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/loggingtrait-helidonworkload"
   423                              }
   424                              steps {
   425                                  runGinkgo('loggingtrait/helidonworkload')
   426                              }
   427                          }
   428                          stage('logging trait Coherence workload') {
   429                              when {
   430                                  expression {params.RUN_COHERENCE_TESTS}
   431                              }
   432                              environment {
   433                                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/loggingtrait-coherenceworkload"
   434                              }
   435                              steps {
   436                                  runGinkgo('loggingtrait/coherenceworkload')
   437                              }
   438                          }
   439                      }
   440                      post {
   441                          always {
   442                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/Screenshot*.png,**/ConsoleLog*.log', allowEmptyArchive: true
   443                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   444                          }
   445                      }
   446                  }
   447  
   448              }
   449  
   450              post {
   451                  failure {
   452                      script {
   453                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   454                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   455                          }
   456                      }
   457                  }
   458                  success {
   459                      script {
   460                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   461                              dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot')
   462                          }
   463                      }
   464                  }
   465              }
   466          }
   467      }
   468  
   469      post {
   470          always {
   471              script {
   472                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   473                      dumpVerrazzanoSystemPods()
   474                      dumpCattleSystemPods()
   475                      dumpNginxIngressControllerLogs()
   476                      dumpVerrazzanoPlatformOperatorLogs()
   477                      dumpVerrazzanoApplicationOperatorLogs()
   478                      dumpOamKubernetesRuntimeLogs()
   479                      dumpVerrazzanoApiLogs()
   480                  }
   481              }
   482  
   483              sh """
   484                  # Copy the generated test reports to WORKSPACE to archive them
   485                  mkdir -p ${TEST_REPORT_DIR}
   486                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   487                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   488              """
   489              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   490              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   491              deleteCluster()
   492          }
   493          failure {
   494              sh """
   495                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   496              """
   497              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   498              sh """
   499                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   500                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   501                  rm archive.zip
   502              """
   503          }
   504          cleanup {
   505              deleteDir()
   506          }
   507      }
   508  }
   509  
   510  def runGinkgoRandomize(testSuitePath, kubeConfig = '') {
   511      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   512          sh """
   513              if [ ! -z "${kubeConfig}" ]; then
   514                  export KUBECONFIG="${kubeConfig}"
   515              fi
   516              if [ -d "${testSuitePath}" ]; then
   517                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   518                  ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --istioInjection="disabled"
   519              fi
   520          """
   521      }
   522  }
   523  
   524  def runSocksVariant(variant) {
   525      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   526              sh """
   527                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   528                  SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}"  examples/socks/...
   529              """
   530          }
   531  }
   532  
   533  def runGinkgo(testSuitePath) {
   534      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   535          sh """
   536              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   537              ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...  -- --istioInjection="disabled"
   538          """
   539      }
   540  }
   541  
   542  def runGinkgoNoFlags(testSuitePath) {
   543      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   544          sh """
   545              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   546              ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   547          """
   548      }
   549  }
   550  
   551  def dumpK8sCluster(dumpDirectory) {
   552      sh """
   553          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   554      """
   555  }
   556  
   557  def dumpVerrazzanoSystemPods() {
   558      sh """
   559          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   560          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   561          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   562          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   563          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   564          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   565          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   566          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   567          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   568      """
   569  }
   570  
   571  def dumpCattleSystemPods() {
   572      sh """
   573          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   574          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   575          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   576          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   577          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   578      """
   579  }
   580  
   581  def dumpNginxIngressControllerLogs() {
   582      sh """
   583          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   584          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   585          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   586      """
   587  }
   588  
   589  def dumpVerrazzanoPlatformOperatorLogs() {
   590      sh """
   591          ## dump out verrazzano-platform-operator logs
   592          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   593          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   594          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   595          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   596          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   597          echo "------------------------------------------"
   598      """
   599  }
   600  
   601  def dumpVerrazzanoApplicationOperatorLogs() {
   602      sh """
   603          ## dump out verrazzano-application-operator logs
   604          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   605          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   606          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   607          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   608          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   609          echo "------------------------------------------"
   610      """
   611  }
   612  
   613  def dumpOamKubernetesRuntimeLogs() {
   614      sh """
   615          ## dump out oam-kubernetes-runtime logs
   616          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   617          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   618          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   619          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   620          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   621          echo "------------------------------------------"
   622      """
   623  }
   624  
   625  def dumpVerrazzanoApiLogs() {
   626      sh """
   627          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   628          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log"
   629          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   630      """
   631  }
   632  
   633  def getEffectiveDumpOnSuccess() {
   634      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   635      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   636          effectiveValue = true
   637          echo "Forcing dump on success based on global override setting"
   638      }
   639      return effectiveValue
   640  }
   641  
   642  def deleteCluster() {
   643      sh """
   644          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   645          make delete-cluster
   646          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   647            echo "Failures seen during dumping of artifacts, treat post as failed"
   648            exit 1
   649          fi
   650      """
   651  }
   652  
   653  def setDisplayName() {
   654      echo "Start setDisplayName"
   655      def causes = currentBuild.getBuildCauses()
   656      echo "causes: " + causes.toString()
   657      for (cause in causes) {
   658          def causeString = cause.toString()
   659          echo "current cause: " + causeString
   660          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   661               echo "This job was caused by " + causeString
   662               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   663                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   664               } else if (causeString.contains("verrazzano-flaky-tests")) {
   665                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   666               }
   667           }
   668      }
   669      echo "End setDisplayName"
   670  }