github.com/verrazzano/verrazzano@v1.7.0/ci/clusterAPI/JenkinsfileCAPIQC (about)

     1  // Copyright (c) 2020, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  def availableRegions = [  "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-frankfurt-1", "eu-zurich-1", "uk-london-1" ]
     8  def okeCNITypes = ['FLANNEL_OVERLAY', 'OCI_VCN_IP_NATIVE']
     9  Collections.shuffle(availableRegions)
    10  Collections.shuffle(okeCNITypes)
    11  
    12  pipeline {
    13      options {
    14          timeout(time: 90, unit: 'MINUTES')
    15          skipDefaultCheckout true
    16          timestamps ()
    17      }
    18  
    19      agent {
    20          docker {
    21              image "${RUNNER_DOCKER_IMAGE}"
    22              args "${RUNNER_DOCKER_ARGS}"
    23              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    24              registryCredentialsId 'ocir-pull-and-push-account'
    25              label "${agentLabel}"
    26          }
    27      }
    28  
    29      parameters {
    30          choice(name: 'KUBERNETES_CLUSTER_VERSION',
    31                  description: 'Kubernetes Version for KinD Cluster',
    32                  // 1st choice is the default value
    33                  choices: ["1.27", "1.26", "1.25", "1.24"])
    34          booleanParam(description: 'Deploy OCNE OCI Cluster', name: 'DEPLOY_OCNEOCI_CLUSTER', defaultValue: false)
    35          booleanParam(description: 'Deploy OKE Cluster', name: 'DEPLOY_OKE_CLUSTER', defaultValue: true)
    36          choice (description: 'CNI Type for OKE', name: 'OKE_CNI_TYPE', choices: okeCNITypes )
    37          choice (description: 'OCI region to launch CAPI clusters in', name: 'CAPI_CLUSTER_REGION', choices: availableRegions )
    38          string(name: 'GIT_COMMIT_TO_USE',
    39                  defaultValue: 'NONE',
    40                  description: 'This is the full git commit hash from the source build to be used for all jobs',
    41                  trim: true)
    42          string(name: 'VERRAZZANO_OPERATOR_IMAGE',
    43                  defaultValue: 'NONE',
    44                  description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    45                  trim: true)
    46          choice(name: 'WILDCARD_DNS_DOMAIN',
    47                  description: 'This is the wildcard DNS domain',
    48                  // 1st choice is the default value
    49                  choices: ["nip.io", "sslip.io"])
    50          choice(name: 'CRD_API_VERSION',
    51                  description: 'This is the API crd version.',
    52                  // 1st choice is the default value
    53                  choices: ["v1beta1", "v1alpha1"])
    54          booleanParam(description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    55          booleanParam(description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    56          booleanParam(description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true)
    57          string(name: 'TAGGED_TESTS',
    58                  defaultValue: '',
    59                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    60                  trim: true)
    61          string(name: 'INCLUDED_TESTS',
    62                  defaultValue: '.*',
    63                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    64                  trim: true)
    65          string(name: 'EXCLUDED_TESTS',
    66                  defaultValue: '_excluded_test',
    67                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    68                  trim: true)
    69          booleanParam(description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    70      }
    71      environment {
    72          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    73          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    74          GOPATH = '/home/opc/go'
    75          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    76          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    77          DOCKER_EMAIL = credentials('github-packages-email')
    78          DOCKER_REPO = 'ghcr.io'
    79          DOCKER_NAMESPACE = 'verrazzano'
    80          NETRC_FILE = credentials('netrc')
    81          CLUSTER_NAME = 'verrazzano'
    82          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    83          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    84          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    85          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    86          OCR_CREDS = credentials('ocr-pull-and-push-account')
    87          OCR_REPO = 'container-registry.oracle.com'
    88          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    89          INSTALL_CONFIG_FILE_KIND_TESTS = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
    90          INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/install-verrazzano.yaml"
    91          INSTALL_PROFILE = "dev"
    92          KIND_NODE_COUNT = "1"
    93          VZ_ENVIRONMENT_NAME = "default"
    94          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    95          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    96  
    97          // Environment variables required to capture cluster snapshot and bug report on test failure
    98          DUMP_KUBECONFIG="${KUBECONFIG}"
    99          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   100          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
   101          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   102  
   103          // Environment variable for Verrazzano CLI executable
   104          VZ_COMMAND="${GO_REPO_PATH}/vz"
   105  
   106          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   107          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   108  
   109          // used for console artifact capture on failure
   110          JENKINS_READ = credentials('jenkins-auditor')
   111          OCI_CLI_AUTH="instance_principal"
   112          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   113          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   114          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   115          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   116  
   117          // used to emit metrics from Ginkgo suites
   118          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   119          TEST_ENV_LABEL = "kind"
   120          TEST_ENV = "KIND"
   121          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   122          SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
   123          SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
   124          SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
   125  
   126          // used to generate Ginkgo test reports
   127          TEST_REPORT = "test-report.xml"
   128          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   129          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   130  
   131          // CAPI variables
   132          OCI_USER_ID = credentials('oci-user-ocid')
   133          OCI_CREDENTIALS_FINGERPRINT = credentials('oci-api-key-fingerprint')
   134          OCI_TENANCY_ID = credentials('oci-tenancy')
   135          OCI_REGION = "${params.CAPI_CLUSTER_REGION}"
   136          OCI_COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid')
   137          CAPI_NODE_SSH_KEY_PATH = credentials('oci-tf-pub-ssh-key')
   138          CAPI_OCI_PRIVATE_KEY_PATH = credentials('oci-api-key')
   139          CNI_TYPE = "${params.OKE_CNI_TYPE}"
   140      }
   141  
   142      stages {
   143          stage('Clean workspace and checkout') {
   144              steps {
   145                  sh """
   146                      echo "${NODE_LABELS}"
   147                  """
   148  
   149                  script {
   150                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   151                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   152                          echo "Specific GIT commit was not specified, use current head"
   153                          def scmInfo = checkout scm
   154                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   155                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   156                      } else {
   157                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   158                          def scmInfo = checkout([
   159                                  $class: 'GitSCM',
   160                                  branches: [[name: params.GIT_COMMIT_TO_USE]],
   161                                  doGenerateSubmoduleConfigurations: false,
   162                                  extensions: [],
   163                                  submoduleCfg: [],
   164                                  userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   165                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   166                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   167                          // If the commit we were handed is not what the SCM says we are using, fail
   168                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   169                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   170                              exit 1
   171                          }
   172                      }
   173                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   174                  }
   175  
   176                  sh """
   177                      cp -f "${NETRC_FILE}" $HOME/.netrc
   178                      chmod 600 $HOME/.netrc
   179                  """
   180  
   181                  performDockerLogin()
   182  
   183                  sh """
   184                      rm -rf ${GO_REPO_PATH}/verrazzano
   185                      mkdir -p ${GO_REPO_PATH}/verrazzano
   186                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   187                  """
   188  
   189                  script {
   190                      def props = readProperties file: '.verrazzano-development-version'
   191                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   192                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   193                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   194                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   195                      // update the description with some meaningful info
   196                      setDisplayName()
   197                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   198                  }
   199                  script {
   200                      sh """
   201                          echo "Downloading VZ CLI from object storage"
   202                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   203                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   204                          ${GO_REPO_PATH}/vz version
   205                      """
   206                  }
   207              }
   208          }
   209  
   210          stage('Tests') {
   211              stages {
   212                  stage('Prepare AT environment') {
   213                      environment {
   214                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   215                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   216                          REALM_USER_PASSWORD = credentials('todo-mysql-password')
   217                          REALM_NAME = "test-realm"
   218                          GITHUB_TOKEN = credentials('github-api-token-release-process')
   219                      }
   220                      steps {
   221                          script {
   222                              sh """
   223                                  cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND}
   224                              """
   225                          }
   226                          sh """
   227                              cd ${GO_REPO_PATH}/verrazzano
   228                              ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA}
   229                          """
   230                      }
   231                      post {
   232                          failure {
   233                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   234                          }
   235                          always {
   236                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   237                              // enable debug logging of Verrazzano api istio proxy
   238                              script {
   239                                  if (params.ENABLE_API_ENVOY_LOGGING) {
   240                                      sh '''
   241                                          vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\")
   242                                          if [ -z "\$vz_api_pod" ]; then
   243                                            echo "Could not find verrazzano-authproxy pod, not enabling debug logging"
   244                                          else
   245                                            kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug
   246                                          fi
   247                                          nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\")
   248                                          if [ -z "\$nginx_ing_pod" ]; then
   249                                            echo "Could not find nginx ingress controller pod, not enabling debug logging"
   250                                          else
   251                                            kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug
   252                                          fi
   253                                      '''
   254                                  }
   255                              }
   256                          }
   257                      }
   258                  }
   259  
   260                  stage('Verify Install') {
   261                      environment {
   262                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install"
   263                      }
   264                      steps {
   265                          runGinkgoRandomize('verify-install')
   266                      }
   267                      post {
   268                          always {
   269                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   270                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   271                          }
   272                      }
   273                  }
   274                  stage('Deploy Quick Create Clusters in parallel') {
   275                      parallel {
   276                          stage('OCNEOCI Cluster - Run verrazzano addon tests') {
   277                              when {
   278                                  expression { params.DEPLOY_OCNEOCI_CLUSTER == true }
   279                              }
   280                              steps {
   281                                  runQuickCreateSuite('ocneoci')
   282                              }
   283                              post {
   284                                  always {
   285                                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true
   286                                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   287                                  }
   288                                  failure {
   289                                      script {
   290                                          dumpK8sCluster('ocne-deploy-failure')
   291                                      }
   292                                  }
   293                              }
   294                          }
   295                          stage('OKE Cluster - Run verrazzano addon tests') {
   296                              when {
   297                                  expression { params.DEPLOY_OKE_CLUSTER == true }
   298                              }
   299                              steps {
   300                                  runQuickCreateSuite('oke')
   301                              }
   302                              post {
   303                                  always {
   304                                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true
   305                                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   306                                  }
   307                                  failure {
   308                                      script {
   309                                          dumpK8sCluster('oke-deploy-failure')
   310                                      }
   311                                  }
   312                              }
   313                          }
   314                      }
   315                  }
   316              }
   317  
   318              post {
   319                  aborted {
   320                      script {
   321                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   322                              dumpK8sCluster('clusterAPI-qc-kind-tests-cluster-snapshot')
   323                          }
   324                      }
   325                  }
   326                  failure {
   327                      script {
   328                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   329                              dumpK8sCluster('clusterAPI-qc-kind-tests-cluster-snapshot')
   330                          }
   331                      }
   332                  }
   333                  success {
   334                      script {
   335                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   336                              dumpK8sCluster('clusterAPI-qc-kind-tests-cluster-snapshot')
   337                          }
   338                      }
   339                  }
   340              }
   341          }
   342      }
   343  
   344      post {
   345          always {
   346              script {
   347                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   348                      dumpVerrazzanoSystemPods()
   349                      dumpCattleSystemPods()
   350                      dumpNginxIngressControllerLogs()
   351                      dumpVerrazzanoPlatformOperatorLogs()
   352                      dumpVerrazzanoApplicationOperatorLogs()
   353                      dumpOamKubernetesRuntimeLogs()
   354                      dumpVerrazzanoApiLogs()
   355                  }
   356              }
   357  
   358              sh """
   359                  # Copy the generated test reports to WORKSPACE to archive them
   360                  mkdir -p ${TEST_REPORT_DIR}
   361                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   362                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   363              """
   364              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   365              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   366              deleteCluster()
   367          }
   368          failure {
   369              sh """
   370                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   371              """
   372              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   373              sh """
   374                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   375                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   376                  rm archive.zip
   377              """
   378          }
   379          cleanup {
   380              deleteDir()
   381          }
   382      }
   383  }
   384  
   385  def runQuickCreateSuite(clusterType) {
   386      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   387          script {
   388              sh """
   389              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   390              if [ -d "quickcreate" ]; then
   391                  ginkgo -vv --progress --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" quickcreate/... -- --clusterType="${clusterType}"
   392              fi
   393              """
   394          }
   395      }
   396  }
   397  
   398  def runGinkgoRandomize(testSuitePath, kubeConfig = '') {
   399      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   400          sh """
   401              if [ ! -z "${kubeConfig}" ]; then
   402                  export KUBECONFIG="${kubeConfig}"
   403              fi
   404              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   405              if [ -d "${testSuitePath}" ]; then
   406                  ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   407              fi
   408          """
   409      }
   410  }
   411  
   412  def dumpK8sCluster(dumpDirectory) {
   413      sh """
   414          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   415      """
   416  }
   417  
   418  def dumpVerrazzanoSystemPods() {
   419      sh """
   420          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   421          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   422          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   423          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   424          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   425          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   426          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   427          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   428          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   429      """
   430  }
   431  
   432  def dumpCattleSystemPods() {
   433      sh """
   434          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   435          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   436          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   437          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   438          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   439      """
   440  }
   441  
   442  def dumpNginxIngressControllerLogs() {
   443      sh """
   444          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   445          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   446          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   447      """
   448  }
   449  
   450  def dumpVerrazzanoPlatformOperatorLogs() {
   451      sh """
   452          ## dump out verrazzano-platform-operator logs
   453          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   454          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   455          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   456          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   457          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   458          echo "------------------------------------------"
   459      """
   460  }
   461  
   462  def dumpVerrazzanoApplicationOperatorLogs() {
   463      sh """
   464          ## dump out verrazzano-application-operator logs
   465          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   466          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   467          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   468          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   469          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   470          echo "------------------------------------------"
   471      """
   472  }
   473  
   474  def dumpOamKubernetesRuntimeLogs() {
   475      sh """
   476          ## dump out oam-kubernetes-runtime logs
   477          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   478          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   479          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   480          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   481          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   482          echo "------------------------------------------"
   483      """
   484  }
   485  
   486  def dumpVerrazzanoApiLogs() {
   487      sh """
   488          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   489          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log"
   490          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   491      """
   492  }
   493  
   494  def getEffectiveDumpOnSuccess() {
   495      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   496      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   497          effectiveValue = true
   498          echo "Forcing dump on success based on global override setting"
   499      }
   500      return effectiveValue
   501  }
   502  
   503  def deleteCluster() {
   504      sh """
   505          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   506          make delete-cluster
   507          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   508            echo "Failures seen during dumping of artifacts, treat post as failed"
   509            exit 1
   510          fi
   511      """
   512  }
   513  
   514  def setDisplayName() {
   515      echo "Start setDisplayName"
   516      def causes = currentBuild.getBuildCauses()
   517      echo "causes: " + causes.toString()
   518      for (cause in causes) {
   519          def causeString = cause.toString()
   520          echo "current cause: " + causeString
   521          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   522              echo "This job was caused by " + causeString
   523              if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   524                  currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   525              } else if (causeString.contains("verrazzano-flaky-tests")) {
   526                  currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   527              }
   528          }
   529      }
   530      echo "End setDisplayName"
   531  }
   532  
   533  def performDockerLogin() {
   534      script {
   535          try {
   536              sh """
   537                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   538              """
   539          } catch(error) {
   540              echo "docker login failed, retrying after sleep"
   541              retry(4) {
   542                  sleep(30)
   543                  sh """
   544                      echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   545                  """
   546              }
   547          }
   548      }
   549  }