github.com/verrazzano/verrazzano@v1.7.0/ci/dynamic-updates/JenkinsfileUpdateDuringInstall (about)

     1  // Copyright (c) 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          skipDefaultCheckout true
    11          timestamps ()
    12      }
    13  
    14      agent {
    15         docker {
    16              image "${RUNNER_DOCKER_IMAGE}"
    17              args "${RUNNER_DOCKER_ARGS}"
    18              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    19              registryCredentialsId 'ocir-pull-and-push-account'
    20              label "${agentLabel}"
    21          }
    22      }
    23  
    24      parameters {
    25          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    26                  description: 'Kubernetes Version for KinD Cluster',
    27                  // 1st choice is the default value
    28                  choices: ["1.27", "1.26", "1.25", "1.24" ])
    29          string (name: 'GIT_COMMIT_TO_USE',
    30                          defaultValue: 'NONE',
    31                          description: 'This is the full git commit hash from the source build to be used for all jobs',
    32                          trim: true)
    33          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    34                          defaultValue: 'NONE',
    35                          description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    36                          trim: true)
    37          choice (name: 'WILDCARD_DNS_DOMAIN',
    38                  description: 'This is the wildcard DNS domain',
    39                  // 1st choice is the default value
    40                  choices: [ "nip.io", "sslip.io"])
    41          choice (name: 'CRD_API_VERSION',
    42                  description: 'This is the API crd version.',
    43                  // 1st choice is the default value
    44                  choices: [ "v1beta1", "v1alpha1"])
    45          string (name: 'TAGGED_TESTS',
    46                  defaultValue: '',
    47                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    48                  trim: true)
    49          string (name: 'INCLUDED_TESTS',
    50                  defaultValue: '.*',
    51                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    52                  trim: true)
    53          string (name: 'EXCLUDED_TESTS',
    54                  defaultValue: '_excluded_test',
    55                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    56                  trim: true)
    57          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    58          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    59          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    60          choice (name:'WAIT_FOR_RESOURCE_BEFORE_UPDATE',
    61                  description: 'The update to the VZ CR will occur after this specified resource is created. This affects the timing of the update.',
    62                  choices: [ "mysql statefulset", "rancher bootstrap secret"])
    63      }
    64  
    65      environment {
    66          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    67          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    68          GOPATH = '/home/opc/go'
    69          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    70          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    71          DOCKER_EMAIL = credentials('github-packages-email')
    72          DOCKER_REPO = 'ghcr.io'
    73          DOCKER_NAMESPACE = 'verrazzano'
    74          NETRC_FILE = credentials('netrc')
    75          CLUSTER_NAME = 'verrazzano'
    76          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    77          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    78          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    79          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    80          OCR_CREDS = credentials('ocr-pull-and-push-account')
    81          OCR_REPO = 'container-registry.oracle.com'
    82          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    83          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-with-persistence.yaml"
    84          INSTALL_PROFILE = "dev"
    85          VZ_ENVIRONMENT_NAME = "default"
    86          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    87          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    88  
    89          // Environment variables required to capture cluster snapshot and bug report on test failure
    90          DUMP_KUBECONFIG="${KUBECONFIG}"
    91          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
    92          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
    93          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
    94  
    95          // Environment variable for Verrazzano CLI executable
    96          VZ_COMMAND="${GO_REPO_PATH}/vz"
    97  
    98          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
    99          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   100  
   101          // used for console artifact capture on failure
   102          JENKINS_READ = credentials('jenkins-auditor')
   103          OCI_CLI_AUTH="instance_principal"
   104          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   105          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   106          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   107          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   108  
   109          // used to emit metrics
   110          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   111          TEST_ENV_LABEL = "kind"
   112          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   113          TEST_ENV = "KIND"
   114          SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
   115          SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
   116          SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
   117  
   118          // used to generate Ginkgo test reports
   119          TEST_REPORT = "test-report.xml"
   120          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   121          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   122  
   123          // how many nodes in the KIND cluster
   124          KIND_NODE_COUNT = 3
   125      }
   126  
   127      stages {
   128          stage('Clean workspace and checkout') {
   129              steps {
   130                  sh """
   131                      echo "${NODE_LABELS}"
   132                  """
   133  
   134                  script {
   135                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   136                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   137                          echo "Specific GIT commit was not specified, use current head"
   138                          def scmInfo = checkout scm
   139                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   140                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   141                      } else {
   142                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   143                          def scmInfo = checkout([
   144                              $class: 'GitSCM',
   145                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   146                              doGenerateSubmoduleConfigurations: false,
   147                              extensions: [],
   148                              submoduleCfg: [],
   149                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   150                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   151                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   152                          // If the commit we were handed is not what the SCM says we are using, fail
   153                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   154                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   155                              exit 1
   156                          }
   157                      }
   158                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   159                  }
   160  
   161                  sh """
   162                      cp -f "${NETRC_FILE}" $HOME/.netrc
   163                      chmod 600 $HOME/.netrc
   164                  """
   165  
   166                  script {
   167                      try {
   168                      sh """
   169                          echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   170                      """
   171                      } catch(error) {
   172                          echo "docker login failed, retrying after sleep"
   173                          retry(4) {
   174                              sleep(30)
   175                              sh """
   176                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   177                              """
   178                          }
   179                      }
   180                  }
   181                  sh """
   182                      rm -rf ${GO_REPO_PATH}/verrazzano
   183                      mkdir -p ${GO_REPO_PATH}/verrazzano
   184                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   185                  """
   186  
   187                  script {
   188                      def props = readProperties file: '.verrazzano-development-version'
   189                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   190                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   191                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   192                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   193                      // update the description with some meaningful info
   194                      setDisplayName()
   195                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   196                  }
   197                  script {
   198                      sh """
   199                          echo "Downloading VZ CLI from object storage"
   200                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   201                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   202                          ${VZ_COMMAND} version
   203                      """
   204                  }
   205              }
   206          }
   207  
   208          // This kicks off two parallel stages:
   209          //     1. A stage to install Verrazzano, with a typo in the Verrazzano CRD
   210          //     2. A stage that waits a certain amount of time, then updates the Verrazzano CRD to fix the typo while the first stage is installing
   211          // This aims to test that updates to the Verrazzano CRD during initial install does not break the installation.
   212          stage('Update During Install') {
   213              parallel {
   214                  stage('Install Verrazzano with Wildcard DNS Typo') {
   215                      environment {
   216                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   217                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   218                      }
   219                      steps {
   220                          // This installs Verrazzano with an invalid WildcardDNS domain of "typodomain.io"
   221                          // This also makes sure that certain components relevant to testing are enabled
   222                          sh """
   223                              cd ${GO_REPO_PATH}/verrazzano
   224                              yq -i eval '.spec.components.keycloak.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   225                              yq -i eval '.spec.components.rancher.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   226                              ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} "typodomain.io"
   227                          """
   228                      }
   229                      post {
   230                          success {
   231                              script {
   232                                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   233                                      dumpK8sCluster('dynamic-install-post-install-cluster-snapshot')
   234                                  }
   235                              }
   236                          }
   237                          failure {
   238                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   239                          }
   240                          always {
   241                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true
   242                          }
   243                      }
   244                  }
   245  
   246                  stage('Update the VZ CR') {
   247                      options {
   248                          timeout(time: 30, unit: "MINUTES")
   249                      }
   250                      steps {
   251                          script {
   252                              // first wait for certain resources to be created, then update the Verrazzano CRD to fix the WildcardDNS typo
   253                              if (params.WAIT_FOR_RESOURCE_BEFORE_UPDATE == 'mysql statefulset') {
   254                                  waitForKeycloakMySqlStatefulset()
   255                              } else if (params.WAIT_FOR_RESOURCE_BEFORE_UPDATE == 'rancher bootstrap secret') {
   256                                  waitForRancherBootstrapSecret()
   257                              }
   258                              patchWildcardDNSDomain()
   259                          }
   260                      }
   261                  }
   262              }
   263  
   264              post {
   265                  failure {
   266                      script {
   267                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   268                              dumpK8sCluster('dynamic-install-failed-cluster-snapshot')
   269                          }
   270                      }
   271                  }
   272                  success {
   273                      script {
   274                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   275                              dumpK8sCluster('dynamic-install-success-cluster-snapshot')
   276                          }
   277                      }
   278                  }
   279              }
   280          }
   281  
   282          stage('Verify Tests') {
   283              environment {
   284                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-install-verify-infra"
   285              }
   286              steps {
   287                  runGinkgoRandomize('verify-install')
   288              }
   289              post {
   290                  failure {
   291                      script {
   292                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   293                              dumpK8sCluster('dynamic-install-verify-install-failed-cluster-snapshot')
   294                          }
   295                      }
   296                  }
   297                  always {
   298                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   299                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   300                  }
   301              }
   302          }
   303  
   304          stage('Infra Tests') {
   305              environment {
   306                  DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-install-infra-tests"
   307              }
   308              steps {
   309                  script {
   310                      parallel generateVerifyInfraStages("${TEST_DUMP_ROOT}/post-install-infra-tests")
   311                  }
   312              }
   313              post {
   314                  failure {
   315                      script {
   316                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   317                              dumpK8sCluster('dynamic-install-verify-infra-failed-cluster-snapshot')
   318                          }
   319                      }
   320                  }
   321                  always {
   322                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   323                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   324                  }
   325              }
   326          }
   327      }
   328  
   329      post {
   330          always {
   331              script {
   332                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   333                      dumpVerrazzanoSystemPods()
   334                      dumpCattleSystemPods()
   335                      dumpNginxIngressControllerLogs()
   336                      dumpVerrazzanoPlatformOperatorLogs("post-run")
   337                      dumpVerrazzanoApplicationOperatorLogs()
   338                      dumpOamKubernetesRuntimeLogs()
   339                      dumpVerrazzanoApiLogs()
   340                  }
   341              }
   342              sh """
   343                  # Copy the generated test reports to WORKSPACE to archive them
   344                  mkdir -p ${TEST_REPORT_DIR}
   345                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   346                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   347              """
   348              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   349              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   350              deleteCluster()
   351          }
   352          failure {
   353              sh """
   354                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   355              """
   356              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   357              sh """
   358                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   359                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   360                  rm archive.zip
   361              """
   362          }
   363          cleanup {
   364              deleteDir()
   365          }
   366      }
   367  }
   368  
   369  def generateVerifyInfraStages(dumpRoot) {
   370      return [
   371          "verify-infra restapi": {
   372              runGinkgoRandomize('verify-infra/restapi', "${dumpRoot}/verify-infra-restapi")
   373          },
   374          "verify-infra oam": {
   375              runGinkgoRandomize('verify-infra/oam', "${dumpRoot}/verify-infra-oam")
   376          },
   377      ]
   378  }
   379  
   380  def runGinkgoRandomize(testSuitePath, dumpDir = '') {
   381      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   382          sh """
   383              if [ ! -z "${dumpDir}" ]; then
   384                  export DUMP_DIRECTORY=${dumpDir}
   385              fi
   386              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   387              ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   388          """
   389      }
   390  }
   391  
   392  def dumpK8sCluster(dumpDirectory) {
   393      sh """
   394          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   395      """
   396  }
   397  
   398  def dumpVerrazzanoSystemPods() {
   399      sh """
   400          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   401          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   402          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   403          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   404          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   405          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   406          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   407          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   408          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   409      """
   410  }
   411  
   412  def dumpCattleSystemPods() {
   413      sh """
   414          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   415          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   416          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   417          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   418          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   419      """
   420  }
   421  
   422  def dumpNginxIngressControllerLogs() {
   423      sh """
   424          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   425          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   426          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   427      """
   428  }
   429  
   430  def dumpVerrazzanoPlatformOperatorLogs(stage) {
   431      sh """
   432          ## dump out verrazzano-platform-operator logs
   433          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   434          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   435          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   436          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   437          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   438          echo "------------------------------------------"
   439      """
   440  }
   441  
   442  def dumpVerrazzanoApplicationOperatorLogs() {
   443      sh """
   444          ## dump out verrazzano-application-operator logs
   445          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   446          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   447          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   448          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   449          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   450          echo "------------------------------------------"
   451      """
   452  }
   453  
   454  def dumpOamKubernetesRuntimeLogs() {
   455      sh """
   456          ## dump out oam-kubernetes-runtime logs
   457          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   458          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   459          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   460          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   461          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   462          echo "------------------------------------------"
   463      """
   464  }
   465  
   466  def dumpVerrazzanoApiLogs() {
   467      sh """
   468          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   469          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-api.log"
   470          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-api-*" -m "verrazzano api" -c verrazzano-api -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   471      """
   472  }
   473  
   474  def getEffectiveDumpOnSuccess() {
   475      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   476      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   477          effectiveValue = true
   478          echo "Forcing dump on success based on global override setting"
   479      }
   480      return effectiveValue
   481  }
   482  
   483  def deleteCluster() {
   484      sh """
   485          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   486          make delete-cluster
   487          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   488            echo "Failures seen during dumping of artifacts, treat post as failed"
   489            exit 1
   490          fi
   491      """
   492  }
   493  
   494  def setDisplayName() {
   495      echo "Start setDisplayName"
   496      def causes = currentBuild.getBuildCauses()
   497      echo "causes: " + causes.toString()
   498      for (cause in causes) {
   499          def causeString = cause.toString()
   500          echo "current cause: " + causeString
   501          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   502               echo "This job was caused by " + causeString
   503               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   504                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   505               } else if (causeString.contains("verrazzano-flaky-tests")) {
   506                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   507               }
   508           }
   509      }
   510      echo "End setDisplayName"
   511  }
   512  
   513  def patchWildcardDNSDomain() {
   514      sh """
   515          echo "WildcardDNS Configuration before patching:"
   516          kubectl get vz my-verrazzano -o yaml | grep 'domain:'
   517  
   518          # change the Wildcard DNS domain
   519          kubectl patch vz my-verrazzano -p '{"spec":{"components":{"dns":{"wildcard":{"domain":"${params.WILDCARD_DNS_DOMAIN}"}}}}}' --type=merge
   520  
   521          echo "WildcardDNSConfiguration after patching:"
   522          kubectl get vz my-verrazzano -o yaml | grep 'domain:'
   523      """
   524  }
   525  
   526  def waitForRancherBootstrapSecret() {
   527      sh """
   528          while ! kubectl get vz
   529          do
   530              echo "Waiting for verrazzano resource to be created..."
   531              sleep 30s
   532          done
   533  
   534          while ! kubectl get secret bootstrap-secret --namespace cattle-system
   535          do
   536              echo "Waiting for bootstrap-secret Secret to be created..."
   537              sleep 1m
   538          done
   539      """
   540  }
   541  
   542  def waitForKeycloakMySqlStatefulset() {
   543      sh """
   544          while ! kubectl get vz
   545          do
   546              echo "Waiting for verrazzano resource to be created..."
   547              sleep 30s
   548          done
   549  
   550          while ! kubectl get statefulset mysql -n keycloak
   551          do
   552              echo "Waiting for keycloak/mysql statefuleset to be created..."
   553              sleep 15s
   554          done
   555      """
   556  }