github.com/verrazzano/verrazzano@v1.7.0/ci/uninstall/JenkinsfileUninstallTest (about)

     1  // Copyright (c) 2022, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          timeout(time: 200, unit: 'MINUTES')
    11          skipDefaultCheckout true
    12          timestamps ()
    13      }
    14  
    15      agent {
    16          docker {
    17              image "${RUNNER_DOCKER_IMAGE}"
    18              args "${RUNNER_DOCKER_ARGS}"
    19              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    20              registryCredentialsId 'ocir-pull-and-push-account'
    21              label "${agentLabel}"
    22          }
    23      }
    24  
    25      parameters {
    26          choice (name: 'CHAOS_TEST_TYPE',
    27                  description: 'Type of chaos to inflict',
    28                  // 1st choice is the default value
    29                  choices: [ "uninstall.interrupt.uninstall", "uninstall.reinstall.loop" ])
    30  
    31          string (name: 'INSTALL_LOOP_COUNT',
    32                  description: 'Install loop count, valid for test type uninstall.reinstall.loop only',
    33                  defaultValue: "1")
    34  
    35          choice (name: 'INSTALL_PROFILE',
    36                  description: 'Verrazzano install profile name',
    37                  // 1st choice is the default value
    38                  choices: [ "prod", "dev", "managed-cluster" ])
    39  
    40          booleanParam (name: 'RUN_APPLICATION_TESTS',
    41                  description: 'Whether to run the application tests for the final install loop',
    42                  defaultValue: true)
    43  
    44          choice (name: 'WILDCARD_DNS_DOMAIN',
    45                  description: 'This is the wildcard DNS domain',
    46                  // 1st choice is the default value
    47                  choices: [ "nip.io", "sslip.io" ])
    48  
    49          choice (name: 'CRD_API_VERSION',
    50                  description: 'This is the API crd version.',
    51                  // 1st choice is the default value
    52                  choices: [ "v1beta1", "v1alpha1"])
    53  
    54          string (name: 'CONSOLE_REPO_BRANCH',
    55                  defaultValue: '',
    56                  description: 'The branch to check out after cloning the console repository.',
    57                  trim: true)
    58          string (name: 'GIT_COMMIT_TO_USE',
    59                  defaultValue: 'NONE',
    60                  description: 'This is the full git commit hash from the source build to be used for all jobs',
    61                  trim: true)
    62          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    63                  defaultValue: 'NONE',
    64                  description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator',
    65                  trim: true)
    66          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    67                  description: 'Kubernetes Version for KinD Cluster',
    68                  // 1st choice is the default value
    69                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    70  
    71          booleanParam (description: 'Whether to create kind cluster with Calico for AT testing (defaults to true)', name: 'CREATE_KIND_USE_CALICO', defaultValue: true)
    72          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    73  
    74          string (name: 'TAGGED_TESTS',
    75                  defaultValue: '',
    76                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    77                  trim: true)
    78          string (name: 'INCLUDED_TESTS',
    79                  defaultValue: '.*',
    80                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    81                  trim: true)
    82          string (name: 'EXCLUDED_TESTS',
    83                  defaultValue: '_excluded_test',
    84                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    85                  trim: true)
    86          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    87      }
    88  
    89      environment {
    90          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    91          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    92          GOPATH = '/home/opc/go'
    93          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    94          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    95          DOCKER_EMAIL = credentials('github-packages-email')
    96          DOCKER_REPO = 'ghcr.io'
    97          DOCKER_NAMESPACE = 'verrazzano'
    98          NETRC_FILE = credentials('netrc')
    99          SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY')
   100  
   101          // Environment variables required to capture cluster snapshot and bug report on test failure
   102          DUMP_KUBECONFIG="${KUBECONFIG}"
   103          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   104          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   105          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
   106  
   107          // Environment variable for Verrazzano CLI executable
   108          VZ_COMMAND="${GO_REPO_PATH}/vz"
   109  
   110          CLUSTER_NAME = 'verrazzano'
   111          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
   112          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
   113          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
   114          ADMIN_KUBECONFIG = "${KUBECONFIG}"
   115          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
   116          OCR_CREDS = credentials('ocr-pull-and-push-account')
   117          OCR_REPO = 'container-registry.oracle.com'
   118          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
   119  
   120          INSTALL_CONFIG_FILE_KIND_PERSISTENCE = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-with-persistence.yaml"
   121          INSTALL_CONFIG_FILE_KIND_EPHEMERAL = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-no-persistence.yaml"
   122          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-no-persistence.yaml"
   123  
   124          VZ_ENVIRONMENT_NAME = "default"
   125          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
   126          LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test"
   127  
   128          // Location to store Platform Operator manifest
   129          TARGET_OPERATOR_FILE="${WORKSPACE}/acceptance-test-operator.yaml"
   130          // Location to store VZ install file, used to install/re-install VZ
   131          VZ_INSTALL_FILE="${WORKSPACE}/acceptance-test-config.yaml"
   132  
   133          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test
   134          DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test
   135  
   136          OLD_PODS_FILE="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-old-vpo-app-pods.out"
   137          NEW_PODS_FILE="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-new-vpo-app-pods.out"
   138          NEW_PODS_FILE2="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-new-vpo-app-pods-2.out"
   139          APP_NAMEPACES="'todo-list bobs-books hello-helidon springboot sockshop'"
   140  
   141          // used to emit metrics
   142          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   143          TEST_ENV_LABEL = "kind"
   144          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   145          TEST_ENV = "KIND"
   146          SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
   147          SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
   148          SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
   149  
   150          // used to generate Ginkgo test reports
   151          TEST_REPORT = "test-report.xml"
   152          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   153          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   154      }
   155  
   156      stages {
   157          stage('Clean workspace and checkout') {
   158              environment {
   159                  OCI_CLI_AUTH="instance_principal"
   160                  OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   161                  OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   162                  OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   163                  VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   164              }
   165              steps {
   166                  sh """
   167                      echo "${NODE_LABELS}"
   168                  """
   169                  script {
   170                     EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   171                     if (params.GIT_COMMIT_TO_USE == "NONE") {
   172                          echo "Specific GIT commit was not specified, use current head"
   173                          def scmInfo = checkout scm
   174                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   175                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   176                      } else {
   177                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   178                          def scmInfo = checkout([
   179                              $class: 'GitSCM',
   180                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   181                              doGenerateSubmoduleConfigurations: false,
   182                              extensions: [],
   183                              submoduleCfg: [],
   184                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   185                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   186                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   187                          // If the commit we were handed is not what the SCM says we are using, fail
   188                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   189                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   190                              exit 1
   191                          }
   192                      }
   193                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   194                  }
   195                  sh """
   196                      cp -f "${NETRC_FILE}" $HOME/.netrc
   197                      chmod 600 $HOME/.netrc
   198                  """
   199  
   200                  script {
   201                      try {
   202                          sh """
   203                              echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   204                          """
   205                      } catch(error) {
   206                          echo "docker login failed, retrying after sleep"
   207                          retry(4) {
   208                              sleep(30)
   209                              sh """
   210                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   211                              """
   212                          }
   213                      }
   214                  }
   215  
   216                  sh """
   217                      rm -rf ${GO_REPO_PATH}/verrazzano
   218                      mkdir -p ${GO_REPO_PATH}/verrazzano
   219                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   220                      cd ${GO_REPO_PATH}/verrazzano
   221                      git config --global credential.helper "!f() { echo username=\\$DOCKER_CREDS_USR; echo password=\\$DOCKER_CREDS_PSW; }; f"
   222                      git config --global user.name $DOCKER_CREDS_USR
   223                      git config --global user.email "${DOCKER_EMAIL}"
   224                      git checkout -b ${env.BRANCH_NAME}
   225                  """
   226  
   227                  script {
   228                      def props = readProperties file: '.verrazzano-development-version'
   229                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   230                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   231                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   232                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   233                      // update the description with some meaningful info
   234                      setDisplayName()
   235                      currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   236                  }
   237  
   238                  script {
   239                      sh """
   240                          echo "Downloading VZ CLI from object storage"
   241                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   242                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   243                          ${VZ_COMMAND} version
   244                      """
   245                  }
   246              }
   247          }
   248  
   249  		stage ('Uninstall Tests') {
   250  		    environment {
   251                  KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   252                  OCI_CLI_AUTH="instance_principal"
   253                  OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   254                  OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   255                  OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   256                  CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   257  		    }
   258  		    steps {
   259  		        script {
   260  		            int maxIterations = getMaxIterations()
   261                      for (int count = 1; count <= maxIterations; count++) {
   262                          if (count == 1) {
   263                              // Create cluster and install
   264                              runInitialInstall(count)
   265                          } else {
   266                              // Run just the install again
   267                              runInstallOnly("Install Verrazzano", count)
   268                          }
   269                          runVerifyTests("Run Test", count)
   270                          if (params.RUN_APPLICATION_TESTS && count == maxIterations && params.INSTALL_PROFILE != 'managed-cluster') {
   271                              runApplicationTests()
   272                          }
   273                          runUninstall(count)
   274                          if (params.CHAOS_TEST_TYPE == "uninstall.interrupt.uninstall") {
   275                              // Run re-install once and re-verify
   276                              runInstallOnly("Reinstall Verrazzano", count)
   277                              runVerifyTests("Rerun Test", count)
   278                          }
   279                      }
   280  		        }
   281  		    }
   282  		}
   283      }
   284      post {
   285          always {
   286              runnerCleanup()
   287          }
   288          cleanup {
   289              deleteDir()
   290          }
   291      }
   292  }
   293  
   294  def getMaxIterations() {
   295      int maxIterations = 1
   296      if (params.CHAOS_TEST_TYPE == "uninstall.reinstall.loop") {
   297          maxIterations = params.INSTALL_LOOP_COUNT.toInteger()
   298          echo "Max loop count: $maxIterations"
   299      }
   300      return maxIterations
   301  }
   302  
   303  def runInitialInstall(iteration) {
   304      stage("Install Verrazzano #${iteration}") {
   305          env.CLUSTER_SNAPSHOT_DIR="${WORKSPACE}/verrazzano/build/resources-${iteration}/pre-install-resources"
   306          try {
   307              script {
   308                  sh """
   309                      echo "CLUSTER_SNAPSHOT_DIR: ${CLUSTER_SNAPSHOT_DIR}"
   310                      echo "Branch: ${BRANCH_NAME}"
   311                      echo "Operator Image: ${VERRAZZANO_OPERATOR_IMAGE}"
   312                      cd ${GO_REPO_PATH}/verrazzano
   313                      ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_KIND_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
   314                  """
   315              }
   316              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   317                  dumpK8sCluster("install-snapshot-${iteration}")
   318              }
   319          } catch (err) {
   320              dumpK8sCluster("install-snapshot-${iteration}")
   321              throw err
   322          } finally {
   323              dumpPodsAndLogs("post-install-${iteration}")
   324              archiveArtifacts artifacts: "**/logs/**,${env.INSTALL_CONFIG_FILE_KIND}", allowEmptyArchive: true
   325          }
   326      }
   327  }
   328  
   329  def runUninstall(iteration) {
   330      stage("Uninstall Verrazzano #${iteration}") {
   331          try {
   332              vzUninstall(iteration)
   333              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   334                  dumpK8sCluster("uninstall-snapshot-${iteration}")
   335              }
   336          } catch (err) {
   337              dumpK8sCluster("uninstall-failure-snapshot-${iteration}")
   338              throw err
   339          } finally {
   340              dumpPodsAndLogs("post-uninstall-${iteration}")
   341              listNamespacesAndPods('after Verrazzano uninstall')
   342              listHelmReleases('after Verrazzano uninstall')
   343          }
   344      }
   345      stage("Verify Uninstall #${iteration}") {
   346          try {
   347              sh """
   348                  ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources-${iteration}/post-uninstall-resources false
   349                  ${LOOPING_TEST_SCRIPTS_DIR}/verify_uninstall.sh ${WORKSPACE}/verrazzano/build/resources-${iteration}
   350              """
   351              parallel generateVerifyUninstallStages()
   352              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   353                  dumpK8sCluster("verify-uninstall-snapshot-${iteration}")
   354              }
   355          } catch (err) {
   356              dumpK8sCluster("verify-uninstall-snapshot-${iteration}")
   357              throw err
   358          }
   359      }
   360  }
   361  
   362  def runInstallOnly(stagePrefix, iteration) {
   363      def dirPrefix=stagePrefix.replaceAll("\\s","-").toLowerCase()
   364      stage("${stagePrefix} #${iteration}") {
   365          def vpoLogsDir = "${WORKSPACE}/verrazzano-platform-operator/scripts/reinstall/build/logs/${iteration}"
   366          echo "Platform operator logs dir: ${vpoLogsDir}"
   367          try {
   368              sh """
   369                  # sleep for a period to ensure async deletion of Verrazzano components from uninstall above has completed
   370                  #sleep 90
   371                  ${VZ_COMMAND} install --filename ${VZ_INSTALL_FILE} --manifests ${TARGET_OPERATOR_FILE} --timeout 20m
   372              """
   373              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   374                 dumpK8sCluster("install-snapshot-${iteration}")
   375              }
   376          } catch (err) {
   377              dumpK8sCluster("install-failed-snapshot-${iteration}")
   378              sh """
   379                 mkdir -p ${vpoLogsDir}
   380                 ${LOOPING_TEST_SCRIPTS_DIR}/dump_resources.sh > ${vpoLogsDir}/resources.log
   381              """
   382              throw err
   383          } finally {
   384              dumpPodsAndLogs("${dirPrefix}-${iteration}")
   385              listNamespacesAndPods('after reinstalling Verrazzano')
   386              listHelmReleases('after reinstalling Verrazzano')
   387          }
   388      }
   389  }
   390  
   391  def runVerifyTests(stageName, iteration) {
   392      stage("${stageName} #${iteration}") {
   393          try {
   394              parallel generateVerifyInfraStages()
   395              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   396                  dumpK8sCluster("verify-test-snapshot-${iteration}")
   397              }
   398          } catch (err) {
   399              dumpK8sCluster("verify-test-failure-snapshot-${iteration}")
   400              throw err
   401          }
   402      }
   403  }
   404  
   405  def generateVerifyUninstallStages() {
   406      boolean isManagedCluster = params.INSTALL_PROFILE == 'managed-cluster'
   407      uninstallTests = [
   408          "verifycrds": {
   409              runGinkgoWithPassThroughs('uninstall/verifycrds', isManagedCluster)
   410          },
   411          "verifyrancher": {
   412              runGinkgo('uninstall/verifyrancher')
   413          },
   414      ]
   415  
   416      return uninstallTests
   417  }
   418  
   419  def generateVerifyInfraStages() {
   420      infraTests = [
   421          "verify-install": {
   422              runGinkgo('verify-install')
   423          },
   424          "verify-scripts": {
   425              runGinkgo('scripts', '', "${KUBECONFIG}")
   426          },
   427          "verify-infra": {
   428              runGinkgoRandomize('verify-infra')
   429          },
   430      ]
   431      if (params.INSTALL_PROFILE != 'managed-cluster') {
   432          infraTests = infraTests + [
   433              "system component metrics": {
   434                  runGinkgoRandomize('metrics/syscomponents')
   435              },
   436              "console": {
   437                  acceptanceTestsConsole()
   438              },
   439          ]
   440      }
   441      return infraTests
   442  }
   443  
   444  // vzUninstall - Start parallel uninstall and chaos stages
   445  def vzUninstall(iteration) {
   446      stage("Uninstall #${iteration}") {
   447          script {
   448              def vpoLogsDir = "${WORKSPACE}/verrazzano-platform-operator/logs/${iteration}"
   449              echo "Platform operator uninstall logs dir: ${vpoLogsDir}"
   450              sh (script: "mkdir -p ${vpoLogsDir}")
   451              parallel getUninstallStages(vpoLogsDir)
   452          }
   453      }
   454  }
   455  
   456  def getUninstallStages(vpoLogsDir) {
   457      stages = [
   458           "uninstall-verrazzano": {
   459                  uninstallVerrazzano(vpoLogsDir)
   460               },
   461      ]
   462      if (params.CHAOS_TEST_TYPE == "uninstall.interrupt.uninstall") {
   463          stages = stages + [
   464               "uninstall-chaos": {
   465                      kill_vpo_loop(vpoLogsDir)
   466                   },
   467          ]
   468      }
   469      return stages
   470  }
   471  
   472  def uninstallVerrazzano(vpoLogsDir) {
   473      script {
   474          echo "Platform operator uninstall logs dir: ${vpoLogsDir}"
   475  
   476          // Delete VZ in the background
   477          while (true) {
   478              echo "Deleting the Verrazzano resource..."
   479              result = sh(returnStatus: true, script: "${VZ_COMMAND} uninstall -y --timeout=20m")
   480              if (result == 0) {
   481                  break
   482              }
   483              echo "vz uninstall result: ${result}"
   484              sh "sleep 5"
   485          }
   486      }
   487  }
   488  
   489  def kill_vpo_loop(vpoLogsDir) {
   490      script {
   491          echo "Stopping the VPO on loop in 15 to 60 second intervals"
   492  
   493          sh """
   494              # Initially sleep for 30 to 60 seconds
   495              SLEEP_SEC=\$[ \$RANDOM % 30 + 30 ]
   496              echo "Sleeping for \$SLEEP_SEC seconds"
   497              sleep \$SLEEP_SEC
   498  
   499              VZ_STATUS=\$(kubectl get vz my-verrazzano --ignore-not-found -o jsonpath={.status.conditions[-1].type})
   500              RESTARTS=1
   501              while [ ! -z \$VZ_STATUS ] && [ \$VZ_STATUS == "UninstallStarted" ] ; do
   502                  NS_STATUS=\$(kubectl get ns verrazzano-install --ignore-not-found -o jsonpath={.status.phase})
   503                  if [ ! -z \$NS_STATUS ]; then
   504                      mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   505                      kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator >> ${vpoLogsDir}/verrazzano-platform-operator-before-vpo-killed-pod-\${RESTARTS}.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   506                      kubectl -n verrazzano-install delete pod -l app=verrazzano-platform-operator --ignore-not-found
   507                      kubectl -n verrazzano-install rollout status deployments/verrazzano-platform-operator --timeout=600s
   508                      echo "VPO pod successfully restarted"
   509                  fi
   510  
   511                  # Sleep a random interval between 15 and 60 seconds
   512                  SLEEP_SEC=\$[ \$RANDOM % 45 + 15 ]
   513                  echo "Sleeping for \$SLEEP_SEC seconds"
   514                  sleep \$SLEEP_SEC
   515  
   516                  # Update the Verrazzano status and update restart counter
   517                  VZ_STATUS=\$(kubectl get vz my-verrazzano --ignore-not-found -o jsonpath={.status.conditions[-1].type})
   518                  ((RESTARTS++))
   519              done
   520          """
   521      }
   522  }
   523  
   524  def runApplicationTests() {
   525      stage("Application Tests") {
   526          try {
   527              parallel getApplicationStages()
   528              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   529                  dumpK8sCluster("application-test-failure-snapshot")
   530              }
   531          } catch (err) {
   532              dumpK8sCluster("application-test-failure-snapshot")
   533              throw err
   534          }
   535      }
   536  }
   537  
   538  def getApplicationStages() {
   539      return [
   540         "examples helidon": {
   541             runGinkgo('examples/helidon', 'examples-helidon')
   542         },
   543         "examples socks": {
   544             runGinkgo('examples/socks', 'examples-socks')
   545         },
   546         "examples springboot": {
   547             runGinkgo('examples/springboot', 'examples-springboot')
   548         },
   549         "examples todo": {
   550             runGinkgo('examples/todo', 'examples-todo')
   551         },
   552     ]
   553  }
   554  
   555  // Called in parallel Stage console of Stage Run Acceptance Tests
   556  def acceptanceTestsConsole() {
   557      catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
   558          try {
   559              sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh"
   560          } catch (err) {
   561              saveConsoleScreenShots()
   562              error "${err}"
   563          }
   564      }
   565  }
   566  
   567  def saveConsoleScreenShots() {
   568      sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh"
   569  }
   570  
   571  def runGinkgoRandomize(testSuitePath, dumpDir = '') {
   572      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   573          sh """
   574              if [ ! -z "${dumpDir}" ]; then
   575                  export DUMP_DIRECTORY=${TEST_DUMP_ROOT}/${dumpDir}
   576              fi
   577              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   578              ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   579          """
   580      }
   581  }
   582  
   583  def runGinkgo(testSuitePath, dumpDir = '', kubeconfig = '') {
   584      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   585          sh """
   586              if [ ! -z "${dumpDir}" ]; then
   587                  export DUMP_DIRECTORY=${TEST_DUMP_ROOT}/${dumpDir}
   588              fi
   589              if [ ! -z "${kubeConfig}" ]; then
   590                  export KUBECONFIG="${kubeConfig}"
   591              fi
   592              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   593              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   594          """
   595      }
   596  }
   597  
   598  def runGinkgoWithPassThroughs(testSuitePath, isManagedCluster, dumpDir = '', kubeconfig = '') {
   599      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   600          sh """
   601              if [ ! -z "${dumpDir}" ]; then
   602                  export DUMP_DIRECTORY=${TEST_DUMP_ROOT}/${dumpDir}
   603              fi
   604              if [ ! -z "${kubeConfig}" ]; then
   605                  export KUBECONFIG="${kubeConfig}"
   606              fi
   607              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   608              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --isManagedCluster=${isManagedCluster}
   609          """
   610      }
   611  }
   612  
   613  def runnerCleanup() {
   614      script {
   615          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   616              dumpPodsAndLogs("runnerCleanup")
   617          }
   618      }
   619  
   620      sh """
   621          # Copy the generated test reports to WORKSPACE to archive them
   622          mkdir -p ${TEST_REPORT_DIR}
   623          cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   624          find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   625      """
   626      archiveArtifacts artifacts: "**/coverage.html,${WORKSPACE}/console/**,**/logs/**,**/verrazzano_images.txt,**/build/resources/**,**/*full-cluster*/**,**/bug-report/**,**/${TEST_REPORT}", allowEmptyArchive: true
   627      junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   628  
   629      sh """
   630          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   631          make delete-cluster
   632          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   633            echo "Failures seen during dumping of artifacts, treat post as failed"
   634            exit 1
   635          fi
   636      """
   637  }
   638  
   639  def dumpK8sCluster(dumpDirectory) {
   640      sh """
   641          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${TEST_DUMP_ROOT}/${dumpDirectory}
   642      """
   643  }
   644  
   645  def dumpPodsAndLogs(stage) {
   646      def dumpPath="${WORKSPACE}/verrazzano/logs/${stage}"
   647      dumpVerrazzanoSystemPods(dumpPath)
   648      dumpCattleSystemPods(dumpPath)
   649      dumpNginxIngressControllerLogs(dumpPath)
   650      dumpVerrazzanoPlatformOperatorLogs(dumpPath)
   651      dumpVerrazzanoApplicationOperatorLogs(dumpPath)
   652      dumpOamKubernetesRuntimeLogs(dumpPath)
   653      dumpVerrazzanoApiLogs(dumpPath)
   654  }
   655  
   656  def dumpVerrazzanoSystemPods(dumpPath) {
   657      createPathIfNecessary(dumpPath)
   658      sh """
   659          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   660          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-system-pods.log"
   661          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   662          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-system-certs.log"
   663          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   664          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-system-osd.log"
   665          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   666          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-system-es-master.log"
   667          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   668      """
   669  }
   670  
   671  def dumpCattleSystemPods(dumpPath) {
   672      createPathIfNecessary(dumpPath)
   673      sh """
   674          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   675          export DIAGNOSTIC_LOG="${dumpPath}/cattle-system-pods.log"
   676          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   677          export DIAGNOSTIC_LOG="${dumpPath}/rancher.log"
   678          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   679      """
   680  }
   681  
   682  def dumpNginxIngressControllerLogs(dumpPath) {
   683      sh """
   684          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   685          export DIAGNOSTIC_LOG="${dumpPath}/nginx-ingress-controller.log"
   686          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   687      """
   688  }
   689  
   690  def dumpVerrazzanoPlatformOperatorLogs(dumpPath) {
   691      createPathIfNecessary(dumpPath)
   692      sh """
   693          ## dump out verrazzano-platform-operator logs
   694          kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${dumpPath}/verrazzano-platform-operator-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   695          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${dumpPath}/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   696          echo "------------------------------------------"
   697      """
   698  }
   699  
   700  def dumpVerrazzanoApplicationOperatorLogs(dumpPath) {
   701      createPathIfNecessary(dumpPath)
   702      sh """
   703          ## dump out verrazzano-application-operator logs
   704          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator --tail -1 > ${dumpPath}/verrazzano-application-operator-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   705          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${dumpPath}/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   706          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   707          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   708          echo "------------------------------------------"
   709      """
   710  }
   711  
   712  def dumpOamKubernetesRuntimeLogs(dumpPath) {
   713      createPathIfNecessary(dumpPath)
   714      sh """
   715          ## dump out oam-kubernetes-runtime logs
   716          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime --tail -1 > ${dumpPath}/oam-kubernetes-runtime-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   717          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${dumpPath}/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   718          echo "oam-kubernetes-runtime logs dumped to oam-kubernetes-runtime-pod.log"
   719          echo "oam-kubernetes-runtime pod description dumped to oam-kubernetes-runtime-pod.out"
   720          echo "------------------------------------------"
   721      """
   722  }
   723  
   724  def dumpVerrazzanoApiLogs(dumpPath) {
   725      createPathIfNecessary(dumpPath)
   726      sh """
   727          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   728          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-authproxy.log"
   729          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   730      """
   731  }
   732  
   733  def createPathIfNecessary(pathName) {
   734      sh """
   735          if [ ! -e ${pathName} ]; then
   736              echo "Creating ${pathName}"
   737              mkdir -p ${pathName}
   738          else
   739              echo "Dump path ${pathName} exists"
   740          fi
   741      """
   742  }
   743  
   744  def setDisplayName() {
   745      echo "Start setDisplayName"
   746      def causes = currentBuild.getBuildCauses()
   747      echo "causes: " + causes.toString()
   748      for (cause in causes) {
   749          def causeString = cause.toString()
   750          echo "current cause: " + causeString
   751          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   752               echo "This job was caused by " + causeString
   753               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   754                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   755               } else if (causeString.contains("verrazzano-flaky-tests")) {
   756                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   757               }
   758           }
   759      }
   760      echo "End setDisplayName"
   761  }
   762  
   763  def getEffectiveDumpOnSuccess() {
   764      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   765      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   766          effectiveValue = true
   767          echo "Forcing dump on success based on global override setting"
   768      }
   769      return effectiveValue
   770  }
   771  
   772  def listHelmReleases(customMessage) {
   773      sh """
   774          echo "Listing the releases across all namespaces ${customMessage}."
   775          helm list -A
   776          echo "-----------------------------------------------------"
   777      """
   778  }
   779  
   780  def listNamespacesAndPods(customMessage) {
   781      sh """
   782          echo "Listing all the namespaces and pods the namespaces ${customMessage}."
   783          kubectl get namespaces
   784          kubectl get pods -A
   785          echo "-----------------------------------------------------"
   786      """
   787  }