github.com/verrazzano/verrazzano@v1.7.1/ci/uninstall/JenkinsfileUninstallTest (about)

     1  // Copyright (c) 2022, 2024, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          timeout(time: 200, unit: 'MINUTES')
    11          skipDefaultCheckout true
    12          timestamps ()
    13      }
    14  
    15      agent {
    16          docker {
    17              image "${RUNNER_DOCKER_IMAGE}"
    18              args "${RUNNER_DOCKER_ARGS}"
    19              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    20              registryCredentialsId 'ocir-pull-and-push-account'
    21              label "${agentLabel}"
    22          }
    23      }
    24  
    25      parameters {
    26          choice (name: 'CHAOS_TEST_TYPE',
    27                  description: 'Type of chaos to inflict',
    28                  // 1st choice is the default value
    29                  choices: [ "uninstall.interrupt.uninstall", "uninstall.reinstall.loop" ])
    30  
    31          string (name: 'INSTALL_LOOP_COUNT',
    32                  description: 'Install loop count, valid for test type uninstall.reinstall.loop only',
    33                  defaultValue: "1")
    34  
    35          choice (name: 'INSTALL_PROFILE',
    36                  description: 'Verrazzano install profile name',
    37                  // 1st choice is the default value
    38                  choices: [ "prod", "dev", "managed-cluster" ])
    39  
    40          booleanParam (name: 'RUN_APPLICATION_TESTS',
    41                  description: 'Whether to run the application tests for the final install loop',
    42                  defaultValue: true)
    43  
    44          choice (name: 'WILDCARD_DNS_DOMAIN',
    45                  description: 'This is the wildcard DNS domain',
    46                  // 1st choice is the default value
    47                  choices: [ "nip.io", "sslip.io" ])
    48  
    49          choice (name: 'CRD_API_VERSION',
    50                  description: 'This is the API crd version.',
    51                  // 1st choice is the default value
    52                  choices: [ "v1beta1", "v1alpha1"])
    53  
    54          string (name: 'CONSOLE_REPO_BRANCH',
    55                  defaultValue: '',
    56                  description: 'The branch to check out after cloning the console repository.',
    57                  trim: true)
    58          string (name: 'GIT_COMMIT_TO_USE',
    59                  defaultValue: 'NONE',
    60                  description: 'This is the full git commit hash from the source build to be used for all jobs',
    61                  trim: true)
    62          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    63                  defaultValue: 'NONE',
    64                  description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator',
    65                  trim: true)
    66          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    67                  description: 'Kubernetes Version for KinD Cluster',
    68                  // 1st choice is the default value
    69                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    70  
    71          booleanParam (description: 'Whether to create kind cluster with Calico for AT testing (defaults to true)', name: 'CREATE_KIND_USE_CALICO', defaultValue: true)
    72          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    73  
    74          string (name: 'TAGGED_TESTS',
    75                  defaultValue: '',
    76                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    77                  trim: true)
    78          string (name: 'INCLUDED_TESTS',
    79                  defaultValue: '.*',
    80                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    81                  trim: true)
    82          string (name: 'EXCLUDED_TESTS',
    83                  defaultValue: '_excluded_test',
    84                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    85                  trim: true)
    86          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    87      }
    88  
    89      environment {
    90          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    91          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    92          GOPATH = '/home/opc/go'
    93          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    94          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    95          DOCKER_EMAIL = credentials('github-packages-email')
    96          DOCKER_REPO = 'ghcr.io'
    97          DOCKER_NAMESPACE = 'verrazzano'
    98          NETRC_FILE = credentials('netrc')
    99          SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY')
   100  
   101          // Environment variables required to capture cluster snapshot and bug report on test failure
   102          DUMP_KUBECONFIG="${KUBECONFIG}"
   103          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   104          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   105          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
   106  
   107          // Environment variable for Verrazzano CLI executable
   108          VZ_COMMAND="${GO_REPO_PATH}/vz"
   109  
   110          CLUSTER_NAME = 'verrazzano'
   111          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
   112          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
   113          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
   114          ADMIN_KUBECONFIG = "${KUBECONFIG}"
   115          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
   116          OCR_CREDS = credentials('ocr-pull-and-push-account')
   117          OCR_REPO = 'container-registry.oracle.com'
   118          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
   119  
   120          INSTALL_CONFIG_FILE_KIND_PERSISTENCE = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-with-persistence.yaml"
   121          INSTALL_CONFIG_FILE_KIND_EPHEMERAL = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-no-persistence.yaml"
   122          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-no-persistence.yaml"
   123  
   124          VZ_ENVIRONMENT_NAME = "default"
   125          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
   126          LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test"
   127  
   128          // Location to store Platform Operator manifest
   129          TARGET_OPERATOR_FILE="${WORKSPACE}/acceptance-test-operator.yaml"
   130          // Location to store VZ install file, used to install/re-install VZ
   131          VZ_INSTALL_FILE="${WORKSPACE}/acceptance-test-config.yaml"
   132  
   133          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test
   134          DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test
   135  
   136          OLD_PODS_FILE="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-old-vpo-app-pods.out"
   137          NEW_PODS_FILE="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-new-vpo-app-pods.out"
   138          NEW_PODS_FILE2="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-new-vpo-app-pods-2.out"
   139          APP_NAMEPACES="'todo-list bobs-books hello-helidon springboot sockshop'"
   140  
   141          // used to emit metrics
   142          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   143          TEST_ENV_LABEL = "kind"
   144          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   145          TEST_ENV = "KIND"
   146  
   147          // used to generate Ginkgo test reports
   148          TEST_REPORT = "test-report.xml"
   149          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   150          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   151      }
   152  
   153      stages {
   154          stage('Clean workspace and checkout') {
   155              environment {
   156                  OCI_CLI_AUTH="instance_principal"
   157                  OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   158                  OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   159                  VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   160              }
   161              steps {
   162                  sh """
   163                      echo "${NODE_LABELS}"
   164                  """
   165                  script {
   166                     EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   167                     if (params.GIT_COMMIT_TO_USE == "NONE") {
   168                          echo "Specific GIT commit was not specified, use current head"
   169                          def scmInfo = checkout scm
   170                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   171                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   172                      } else {
   173                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   174                          def scmInfo = checkout([
   175                              $class: 'GitSCM',
   176                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   177                              doGenerateSubmoduleConfigurations: false,
   178                              extensions: [],
   179                              submoduleCfg: [],
   180                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   181                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   182                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   183                          // If the commit we were handed is not what the SCM says we are using, fail
   184                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   185                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   186                              exit 1
   187                          }
   188                      }
   189                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   190                  }
   191                  sh """
   192                      cp -f "${NETRC_FILE}" $HOME/.netrc
   193                      chmod 600 $HOME/.netrc
   194                  """
   195  
   196                  script {
   197                      try {
   198                          sh """
   199                              echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   200                          """
   201                      } catch(error) {
   202                          echo "docker login failed, retrying after sleep"
   203                          retry(4) {
   204                              sleep(30)
   205                              sh """
   206                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   207                              """
   208                          }
   209                      }
   210                  }
   211  
   212                  sh """
   213                      rm -rf ${GO_REPO_PATH}/verrazzano
   214                      mkdir -p ${GO_REPO_PATH}/verrazzano
   215                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   216                      cd ${GO_REPO_PATH}/verrazzano
   217                      git config --global credential.helper "!f() { echo username=\\$DOCKER_CREDS_USR; echo password=\\$DOCKER_CREDS_PSW; }; f"
   218                      git config --global user.name $DOCKER_CREDS_USR
   219                      git config --global user.email "${DOCKER_EMAIL}"
   220                      git checkout -b ${env.BRANCH_NAME}
   221                  """
   222  
   223                  script {
   224                      def props = readProperties file: '.verrazzano-development-version'
   225                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   226                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   227                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   228                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   229                      // update the description with some meaningful info
   230                      setDisplayName()
   231                      currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   232                  }
   233  
   234                  script {
   235                      sh """
   236                          echo "Downloading VZ CLI from object storage"
   237                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   238                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   239                          ${VZ_COMMAND} version
   240                      """
   241                  }
   242              }
   243          }
   244  
   245  		stage ('Uninstall Tests') {
   246  		    environment {
   247                  KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   248                  OCI_CLI_AUTH="instance_principal"
   249                  OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   250                  OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   251                  CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   252  		    }
   253  		    steps {
   254  		        script {
   255  		            int maxIterations = getMaxIterations()
   256                      for (int count = 1; count <= maxIterations; count++) {
   257                          if (count == 1) {
   258                              // Create cluster and install
   259                              runInitialInstall(count)
   260                          } else {
   261                              // Run just the install again
   262                              runInstallOnly("Install Verrazzano", count)
   263                          }
   264                          runVerifyTests("Run Test", count)
   265                          if (params.RUN_APPLICATION_TESTS && count == maxIterations && params.INSTALL_PROFILE != 'managed-cluster') {
   266                              runApplicationTests()
   267                          }
   268                          runUninstall(count)
   269                          if (params.CHAOS_TEST_TYPE == "uninstall.interrupt.uninstall") {
   270                              // Run re-install once and re-verify
   271                              runInstallOnly("Reinstall Verrazzano", count)
   272                              runVerifyTests("Rerun Test", count)
   273                          }
   274                      }
   275  		        }
   276  		    }
   277  		}
   278      }
   279      post {
   280          always {
   281              runnerCleanup()
   282          }
   283          cleanup {
   284              deleteDir()
   285          }
   286      }
   287  }
   288  
   289  def getMaxIterations() {
   290      int maxIterations = 1
   291      if (params.CHAOS_TEST_TYPE == "uninstall.reinstall.loop") {
   292          maxIterations = params.INSTALL_LOOP_COUNT.toInteger()
   293          echo "Max loop count: $maxIterations"
   294      }
   295      return maxIterations
   296  }
   297  
   298  def runInitialInstall(iteration) {
   299      stage("Install Verrazzano #${iteration}") {
   300          env.CLUSTER_SNAPSHOT_DIR="${WORKSPACE}/verrazzano/build/resources-${iteration}/pre-install-resources"
   301          try {
   302              script {
   303                  sh """
   304                      echo "CLUSTER_SNAPSHOT_DIR: ${CLUSTER_SNAPSHOT_DIR}"
   305                      echo "Branch: ${BRANCH_NAME}"
   306                      echo "Operator Image: ${VERRAZZANO_OPERATOR_IMAGE}"
   307                      cd ${GO_REPO_PATH}/verrazzano
   308                      ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_KIND_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
   309                  """
   310              }
   311              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   312                  dumpK8sCluster("install-snapshot-${iteration}")
   313              }
   314          } catch (err) {
   315              dumpK8sCluster("install-snapshot-${iteration}")
   316              throw err
   317          } finally {
   318              dumpPodsAndLogs("post-install-${iteration}")
   319              archiveArtifacts artifacts: "**/logs/**,${env.INSTALL_CONFIG_FILE_KIND}", allowEmptyArchive: true
   320          }
   321      }
   322  }
   323  
   324  def runUninstall(iteration) {
   325      stage("Uninstall Verrazzano #${iteration}") {
   326          try {
   327              vzUninstall(iteration)
   328              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   329                  dumpK8sCluster("uninstall-snapshot-${iteration}")
   330              }
   331          } catch (err) {
   332              dumpK8sCluster("uninstall-failure-snapshot-${iteration}")
   333              throw err
   334          } finally {
   335              dumpPodsAndLogs("post-uninstall-${iteration}")
   336              listNamespacesAndPods('after Verrazzano uninstall')
   337              listHelmReleases('after Verrazzano uninstall')
   338          }
   339      }
   340      stage("Verify Uninstall #${iteration}") {
   341          try {
   342              sh """
   343                  ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources-${iteration}/post-uninstall-resources false
   344                  ${LOOPING_TEST_SCRIPTS_DIR}/verify_uninstall.sh ${WORKSPACE}/verrazzano/build/resources-${iteration}
   345              """
   346              parallel generateVerifyUninstallStages()
   347              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   348                  dumpK8sCluster("verify-uninstall-snapshot-${iteration}")
   349              }
   350          } catch (err) {
   351              dumpK8sCluster("verify-uninstall-snapshot-${iteration}")
   352              throw err
   353          }
   354      }
   355  }
   356  
   357  def runInstallOnly(stagePrefix, iteration) {
   358      def dirPrefix=stagePrefix.replaceAll("\\s","-").toLowerCase()
   359      stage("${stagePrefix} #${iteration}") {
   360          def vpoLogsDir = "${WORKSPACE}/verrazzano-platform-operator/scripts/reinstall/build/logs/${iteration}"
   361          echo "Platform operator logs dir: ${vpoLogsDir}"
   362          try {
   363              sh """
   364                  # sleep for a period to ensure async deletion of Verrazzano components from uninstall above has completed
   365                  #sleep 90
   366                  ${VZ_COMMAND} install --filename ${VZ_INSTALL_FILE} --manifests ${TARGET_OPERATOR_FILE} --timeout 20m
   367              """
   368              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   369                 dumpK8sCluster("install-snapshot-${iteration}")
   370              }
   371          } catch (err) {
   372              dumpK8sCluster("install-failed-snapshot-${iteration}")
   373              sh """
   374                 mkdir -p ${vpoLogsDir}
   375                 ${LOOPING_TEST_SCRIPTS_DIR}/dump_resources.sh > ${vpoLogsDir}/resources.log
   376              """
   377              throw err
   378          } finally {
   379              dumpPodsAndLogs("${dirPrefix}-${iteration}")
   380              listNamespacesAndPods('after reinstalling Verrazzano')
   381              listHelmReleases('after reinstalling Verrazzano')
   382          }
   383      }
   384  }
   385  
   386  def runVerifyTests(stageName, iteration) {
   387      stage("${stageName} #${iteration}") {
   388          try {
   389              parallel generateVerifyInfraStages()
   390              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   391                  dumpK8sCluster("verify-test-snapshot-${iteration}")
   392              }
   393          } catch (err) {
   394              dumpK8sCluster("verify-test-failure-snapshot-${iteration}")
   395              throw err
   396          }
   397      }
   398  }
   399  
   400  def generateVerifyUninstallStages() {
   401      boolean isManagedCluster = params.INSTALL_PROFILE == 'managed-cluster'
   402      uninstallTests = [
   403          "verifycrds": {
   404              runGinkgoWithPassThroughs('uninstall/verifycrds', isManagedCluster)
   405          },
   406          "verifyrancher": {
   407              runGinkgo('uninstall/verifyrancher')
   408          },
   409      ]
   410  
   411      return uninstallTests
   412  }
   413  
   414  def generateVerifyInfraStages() {
   415      infraTests = [
   416          "verify-install": {
   417              runGinkgo('verify-install')
   418          },
   419          "verify-scripts": {
   420              runGinkgo('scripts', '', "${KUBECONFIG}")
   421          },
   422          "verify-infra": {
   423              runGinkgoRandomize('verify-infra')
   424          },
   425      ]
   426      if (params.INSTALL_PROFILE != 'managed-cluster') {
   427          infraTests = infraTests + [
   428              "system component metrics": {
   429                  runGinkgoRandomize('metrics/syscomponents')
   430              },
   431              "console": {
   432                  acceptanceTestsConsole()
   433              },
   434          ]
   435      }
   436      return infraTests
   437  }
   438  
   439  // vzUninstall - Start parallel uninstall and chaos stages
   440  def vzUninstall(iteration) {
   441      stage("Uninstall #${iteration}") {
   442          script {
   443              def vpoLogsDir = "${WORKSPACE}/verrazzano-platform-operator/logs/${iteration}"
   444              echo "Platform operator uninstall logs dir: ${vpoLogsDir}"
   445              sh (script: "mkdir -p ${vpoLogsDir}")
   446              parallel getUninstallStages(vpoLogsDir)
   447          }
   448      }
   449  }
   450  
   451  def getUninstallStages(vpoLogsDir) {
   452      stages = [
   453           "uninstall-verrazzano": {
   454                  uninstallVerrazzano(vpoLogsDir)
   455               },
   456      ]
   457      if (params.CHAOS_TEST_TYPE == "uninstall.interrupt.uninstall") {
   458          stages = stages + [
   459               "uninstall-chaos": {
   460                      kill_vpo_loop(vpoLogsDir)
   461                   },
   462          ]
   463      }
   464      return stages
   465  }
   466  
   467  def uninstallVerrazzano(vpoLogsDir) {
   468      script {
   469          echo "Platform operator uninstall logs dir: ${vpoLogsDir}"
   470  
   471          // Delete VZ in the background
   472          while (true) {
   473              echo "Deleting the Verrazzano resource..."
   474              result = sh(returnStatus: true, script: "${VZ_COMMAND} uninstall -y --timeout=20m")
   475              if (result == 0) {
   476                  break
   477              }
   478              echo "vz uninstall result: ${result}"
   479              sh "sleep 5"
   480          }
   481      }
   482  }
   483  
   484  def kill_vpo_loop(vpoLogsDir) {
   485      script {
   486          echo "Stopping the VPO on loop in 15 to 60 second intervals"
   487  
   488          sh """
   489              # Initially sleep for 30 to 60 seconds
   490              SLEEP_SEC=\$[ \$RANDOM % 30 + 30 ]
   491              echo "Sleeping for \$SLEEP_SEC seconds"
   492              sleep \$SLEEP_SEC
   493  
   494              VZ_STATUS=\$(kubectl get vz my-verrazzano --ignore-not-found -o jsonpath={.status.conditions[-1].type})
   495              RESTARTS=1
   496              while [ ! -z \$VZ_STATUS ] && [ \$VZ_STATUS == "UninstallStarted" ] ; do
   497                  NS_STATUS=\$(kubectl get ns verrazzano-install --ignore-not-found -o jsonpath={.status.phase})
   498                  if [ ! -z \$NS_STATUS ]; then
   499                      mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   500                      kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator >> ${vpoLogsDir}/verrazzano-platform-operator-before-vpo-killed-pod-\${RESTARTS}.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   501                      kubectl -n verrazzano-install delete pod -l app=verrazzano-platform-operator --ignore-not-found
   502                      kubectl -n verrazzano-install rollout status deployments/verrazzano-platform-operator --timeout=600s
   503                      echo "VPO pod successfully restarted"
   504                  fi
   505  
   506                  # Sleep a random interval between 15 and 60 seconds
   507                  SLEEP_SEC=\$[ \$RANDOM % 45 + 15 ]
   508                  echo "Sleeping for \$SLEEP_SEC seconds"
   509                  sleep \$SLEEP_SEC
   510  
   511                  # Update the Verrazzano status and update restart counter
   512                  VZ_STATUS=\$(kubectl get vz my-verrazzano --ignore-not-found -o jsonpath={.status.conditions[-1].type})
   513                  ((RESTARTS++))
   514              done
   515          """
   516      }
   517  }
   518  
   519  def runApplicationTests() {
   520      stage("Application Tests") {
   521          try {
   522              parallel getApplicationStages()
   523              if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   524                  dumpK8sCluster("application-test-failure-snapshot")
   525              }
   526          } catch (err) {
   527              dumpK8sCluster("application-test-failure-snapshot")
   528              throw err
   529          }
   530      }
   531  }
   532  
   533  def getApplicationStages() {
   534      return [
   535         "examples helidon": {
   536             runGinkgo('examples/helidon', 'examples-helidon')
   537         },
   538         "examples socks": {
   539             runGinkgo('examples/socks', 'examples-socks')
   540         },
   541         "examples springboot": {
   542             runGinkgo('examples/springboot', 'examples-springboot')
   543         },
   544         "examples todo": {
   545             runGinkgo('examples/todo', 'examples-todo')
   546         },
   547     ]
   548  }
   549  
   550  // Called in parallel Stage console of Stage Run Acceptance Tests
   551  def acceptanceTestsConsole() {
   552      catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
   553          try {
   554              sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh"
   555          } catch (err) {
   556              saveConsoleScreenShots()
   557              error "${err}"
   558          }
   559      }
   560  }
   561  
   562  def saveConsoleScreenShots() {
   563      sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh"
   564  }
   565  
   566  def runGinkgoRandomize(testSuitePath, dumpDir = '') {
   567      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   568          sh """
   569              if [ ! -z "${dumpDir}" ]; then
   570                  export DUMP_DIRECTORY=${TEST_DUMP_ROOT}/${dumpDir}
   571              fi
   572              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   573              ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   574          """
   575      }
   576  }
   577  
   578  def runGinkgo(testSuitePath, dumpDir = '', kubeconfig = '') {
   579      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   580          sh """
   581              if [ ! -z "${dumpDir}" ]; then
   582                  export DUMP_DIRECTORY=${TEST_DUMP_ROOT}/${dumpDir}
   583              fi
   584              if [ ! -z "${kubeConfig}" ]; then
   585                  export KUBECONFIG="${kubeConfig}"
   586              fi
   587              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   588              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   589          """
   590      }
   591  }
   592  
   593  def runGinkgoWithPassThroughs(testSuitePath, isManagedCluster, dumpDir = '', kubeconfig = '') {
   594      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   595          sh """
   596              if [ ! -z "${dumpDir}" ]; then
   597                  export DUMP_DIRECTORY=${TEST_DUMP_ROOT}/${dumpDir}
   598              fi
   599              if [ ! -z "${kubeConfig}" ]; then
   600                  export KUBECONFIG="${kubeConfig}"
   601              fi
   602              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   603              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --isManagedCluster=${isManagedCluster}
   604          """
   605      }
   606  }
   607  
   608  def runnerCleanup() {
   609      script {
   610          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   611              dumpPodsAndLogs("runnerCleanup")
   612          }
   613      }
   614  
   615      sh """
   616          # Copy the generated test reports to WORKSPACE to archive them
   617          mkdir -p ${TEST_REPORT_DIR}
   618          cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   619          find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   620      """
   621      archiveArtifacts artifacts: "**/coverage.html,${WORKSPACE}/console/**,**/logs/**,**/verrazzano_images.txt,**/build/resources/**,**/*full-cluster*/**,**/bug-report/**,**/${TEST_REPORT}", allowEmptyArchive: true
   622      junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   623  
   624      sh """
   625          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   626          make delete-cluster
   627          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   628            echo "Failures seen during dumping of artifacts, treat post as failed"
   629            exit 1
   630          fi
   631      """
   632  }
   633  
   634  def dumpK8sCluster(dumpDirectory) {
   635      sh """
   636          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${TEST_DUMP_ROOT}/${dumpDirectory}
   637      """
   638  }
   639  
   640  def dumpPodsAndLogs(stage) {
   641      def dumpPath="${WORKSPACE}/verrazzano/logs/${stage}"
   642      dumpVerrazzanoSystemPods(dumpPath)
   643      dumpCattleSystemPods(dumpPath)
   644      dumpNginxIngressControllerLogs(dumpPath)
   645      dumpVerrazzanoPlatformOperatorLogs(dumpPath)
   646      dumpVerrazzanoApplicationOperatorLogs(dumpPath)
   647      dumpVerrazzanoClusterOperatorLogs(dumpPath)
   648      dumpVerrazzanoClusterAgentLogs(dumpPath)
   649      dumpOamKubernetesRuntimeLogs(dumpPath)
   650      dumpVerrazzanoApiLogs(dumpPath)
   651  }
   652  
   653  def dumpVerrazzanoSystemPods(dumpPath) {
   654      createPathIfNecessary(dumpPath)
   655      sh """
   656          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   657          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-system-pods.log"
   658          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   659          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-system-certs.log"
   660          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   661          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-system-osd.log"
   662          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   663          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-system-es-master.log"
   664          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   665      """
   666  }
   667  
   668  def dumpCattleSystemPods(dumpPath) {
   669      createPathIfNecessary(dumpPath)
   670      sh """
   671          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   672          export DIAGNOSTIC_LOG="${dumpPath}/cattle-system-pods.log"
   673          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   674          export DIAGNOSTIC_LOG="${dumpPath}/rancher.log"
   675          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   676      """
   677  }
   678  
   679  def dumpNginxIngressControllerLogs(dumpPath) {
   680      sh """
   681          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   682          export DIAGNOSTIC_LOG="${dumpPath}/nginx-ingress-controller.log"
   683          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   684      """
   685  }
   686  
   687  def dumpVerrazzanoPlatformOperatorLogs(dumpPath) {
   688      createPathIfNecessary(dumpPath)
   689      sh """
   690          ## dump out verrazzano-platform-operator logs
   691          kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${dumpPath}/verrazzano-platform-operator-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   692          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${dumpPath}/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   693          echo "------------------------------------------"
   694      """
   695  }
   696  
   697  def dumpVerrazzanoApplicationOperatorLogs(dumpPath) {
   698      createPathIfNecessary(dumpPath)
   699      sh """
   700          ## dump out verrazzano-application-operator logs
   701          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator --tail -1 > ${dumpPath}/verrazzano-application-operator-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   702          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${dumpPath}/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   703          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   704          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   705          echo "------------------------------------------"
   706      """
   707  }
   708  
   709  def dumpVerrazzanoClusterAgentLogs(dumpPath) {
   710      createPathIfNecessary(dumpPath)
   711      sh """
   712          ## dump out verrazzano-cluster-agent logs
   713          kubectl -n verrazzano-system logs --selector=app=verrazzano-cluster-agent --tail -1 > ${dumpPath}/verrazzano-cluster-agent-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   714          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-cluster-agent > ${dumpPath}/verrazzano-cluster-agent-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   715          echo "verrazzano-cluster-agent logs dumped to verrazzano-cluster-agent-pod.log"
   716          echo "verrazzano-cluster-agent pod description dumped to verrazzano-cluster-agent-pod.out"
   717          echo "------------------------------------------"
   718      """
   719  }
   720  
   721  def dumpVerrazzanoClusterOperatorLogs(dumpPath) {
   722      createPathIfNecessary(dumpPath)
   723      sh """
   724          ## dump out verrazzano-cluster-operator logs
   725          kubectl -n verrazzano-system logs --selector=app=verrazzano-cluster-operator --tail -1 > ${dumpPath}/verrazzano-cluster-operator-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   726          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-cluster-operator > ${dumpPath}/verrazzano-cluster-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   727          echo "verrazzano-cluster-operator logs dumped to verrazzano-cluster-operator-pod.log"
   728          echo "verrazzano-cluster-operator pod description dumped to verrazzano-cluster-operator-pod.out"
   729          echo "------------------------------------------"
   730      """
   731  }
   732  
   733  def dumpOamKubernetesRuntimeLogs(dumpPath) {
   734      createPathIfNecessary(dumpPath)
   735      sh """
   736          ## dump out oam-kubernetes-runtime logs
   737          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime --tail -1 > ${dumpPath}/oam-kubernetes-runtime-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE}
   738          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${dumpPath}/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   739          echo "oam-kubernetes-runtime logs dumped to oam-kubernetes-runtime-pod.log"
   740          echo "oam-kubernetes-runtime pod description dumped to oam-kubernetes-runtime-pod.out"
   741          echo "------------------------------------------"
   742      """
   743  }
   744  
   745  def dumpVerrazzanoApiLogs(dumpPath) {
   746      createPathIfNecessary(dumpPath)
   747      sh """
   748          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   749          export DIAGNOSTIC_LOG="${dumpPath}/verrazzano-authproxy.log"
   750          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   751      """
   752  }
   753  
   754  def createPathIfNecessary(pathName) {
   755      sh """
   756          if [ ! -e ${pathName} ]; then
   757              echo "Creating ${pathName}"
   758              mkdir -p ${pathName}
   759          else
   760              echo "Dump path ${pathName} exists"
   761          fi
   762      """
   763  }
   764  
   765  def setDisplayName() {
   766      echo "Start setDisplayName"
   767      def causes = currentBuild.getBuildCauses()
   768      echo "causes: " + causes.toString()
   769      for (cause in causes) {
   770          def causeString = cause.toString()
   771          echo "current cause: " + causeString
   772          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   773               echo "This job was caused by " + causeString
   774               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   775                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   776               } else if (causeString.contains("verrazzano-flaky-tests")) {
   777                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   778               }
   779           }
   780      }
   781      echo "End setDisplayName"
   782  }
   783  
   784  def getEffectiveDumpOnSuccess() {
   785      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   786      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   787          effectiveValue = true
   788          echo "Forcing dump on success based on global override setting"
   789      }
   790      return effectiveValue
   791  }
   792  
   793  def listHelmReleases(customMessage) {
   794      sh """
   795          echo "Listing the releases across all namespaces ${customMessage}."
   796          helm list -A
   797          echo "-----------------------------------------------------"
   798      """
   799  }
   800  
   801  def listNamespacesAndPods(customMessage) {
   802      sh """
   803          echo "Listing all the namespaces and pods the namespaces ${customMessage}."
   804          kubectl get namespaces
   805          kubectl get pods -A
   806          echo "-----------------------------------------------------"
   807      """
   808  }