github.com/verrazzano/verrazzano@v1.7.1/ci/a-la-carte/Jenkinsfile (about)

     1  // Copyright (c) 2023, 2024, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','')
     8  def availableRegions = [  "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ]
     9  Collections.shuffle(availableRegions)
    10  
    11  pipeline {
    12      options {
    13          timeout(time: 1, unit: 'HOURS')
    14          skipDefaultCheckout true
    15          timestamps ()
    16      }
    17  
    18      agent {
    19         docker {
    20              image "${RUNNER_DOCKER_IMAGE}"
    21              args "${RUNNER_DOCKER_ARGS}"
    22              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    23              registryCredentialsId 'ocir-pull-and-push-account'
    24              label "${agentLabel}"
    25          }
    26      }
    27  
    28      parameters {
    29          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    30                  description: 'Kubernetes Version for KinD Cluster',
    31                  // 1st choice is the default value
    32                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    33          string (name: 'GIT_COMMIT_TO_USE',
    34                  defaultValue: 'NONE',
    35                  description: 'This is the full git commit hash from the source build to be used for all jobs',
    36                  trim: true)
    37          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    38                  defaultValue: 'NONE',
    39                  description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    40                  trim: true)
    41          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    42          string (name: 'CLUSTER_RESOURCE_NAMESPACE',
    43                  defaultValue: 'my-cert-manager',
    44                  description: 'The namespace for cluster-scoped Cert-Manager resources; defaults to where Cert-Manager is installed.',
    45                  trim: true)
    46          choice (description: 'Specifies  DNS type. Values: wildcard, ocidns. Default: wildcard', name: 'DNS_TYPE',
    47              // 1st choice is the default value
    48              choices: ["wildcard","ocidns"] )
    49          choice (description: 'OCI region to create the DNS Zone in', name: 'OKE_CLUSTER_REGION',
    50              // 1st choice is the default value
    51              choices: availableRegions )
    52          choice (description: 'Specifies  DNS scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'DNS_SCOPE',
    53              // 1st choice is the default value
    54              choices: [ "GLOBAL","PRIVATE" ])
    55          choice (description: 'Specifies certificate issuer. Values: default, letsEncrypt. Default: default', name: 'CERTIFICATE_TYPE',
    56              // 1st choice is the default value
    57              choices: [ "default", "letsEncrypt" ])
    58      }
    59  
    60      environment {
    61          GOPATH = '/home/opc/go'
    62          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    63          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    64          DOCKER_EMAIL = credentials('github-packages-email')
    65          DOCKER_REPO = 'ghcr.io'
    66          DOCKER_NAMESPACE = 'verrazzano'
    67          NETRC_FILE = credentials('netrc')
    68          CLUSTER_NAME = 'verrazzano'
    69          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    70          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    71          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    72          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    73          OCR_CREDS = credentials('ocr-pull-and-push-account')
    74          OCR_REPO = 'container-registry.oracle.com'
    75          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    76          INSTALL_CONFIG_FILE_KIND_TESTS = "./tests/e2e/config/scripts/v1beta1/install-verrazzano-kind-none.yaml"
    77          INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/install-verrazzano.yaml"
    78          INSTALL_PROFILE = "none"
    79          VZ_ENVIRONMENT_NAME = "default"
    80          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    81          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    82  
    83          // Environment variables required for OCIDNS
    84          OCI_ZONE_COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid')
    85          OCI_CLI_TENANCY = credentials('oci-tenancy')
    86          OCI_CLI_USER = credentials('oci-user-ocid')
    87          OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint')
    88          OCI_CLI_KEY_FILE = credentials('oci-api-key')
    89          OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}"
    90          OCI_DNS_ZONE_NAME="z${zoneId}.v8o.io"
    91  
    92          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test
    93          DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test
    94  
    95          // Environment variables required to capture cluster snapshot and bug report on test failure
    96          DUMP_KUBECONFIG="${KUBECONFIG}"
    97          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
    98          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
    99          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   100  
   101          // Environment variable for Verrazzano CLI executable
   102          VZ_COMMAND="${GO_REPO_PATH}/vz"
   103  
   104          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   105          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   106  
   107          // used for console artifact capture on failure
   108          JENKINS_READ = credentials('jenkins-auditor')
   109          OCI_CLI_AUTH="instance_principal"
   110          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   111          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   112          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   113  
   114          // used to emit metrics from Ginkgo suites
   115          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   116          TEST_ENV_LABEL = "kind"
   117          TEST_ENV = "KIND"
   118          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   119  
   120          // used to generate Ginkgo test reports
   121          TEST_REPORT = "test-report.xml"
   122          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   123          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   124      }
   125  
   126      stages {
   127          stage('Clean workspace and checkout') {
   128              steps {
   129                  sh """
   130                      echo "${NODE_LABELS}"
   131                  """
   132  
   133                  script {
   134                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   135                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   136                          echo "Specific GIT commit was not specified, use current head"
   137                          def scmInfo = checkout scm
   138                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   139                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   140                      } else {
   141                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   142                          def scmInfo = checkout([
   143                              $class: 'GitSCM',
   144                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   145                              doGenerateSubmoduleConfigurations: false,
   146                              extensions: [],
   147                              submoduleCfg: [],
   148                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   149                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   150                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   151                          // If the commit we were handed is not what the SCM says we are using, fail
   152                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   153                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   154                              exit 1
   155                          }
   156                      }
   157                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   158                  }
   159  
   160                  sh """
   161                      cp -f "${NETRC_FILE}" $HOME/.netrc
   162                      chmod 600 $HOME/.netrc
   163                  """
   164  
   165                  script {
   166                      try {
   167                      sh """
   168                          echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   169                      """
   170                      } catch(error) {
   171                          echo "docker login failed, retrying after sleep"
   172                          retry(4) {
   173                              sleep(30)
   174                              sh """
   175                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   176                              """
   177                          }
   178                      }
   179                  }
   180                  sh """
   181                      rm -rf ${GO_REPO_PATH}/verrazzano
   182                      mkdir -p ${GO_REPO_PATH}/verrazzano
   183                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   184                  """
   185  
   186                  script {
   187                      def props = readProperties file: '.verrazzano-development-version'
   188                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   189                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   190                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   191                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   192                      // update the description with some meaningful info
   193                      setDisplayName()
   194                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   195                  }
   196                  script {
   197                      sh """
   198                          echo "Downloading VZ CLI from object storage"
   199                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   200                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   201                          ${GO_REPO_PATH}/vz version
   202                      """
   203                  }
   204              }
   205          }
   206  
   207          stage('Install KIND cluster with None profile') {
   208              environment {
   209                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   210                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   211              }
   212              steps {
   213                  sh """
   214                      cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND}
   215                      cd ${GO_REPO_PATH}/verrazzano
   216                      ci/scripts/prepare_jenkins_at_environment.sh true nip.io false
   217                  """
   218              }
   219  
   220              post {
   221                  failure {
   222                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   223                      dumpK8sCluster('kind-cluster-with-none-failure-dump')
   224                  }
   225                  always {
   226                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   227                  }
   228              }
   229          }
   230  
   231          stage('Verify none profile installation') {
   232              parallel {
   233                  stage('verify-none-profile') {
   234                      environment {
   235                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-none-profile"
   236                      }
   237                      steps {
   238                          runGinkgoRandomize('verify-none-profile')
   239                      }
   240                      post {
   241                          always {
   242                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   243                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   244                          }
   245                          failure {
   246                              dumpK8sCluster('verify-none-profile-failure-dump')
   247                          }
   248                          success {
   249                              script {
   250                                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   251                                      dumpK8sCluster('verify-none-profile-failure-dump')
   252                                  }
   253                              }
   254                          }
   255                      }
   256                  }
   257              }
   258          }
   259  
   260          stage('Install Prometheus Edge Stack') {
   261              environment {
   262                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   263                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   264              }
   265              steps {
   266                  runGinkgoUpdate('update/a-la-carte', 'prom-edge-stack')
   267              }
   268  
   269              post {
   270                  failure {
   271                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   272                      dumpK8sCluster('edge-stack-installation-failure-dump')
   273                  }
   274                  always {
   275                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   276                  }
   277              }
   278          }
   279  
   280          stage('Verify Prometheus Edge Stack') {
   281              parallel {
   282                  stage('verify-install/promstack') {
   283                      environment {
   284                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   285                      }
   286                      steps {
   287                          runGinkgoRandomize('verify-install/promstack')
   288                      }
   289                      post {
   290                          always {
   291                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   292                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   293                          }
   294                          failure {
   295                              dumpK8sCluster('verify-install-post-edge-stack-installation-failure-dump')
   296                          }
   297                          success {
   298                              script {
   299                                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   300                                      dumpK8sCluster('edge-stack-installation-cluster-snapshot')
   301                                  }
   302                              }
   303                          }
   304                      }
   305                  }
   306              }
   307          }
   308  
   309  
   310          stage('Install app stack with external cert-manager and ingress-nginx') {
   311              environment {
   312                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   313                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   314                  CLUSTER_RESOURCE_NAMESPACE = "${params.CLUSTER_RESOURCE_NAMESPACE}"
   315                  OCI_CLI_AUTH="api_key"
   316              }
   317              steps {
   318                  sh """
   319                      # install third party cert-manager and nginx
   320                      cd ${GO_REPO_PATH}/verrazzano
   321                      ci/scripts/install_third_party_components.sh
   322                  """
   323  
   324                  script {
   325                      if (params.DNS_TYPE == "ocidns") {
   326                          env.DNS_ZONE_OCID = sh(script: "${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o create -c ${OCI_ZONE_COMPARTMENT_ID} -s z${zoneId} -k ${params.DNS_SCOPE}", returnStdout: true)
   327                      }
   328  
   329                      sh """
   330                          ${WORKSPACE}/tests/e2e/config/scripts/create-test-oci-config-secret.sh
   331                      """
   332                  }
   333  
   334                  runGinkgoUpdate('update/a-la-carte', 'app-stack')
   335  
   336                  sh """
   337                      kubectl get vz -A -o yaml
   338                  """
   339  
   340              }
   341  
   342              post {
   343                  failure {
   344                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   345                      dumpK8sCluster('app-stack-installation-failure-dump')
   346                  }
   347                  always {
   348                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   349                  }
   350              }
   351          }
   352  
   353          stage('Verify app stack with external cert-manager and ingress-nginx') {
   354              parallel {
   355                  stage('verify-install/promstack') {
   356                      environment {
   357                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   358                      }
   359                      steps {
   360                          runGinkgoRandomize('verify-install/promstack')
   361                      }
   362                  }
   363                  stage('verify-infra/vmi') {
   364                      environment {
   365                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra/vmi"
   366                      }
   367                      steps {
   368                          runGinkgoRandomize('verify-infra/vmi')
   369                      }
   370                  }
   371              }
   372              post {
   373                  failure {
   374                      dumpK8sCluster('verify-install-post-app-stack-installation-failure-dump')
   375                  }
   376                  always {
   377                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/*cluster-snapshot*/**', allowEmptyArchive: true
   378                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   379                  }
   380                  success {
   381                      script {
   382                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   383                              dumpK8sCluster('edge-stack-installation-cluster-snapshot')
   384                          }
   385                      }
   386                  }
   387              }
   388          }
   389  
   390          stage('Install Istio App stack') {
   391              environment {
   392                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   393                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   394              }
   395              steps {
   396                  runGinkgoUpdate('update/a-la-carte', 'istio-app-stack')
   397              }
   398  
   399              post {
   400                  failure {
   401                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   402                      dumpK8sCluster('install-istio-app-stack-dump')
   403                  }
   404                  always {
   405                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   406                  }
   407              }
   408          }
   409  
   410          stage('Verify Istio App Stack') {
   411              parallel {
   412                  stage('examples helidon') {
   413                      environment {
   414                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon"
   415                      }
   416                      steps {
   417                          runGinkgo('examples/helidon')
   418                      }
   419                  }
   420                  stage('verify-install/promstack') {
   421                      environment {
   422                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   423                      }
   424                      steps {
   425                          runGinkgoRandomize('verify-install/promstack')
   426                      }
   427                  }
   428                  stage('verify-infra/vmi') {
   429                      environment {
   430                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra/vmi"
   431                      }
   432                      steps {
   433                          runGinkgoRandomize('verify-infra/vmi')
   434                      }
   435                  }
   436              }
   437              post {
   438                  failure {
   439                      dumpK8sCluster('verify-istio-app-stack-dump')
   440                  }
   441                  always {
   442                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/*cluster-snapshot*/**', allowEmptyArchive: true
   443                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   444                  }
   445                  success {
   446                      script {
   447                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   448                              dumpK8sCluster('istio-app-stack-installation-cluster-snapshot')
   449                          }
   450                      }
   451                  }
   452              }
   453          }
   454  
   455          stage('Install Cluster Management stack') {
   456              environment {
   457                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   458                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   459              }
   460              steps {
   461                  runGinkgoUpdate('update/a-la-carte', 'cluster-management-stack')
   462              }
   463  
   464              post {
   465                  failure {
   466                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   467                      dumpK8sCluster('install-cluster-management-stack-dump')
   468                  }
   469                  always {
   470                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   471                  }
   472              }
   473          }
   474  
   475          stage('Verify Cluster Management Stack') {
   476              parallel {
   477                  stage('verify-install/promstack') {
   478                      environment {
   479                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   480                      }
   481                      steps {
   482                          runGinkgoRandomize('verify-install/promstack')
   483                      }
   484                  }
   485                  stage('verify-infra/vmi') {
   486                      environment {
   487                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra/vmi"
   488                      }
   489                      steps {
   490                          runGinkgoRandomize('verify-infra/vmi')
   491                      }
   492                  }
   493              }
   494              post {
   495                  failure {
   496                      dumpK8sCluster('verify-cluster-management-stack-dump')
   497                  }
   498                  always {
   499                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/*cluster-snapshot*/**', allowEmptyArchive: true
   500                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   501                  }
   502                  success {
   503                      script {
   504                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   505                              dumpK8sCluster('verify-cluster-management-stack-dump')
   506                          }
   507                      }
   508                  }
   509              }
   510          }
   511  
   512          stage('Uninstall Prometheus Stack') {
   513              environment {
   514                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   515                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   516              }
   517              steps {
   518                  runGinkgoUpdate('update/a-la-carte', 'none-profile')
   519              }
   520  
   521              post {
   522                  failure {
   523                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   524                      dumpK8sCluster('none-installation-failure-dump')
   525                  }
   526                  always {
   527                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   528                  }
   529              }
   530          }
   531  
   532          stage('Verify Prometheus Stack Gone') {
   533              parallel {
   534                  stage('verify-install/promstack') {
   535                      environment {
   536                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   537                      }
   538                      steps {
   539                          runGinkgoRandomize('verify-install/promstack')
   540                      }
   541                  }
   542                  stage('verify-infra/vmi') {
   543                      environment {
   544                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra/vmi"
   545                      }
   546                      steps {
   547                          runGinkgoRandomize('verify-infra/vmi')
   548                      }
   549                  }
   550              }
   551              post {
   552                  failure {
   553                      dumpK8sCluster('verify-install-post-none-installation-failure-dump')
   554                  }
   555                  always {
   556                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/*cluster-snapshot*/**', allowEmptyArchive: true
   557                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   558                  }
   559                  success {
   560                      script {
   561                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   562                              dumpK8sCluster('edge-stack-installation-cluster-snapshot')
   563                          }
   564                      }
   565                  }
   566              }
   567          }
   568  
   569          stage('Uninstall') {
   570              environment {
   571                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   572                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   573              }
   574              steps {
   575                  sh """
   576                      ${GO_REPO_PATH}/vz uninstall -y --timeout 30m
   577                      ${GO_REPO_PATH}/verrazzano/ci/scripts/wait-for-namespace-not-exist.sh verrazzano-install
   578                  """
   579              }
   580  
   581              post {
   582                  failure {
   583                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   584                      dumpK8sCluster('none-uninstall-failure-dump')
   585                  }
   586                  always {
   587                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   588                  }
   589              }
   590          }
   591  
   592          stage('Reinstall None') {
   593              environment {
   594                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   595                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   596              }
   597              steps {
   598                  sh """
   599                      cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND}
   600                     ${GO_REPO_PATH}/vz install --filename ${env.INSTALL_CONFIG_FILE_KIND} --manifests ${WORKSPACE}/acceptance-test-operator.yaml --timeout 5m
   601                  """
   602              }
   603  
   604              post {
   605                  failure {
   606                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   607                      dumpK8sCluster('none-installation-failure-dump')
   608                  }
   609                  always {
   610                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   611                  }
   612              }
   613          }
   614  
   615          stage('Uninstall Again') {
   616              environment {
   617                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   618                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   619              }
   620              steps {
   621                  sh """
   622                      ${GO_REPO_PATH}/vz uninstall -y --timeout 10m
   623                  """
   624              }
   625  
   626              post {
   627                  failure {
   628                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   629                      dumpK8sCluster('none-uninstall-failure-dump')
   630                  }
   631                  always {
   632                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   633                  }
   634              }
   635          }
   636  
   637  
   638      }
   639      post {
   640          always {
   641              script {
   642                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   643                      dumpVerrazzanoSystemPods()
   644                      dumpCattleSystemPods()
   645                      dumpNginxIngressControllerLogs()
   646                      dumpVerrazzanoPlatformOperatorLogs()
   647                      dumpVerrazzanoApplicationOperatorLogs()
   648                      dumpOamKubernetesRuntimeLogs()
   649                      dumpVerrazzanoApiLogs()
   650                  }
   651              }
   652  
   653              sh """
   654                  # Copy the generated test reports to WORKSPACE to archive them
   655                  mkdir -p ${TEST_REPORT_DIR}
   656                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   657                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   658              """
   659              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*cluster-snapshot*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   660              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   661              script {
   662                  if (params.DNS_TYPE == "ocidns") {
   663                       sh """
   664                          ${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o delete -s z${zoneId} -k ${params.DNS_SCOPE} || echo "Failed to delete DNS zone z${zoneId}"
   665                       """
   666                  }
   667              }
   668  
   669              deleteCluster()
   670          }
   671          failure {
   672              sh """
   673                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   674              """
   675              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   676              sh """
   677                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   678                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   679                  rm archive.zip
   680              """
   681          }
   682          cleanup {
   683              deleteDir()
   684          }
   685      }
   686  }
   687  def dumpK8sCluster(dumpDirectory) {
   688      sh """
   689          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   690      """
   691  }
   692  
   693  def dumpVerrazzanoSystemPods() {
   694      sh """
   695          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   696          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   697          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   698          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   699          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   700          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   701          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   702          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   703          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   704      """
   705  }
   706  
   707  def dumpCattleSystemPods() {
   708      sh """
   709          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   710          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   711          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   712          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   713          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   714      """
   715  }
   716  
   717  def dumpNginxIngressControllerLogs() {
   718      sh """
   719          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   720          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   721          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   722      """
   723  }
   724  
   725  def dumpVerrazzanoPlatformOperatorLogs() {
   726      sh """
   727          ## dump out verrazzano-platform-operator logs
   728          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   729          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   730          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   731          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   732          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   733          echo "------------------------------------------"
   734      """
   735  }
   736  
   737  def dumpVerrazzanoApplicationOperatorLogs() {
   738      sh """
   739          ## dump out verrazzano-application-operator logs
   740          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   741          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   742          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   743          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   744          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   745          echo "------------------------------------------"
   746      """
   747  }
   748  
   749  def dumpOamKubernetesRuntimeLogs() {
   750      sh """
   751          ## dump out oam-kubernetes-runtime logs
   752          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   753          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   754          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   755          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   756          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   757          echo "------------------------------------------"
   758      """
   759  }
   760  
   761  def dumpVerrazzanoApiLogs() {
   762      sh """
   763          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   764          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log"
   765          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   766      """
   767  }
   768  
   769  def getEffectiveDumpOnSuccess() {
   770      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   771      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   772          effectiveValue = true
   773          echo "Forcing dump on success based on global override setting"
   774      }
   775      return effectiveValue
   776  }
   777  
   778  def deleteCluster() {
   779      sh """
   780          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   781          make delete-cluster
   782          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   783            echo "Failures seen during dumping of artifacts, treat post as failed"
   784            exit 1
   785          fi
   786      """
   787  }
   788  
   789  def setDisplayName() {
   790      echo "Start setDisplayName"
   791      def causes = currentBuild.getBuildCauses()
   792      echo "causes: " + causes.toString()
   793      for (cause in causes) {
   794          def causeString = cause.toString()
   795          echo "current cause: " + causeString
   796          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   797               echo "This job was caused by " + causeString
   798               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   799                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   800               } else if (causeString.contains("verrazzano-flaky-tests")) {
   801                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   802               }
   803           }
   804      }
   805      echo "End setDisplayName"
   806  }
   807  
   808  def runGinkgoRandomize(testSuitePath, kubeConfig = '') {
   809      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   810          sh """
   811              if [ ! -z "${kubeConfig}" ]; then
   812                  export KUBECONFIG="${kubeConfig}"
   813              fi
   814              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   815              if [ -d "${testSuitePath}" ]; then
   816                  ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file=".*" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   817              fi
   818          """
   819      }
   820  }
   821  
   822  def runGinkgo(testSuitePath) {
   823      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   824          sh """
   825              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   826              ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   827          """
   828      }
   829  }
   830  
   831  def runGinkgoUpdate(testSuitePath, updateTypeVal = '') {
   832      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   833          sh """
   834              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   835              if [ -d "${testSuitePath}" ]; then
   836                  ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file=".*" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --updateType="${updateTypeVal}" --clusterResourceNamespace="${params.CLUSTER_RESOURCE_NAMESPACE}" --dnsType="${params.DNS_TYPE}" --certificateType="${params.CERTIFICATE_TYPE}"
   837              fi
   838          """
   839      }
   840  }