github.com/verrazzano/verrazzano@v1.7.0/ci/a-la-carte/Jenkinsfile (about)

     1  // Copyright (c) 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','')
     8  def availableRegions = [  "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-frankfurt-1", "eu-zurich-1", "uk-london-1" ]
     9  Collections.shuffle(availableRegions)
    10  
    11  pipeline {
    12      options {
    13          timeout(time: 1, unit: 'HOURS')
    14          skipDefaultCheckout true
    15          timestamps ()
    16      }
    17  
    18      agent {
    19         docker {
    20              image "${RUNNER_DOCKER_IMAGE}"
    21              args "${RUNNER_DOCKER_ARGS}"
    22              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    23              registryCredentialsId 'ocir-pull-and-push-account'
    24              label "${agentLabel}"
    25          }
    26      }
    27  
    28      parameters {
    29          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    30                  description: 'Kubernetes Version for KinD Cluster',
    31                  // 1st choice is the default value
    32                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    33          string (name: 'GIT_COMMIT_TO_USE',
    34                  defaultValue: 'NONE',
    35                  description: 'This is the full git commit hash from the source build to be used for all jobs',
    36                  trim: true)
    37          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    38                  defaultValue: 'NONE',
    39                  description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    40                  trim: true)
    41          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    42          string (name: 'CLUSTER_RESOURCE_NAMESPACE',
    43                  defaultValue: 'my-cert-manager',
    44                  description: 'The namespace for cluster-scoped Cert-Manager resources; defaults to where Cert-Manager is installed.',
    45                  trim: true)
    46          choice (description: 'Specifies  DNS type. Values: wildcard, ocidns. Default: wildcard', name: 'DNS_TYPE',
    47              // 1st choice is the default value
    48              choices: ["wildcard","ocidns"] )
    49          choice (description: 'OCI region to create the DNS Zone in', name: 'OKE_CLUSTER_REGION',
    50              // 1st choice is the default value
    51              choices: availableRegions )
    52          choice (description: 'Specifies  DNS scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'DNS_SCOPE',
    53              // 1st choice is the default value
    54              choices: [ "GLOBAL","PRIVATE" ])
    55          choice (description: 'Specifies certificate issuer. Values: default, letsEncrypt. Default: default', name: 'CERTIFICATE_TYPE',
    56              // 1st choice is the default value
    57              choices: [ "default", "letsEncrypt" ])
    58      }
    59  
    60      environment {
    61          GOPATH = '/home/opc/go'
    62          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    63          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    64          DOCKER_EMAIL = credentials('github-packages-email')
    65          DOCKER_REPO = 'ghcr.io'
    66          DOCKER_NAMESPACE = 'verrazzano'
    67          NETRC_FILE = credentials('netrc')
    68          CLUSTER_NAME = 'verrazzano'
    69          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    70          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    71          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    72          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    73          OCR_CREDS = credentials('ocr-pull-and-push-account')
    74          OCR_REPO = 'container-registry.oracle.com'
    75          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    76          INSTALL_CONFIG_FILE_KIND_TESTS = "./tests/e2e/config/scripts/v1beta1/install-verrazzano-kind-none.yaml"
    77          INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/install-verrazzano.yaml"
    78          INSTALL_PROFILE = "none"
    79          VZ_ENVIRONMENT_NAME = "default"
    80          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    81          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    82  
    83          // Environment variables required for OCIDNS
    84          OCI_ZONE_COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid')
    85          OCI_CLI_TENANCY = credentials('oci-tenancy')
    86          OCI_CLI_USER = credentials('oci-user-ocid')
    87          OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint')
    88          OCI_CLI_KEY_FILE = credentials('oci-api-key')
    89          OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}"
    90          OCI_DNS_ZONE_NAME="z${zoneId}.v8o.io"
    91  
    92          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test
    93          DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test
    94  
    95          // Environment variables required to capture cluster snapshot and bug report on test failure
    96          DUMP_KUBECONFIG="${KUBECONFIG}"
    97          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
    98          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
    99          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   100  
   101          // Environment variable for Verrazzano CLI executable
   102          VZ_COMMAND="${GO_REPO_PATH}/vz"
   103  
   104          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   105          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   106  
   107          // used for console artifact capture on failure
   108          JENKINS_READ = credentials('jenkins-auditor')
   109          OCI_CLI_AUTH="instance_principal"
   110          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   111          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   112          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   113          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   114  
   115          // used to emit metrics from Ginkgo suites
   116          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   117          TEST_ENV_LABEL = "kind"
   118          TEST_ENV = "KIND"
   119          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   120          SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
   121          SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
   122          SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
   123  
   124          // used to generate Ginkgo test reports
   125          TEST_REPORT = "test-report.xml"
   126          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   127          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   128      }
   129  
   130      stages {
   131          stage('Clean workspace and checkout') {
   132              steps {
   133                  sh """
   134                      echo "${NODE_LABELS}"
   135                  """
   136  
   137                  script {
   138                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   139                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   140                          echo "Specific GIT commit was not specified, use current head"
   141                          def scmInfo = checkout scm
   142                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   143                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   144                      } else {
   145                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   146                          def scmInfo = checkout([
   147                              $class: 'GitSCM',
   148                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   149                              doGenerateSubmoduleConfigurations: false,
   150                              extensions: [],
   151                              submoduleCfg: [],
   152                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   153                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   154                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   155                          // If the commit we were handed is not what the SCM says we are using, fail
   156                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   157                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   158                              exit 1
   159                          }
   160                      }
   161                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   162                  }
   163  
   164                  sh """
   165                      cp -f "${NETRC_FILE}" $HOME/.netrc
   166                      chmod 600 $HOME/.netrc
   167                  """
   168  
   169                  script {
   170                      try {
   171                      sh """
   172                          echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   173                      """
   174                      } catch(error) {
   175                          echo "docker login failed, retrying after sleep"
   176                          retry(4) {
   177                              sleep(30)
   178                              sh """
   179                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   180                              """
   181                          }
   182                      }
   183                  }
   184                  sh """
   185                      rm -rf ${GO_REPO_PATH}/verrazzano
   186                      mkdir -p ${GO_REPO_PATH}/verrazzano
   187                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   188                  """
   189  
   190                  script {
   191                      def props = readProperties file: '.verrazzano-development-version'
   192                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   193                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   194                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   195                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   196                      // update the description with some meaningful info
   197                      setDisplayName()
   198                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   199                  }
   200                  script {
   201                      sh """
   202                          echo "Downloading VZ CLI from object storage"
   203                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   204                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   205                          ${GO_REPO_PATH}/vz version
   206                      """
   207                  }
   208              }
   209          }
   210  
   211          stage('Install KIND cluster with None profile') {
   212              environment {
   213                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   214                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   215              }
   216              steps {
   217                  sh """
   218                      cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND}
   219                      cd ${GO_REPO_PATH}/verrazzano
   220                      ci/scripts/prepare_jenkins_at_environment.sh true nip.io false
   221                  """
   222              }
   223  
   224              post {
   225                  failure {
   226                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   227                      dumpK8sCluster('kind-cluster-with-none-failure-dump')
   228                  }
   229                  always {
   230                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   231                  }
   232              }
   233          }
   234  
   235          stage('Verify none profile installation') {
   236              parallel {
   237                  stage('verify-none-profile') {
   238                      environment {
   239                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-none-profile"
   240                      }
   241                      steps {
   242                          runGinkgoRandomize('verify-none-profile')
   243                      }
   244                      post {
   245                          always {
   246                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   247                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   248                          }
   249                          failure {
   250                              dumpK8sCluster('verify-none-profile-failure-dump')
   251                          }
   252                          success {
   253                              script {
   254                                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   255                                      dumpK8sCluster('verify-none-profile-failure-dump')
   256                                  }
   257                              }
   258                          }
   259                      }
   260                  }
   261              }
   262          }
   263  
   264          stage('Install Prometheus Edge Stack') {
   265              environment {
   266                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   267                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   268              }
   269              steps {
   270                  runGinkgoUpdate('update/a-la-carte', 'prom-edge-stack')
   271              }
   272  
   273              post {
   274                  failure {
   275                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   276                      dumpK8sCluster('edge-stack-installation-failure-dump')
   277                  }
   278                  always {
   279                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   280                  }
   281              }
   282          }
   283  
   284          stage('Verify Prometheus Edge Stack') {
   285              parallel {
   286                  stage('verify-install/promstack') {
   287                      environment {
   288                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   289                      }
   290                      steps {
   291                          runGinkgoRandomize('verify-install/promstack')
   292                      }
   293                      post {
   294                          always {
   295                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   296                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   297                          }
   298                          failure {
   299                              dumpK8sCluster('verify-install-post-edge-stack-installation-failure-dump')
   300                          }
   301                          success {
   302                              script {
   303                                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   304                                      dumpK8sCluster('edge-stack-installation-cluster-snapshot')
   305                                  }
   306                              }
   307                          }
   308                      }
   309                  }
   310              }
   311          }
   312  
   313  
   314          stage('Install app stack with external cert-manager and ingress-nginx') {
   315              environment {
   316                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   317                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   318                  CLUSTER_RESOURCE_NAMESPACE = "${params.CLUSTER_RESOURCE_NAMESPACE}"
   319                  OCI_CLI_AUTH="api_key"
   320              }
   321              steps {
   322                  sh """
   323                      # install third party cert-manager and nginx
   324                      cd ${GO_REPO_PATH}/verrazzano
   325                      ci/scripts/install_third_party_components.sh
   326                  """
   327  
   328                  script {
   329                      if (params.DNS_TYPE == "ocidns") {
   330                          env.DNS_ZONE_OCID = sh(script: "${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o create -c ${OCI_ZONE_COMPARTMENT_ID} -s z${zoneId} -k ${params.DNS_SCOPE}", returnStdout: true)
   331                      }
   332  
   333                      sh """
   334                          ${WORKSPACE}/tests/e2e/config/scripts/create-test-oci-config-secret.sh
   335                      """
   336                  }
   337  
   338                  runGinkgoUpdate('update/a-la-carte', 'app-stack')
   339  
   340                  sh """
   341                      kubectl get vz -A -o yaml
   342                  """
   343  
   344              }
   345  
   346              post {
   347                  failure {
   348                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   349                      dumpK8sCluster('app-stack-installation-failure-dump')
   350                  }
   351                  always {
   352                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   353                  }
   354              }
   355          }
   356  
   357          stage('Verify app stack with external cert-manager and ingress-nginx') {
   358              parallel {
   359                  stage('verify-install/promstack') {
   360                      environment {
   361                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   362                      }
   363                      steps {
   364                          runGinkgoRandomize('verify-install/promstack')
   365                      }
   366                  }
   367                  stage('verify-infra/vmi') {
   368                      environment {
   369                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra/vmi"
   370                      }
   371                      steps {
   372                          runGinkgoRandomize('verify-infra/vmi')
   373                      }
   374                  }
   375              }
   376              post {
   377                  failure {
   378                      dumpK8sCluster('verify-install-post-app-stack-installation-failure-dump')
   379                  }
   380                  always {
   381                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/*cluster-snapshot*/**', allowEmptyArchive: true
   382                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   383                  }
   384                  success {
   385                      script {
   386                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   387                              dumpK8sCluster('edge-stack-installation-cluster-snapshot')
   388                          }
   389                      }
   390                  }
   391              }
   392          }
   393  
   394          stage('Install Istio App stack') {
   395              environment {
   396                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   397                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   398              }
   399              steps {
   400                  runGinkgoUpdate('update/a-la-carte', 'istio-app-stack')
   401              }
   402  
   403              post {
   404                  failure {
   405                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   406                      dumpK8sCluster('install-istio-app-stack-dump')
   407                  }
   408                  always {
   409                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   410                  }
   411              }
   412          }
   413  
   414          stage('Verify Istio App Stack') {
   415              parallel {
   416                  stage('examples helidon') {
   417                      environment {
   418                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon"
   419                      }
   420                      steps {
   421                          runGinkgo('examples/helidon')
   422                      }
   423                  }
   424                  stage('verify-install/promstack') {
   425                      environment {
   426                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   427                      }
   428                      steps {
   429                          runGinkgoRandomize('verify-install/promstack')
   430                      }
   431                  }
   432                  stage('verify-infra/vmi') {
   433                      environment {
   434                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra/vmi"
   435                      }
   436                      steps {
   437                          runGinkgoRandomize('verify-infra/vmi')
   438                      }
   439                  }
   440              }
   441              post {
   442                  failure {
   443                      dumpK8sCluster('verify-istio-app-stack-dump')
   444                  }
   445                  always {
   446                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/*cluster-snapshot*/**', allowEmptyArchive: true
   447                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   448                  }
   449                  success {
   450                      script {
   451                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   452                              dumpK8sCluster('istio-app-stack-installation-cluster-snapshot')
   453                          }
   454                      }
   455                  }
   456              }
   457          }
   458  
   459          stage('Install Cluster Management stack') {
   460              environment {
   461                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   462                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   463              }
   464              steps {
   465                  runGinkgoUpdate('update/a-la-carte', 'cluster-management-stack')
   466              }
   467  
   468              post {
   469                  failure {
   470                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   471                      dumpK8sCluster('install-cluster-management-stack-dump')
   472                  }
   473                  always {
   474                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   475                  }
   476              }
   477          }
   478  
   479          stage('Verify Cluster Management Stack') {
   480              parallel {
   481                  stage('verify-install/promstack') {
   482                      environment {
   483                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   484                      }
   485                      steps {
   486                          runGinkgoRandomize('verify-install/promstack')
   487                      }
   488                  }
   489                  stage('verify-infra/vmi') {
   490                      environment {
   491                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra/vmi"
   492                      }
   493                      steps {
   494                          runGinkgoRandomize('verify-infra/vmi')
   495                      }
   496                  }
   497              }
   498              post {
   499                  failure {
   500                      dumpK8sCluster('verify-cluster-management-stack-dump')
   501                  }
   502                  always {
   503                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/*cluster-snapshot*/**', allowEmptyArchive: true
   504                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   505                  }
   506                  success {
   507                      script {
   508                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   509                              dumpK8sCluster('verify-cluster-management-stack-dump')
   510                          }
   511                      }
   512                  }
   513              }
   514          }
   515  
   516          stage('Uninstall Prometheus Stack') {
   517              environment {
   518                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   519                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   520              }
   521              steps {
   522                  runGinkgoUpdate('update/a-la-carte', 'none-profile')
   523              }
   524  
   525              post {
   526                  failure {
   527                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   528                      dumpK8sCluster('none-installation-failure-dump')
   529                  }
   530                  always {
   531                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   532                  }
   533              }
   534          }
   535  
   536          stage('Verify Prometheus Stack Gone') {
   537              parallel {
   538                  stage('verify-install/promstack') {
   539                      environment {
   540                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install/promstack"
   541                      }
   542                      steps {
   543                          runGinkgoRandomize('verify-install/promstack')
   544                      }
   545                  }
   546                  stage('verify-infra/vmi') {
   547                      environment {
   548                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-infra/vmi"
   549                      }
   550                      steps {
   551                          runGinkgoRandomize('verify-infra/vmi')
   552                      }
   553                  }
   554              }
   555              post {
   556                  failure {
   557                      dumpK8sCluster('verify-install-post-none-installation-failure-dump')
   558                  }
   559                  always {
   560                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**,**/*cluster-snapshot*/**', allowEmptyArchive: true
   561                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   562                  }
   563                  success {
   564                      script {
   565                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   566                              dumpK8sCluster('edge-stack-installation-cluster-snapshot')
   567                          }
   568                      }
   569                  }
   570              }
   571          }
   572  
   573          stage('Uninstall') {
   574              environment {
   575                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   576                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   577              }
   578              steps {
   579                  sh """
   580                      ${GO_REPO_PATH}/vz uninstall -y --timeout 30m
   581                      ${GO_REPO_PATH}/verrazzano/ci/scripts/wait-for-namespace-not-exist.sh verrazzano-install
   582                  """
   583              }
   584  
   585              post {
   586                  failure {
   587                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   588                      dumpK8sCluster('none-uninstall-failure-dump')
   589                  }
   590                  always {
   591                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   592                  }
   593              }
   594          }
   595  
   596          stage('Reinstall None') {
   597              environment {
   598                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   599                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   600              }
   601              steps {
   602                  sh """
   603                      cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND}
   604                     ${GO_REPO_PATH}/vz install --filename ${env.INSTALL_CONFIG_FILE_KIND} --manifests ${WORKSPACE}/acceptance-test-operator.yaml --timeout 5m
   605                  """
   606              }
   607  
   608              post {
   609                  failure {
   610                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   611                      dumpK8sCluster('none-installation-failure-dump')
   612                  }
   613                  always {
   614                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   615                  }
   616              }
   617          }
   618  
   619          stage('Uninstall Again') {
   620              environment {
   621                  KIND_KUBERNETES_CLUSTER_VERSION = "${params.KUBERNETES_CLUSTER_VERSION}"
   622                  OCI_OS_LOCATION = "ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   623              }
   624              steps {
   625                  sh """
   626                      ${GO_REPO_PATH}/vz uninstall -y --timeout 10m
   627                  """
   628              }
   629  
   630              post {
   631                  failure {
   632                      archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   633                      dumpK8sCluster('none-uninstall-failure-dump')
   634                  }
   635                  always {
   636                      archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true
   637                  }
   638              }
   639          }
   640  
   641  
   642      }
   643      post {
   644          always {
   645              script {
   646                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   647                      dumpVerrazzanoSystemPods()
   648                      dumpCattleSystemPods()
   649                      dumpNginxIngressControllerLogs()
   650                      dumpVerrazzanoPlatformOperatorLogs()
   651                      dumpVerrazzanoApplicationOperatorLogs()
   652                      dumpOamKubernetesRuntimeLogs()
   653                      dumpVerrazzanoApiLogs()
   654                  }
   655              }
   656  
   657              sh """
   658                  # Copy the generated test reports to WORKSPACE to archive them
   659                  mkdir -p ${TEST_REPORT_DIR}
   660                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   661                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   662              """
   663              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*cluster-snapshot*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   664              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   665              script {
   666                  if (params.DNS_TYPE == "ocidns") {
   667                       sh """
   668                          ${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o delete -s z${zoneId} -k ${params.DNS_SCOPE} || echo "Failed to delete DNS zone z${zoneId}"
   669                       """
   670                  }
   671              }
   672  
   673              deleteCluster()
   674          }
   675          failure {
   676              sh """
   677                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   678              """
   679              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   680              sh """
   681                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   682                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   683                  rm archive.zip
   684              """
   685          }
   686          cleanup {
   687              deleteDir()
   688          }
   689      }
   690  }
   691  def dumpK8sCluster(dumpDirectory) {
   692      sh """
   693          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   694      """
   695  }
   696  
   697  def dumpVerrazzanoSystemPods() {
   698      sh """
   699          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   700          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   701          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   702          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   703          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   704          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   705          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   706          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   707          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   708      """
   709  }
   710  
   711  def dumpCattleSystemPods() {
   712      sh """
   713          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   714          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   715          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   716          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   717          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   718      """
   719  }
   720  
   721  def dumpNginxIngressControllerLogs() {
   722      sh """
   723          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   724          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   725          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   726      """
   727  }
   728  
   729  def dumpVerrazzanoPlatformOperatorLogs() {
   730      sh """
   731          ## dump out verrazzano-platform-operator logs
   732          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   733          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   734          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   735          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   736          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   737          echo "------------------------------------------"
   738      """
   739  }
   740  
   741  def dumpVerrazzanoApplicationOperatorLogs() {
   742      sh """
   743          ## dump out verrazzano-application-operator logs
   744          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   745          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   746          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   747          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   748          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   749          echo "------------------------------------------"
   750      """
   751  }
   752  
   753  def dumpOamKubernetesRuntimeLogs() {
   754      sh """
   755          ## dump out oam-kubernetes-runtime logs
   756          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   757          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   758          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   759          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   760          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   761          echo "------------------------------------------"
   762      """
   763  }
   764  
   765  def dumpVerrazzanoApiLogs() {
   766      sh """
   767          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   768          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log"
   769          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   770      """
   771  }
   772  
   773  def getEffectiveDumpOnSuccess() {
   774      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   775      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   776          effectiveValue = true
   777          echo "Forcing dump on success based on global override setting"
   778      }
   779      return effectiveValue
   780  }
   781  
   782  def deleteCluster() {
   783      sh """
   784          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   785          make delete-cluster
   786          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   787            echo "Failures seen during dumping of artifacts, treat post as failed"
   788            exit 1
   789          fi
   790      """
   791  }
   792  
   793  def setDisplayName() {
   794      echo "Start setDisplayName"
   795      def causes = currentBuild.getBuildCauses()
   796      echo "causes: " + causes.toString()
   797      for (cause in causes) {
   798          def causeString = cause.toString()
   799          echo "current cause: " + causeString
   800          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   801               echo "This job was caused by " + causeString
   802               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   803                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   804               } else if (causeString.contains("verrazzano-flaky-tests")) {
   805                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   806               }
   807           }
   808      }
   809      echo "End setDisplayName"
   810  }
   811  
   812  def runGinkgoRandomize(testSuitePath, kubeConfig = '') {
   813      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   814          sh """
   815              if [ ! -z "${kubeConfig}" ]; then
   816                  export KUBECONFIG="${kubeConfig}"
   817              fi
   818              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   819              if [ -d "${testSuitePath}" ]; then
   820                  ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file=".*" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   821              fi
   822          """
   823      }
   824  }
   825  
   826  def runGinkgo(testSuitePath) {
   827      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   828          sh """
   829              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   830              ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   831          """
   832      }
   833  }
   834  
   835  def runGinkgoUpdate(testSuitePath, updateTypeVal = '') {
   836      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   837          sh """
   838              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   839              if [ -d "${testSuitePath}" ]; then
   840                  ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file=".*" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --updateType="${updateTypeVal}" --clusterResourceNamespace="${params.CLUSTER_RESOURCE_NAMESPACE}" --dnsType="${params.DNS_TYPE}" --certificateType="${params.CERTIFICATE_TYPE}"
   841              fi
   842          """
   843      }
   844  }