github.com/verrazzano/verrazzano@v1.7.0/ci/dynamic-updates/Jenkinsfile (about)

     1  // Copyright (c) 2021, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          skipDefaultCheckout true
    11          timestamps ()
    12      }
    13  
    14      agent {
    15         docker {
    16              image "${RUNNER_DOCKER_IMAGE}"
    17              args "${RUNNER_DOCKER_ARGS}"
    18              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    19              registryCredentialsId 'ocir-pull-and-push-account'
    20              label "${agentLabel}"
    21          }
    22      }
    23  
    24      parameters {
    25          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    26                  description: 'Kubernetes Version for KinD Cluster',
    27                  // 1st choice is the default value
    28                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    29          string (name: 'GIT_COMMIT_TO_USE',
    30                          defaultValue: 'NONE',
    31                          description: 'This is the full git commit hash from the source build to be used for all jobs',
    32                          trim: true)
    33          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    34                          defaultValue: 'NONE',
    35                          description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    36                          trim: true)
    37          choice (name: 'WILDCARD_DNS_DOMAIN',
    38                  description: 'This is the wildcard DNS domain',
    39                  // 1st choice is the default value
    40                  choices: [ "nip.io", "sslip.io"])
    41          choice (name: 'CRD_API_VERSION',
    42                  description: 'This is the API crd version.',
    43                  // 1st choice is the default value
    44                  choices: [ "v1beta1", "v1alpha1"])
    45          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    46          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    47          booleanParam (description: 'Whether to include the slow tests in the acceptance tests', name: 'RUN_SLOW_TESTS', defaultValue: false)
    48          string (name: 'CONSOLE_REPO_BRANCH',
    49                  defaultValue: '',
    50                  description: 'The branch to check out after cloning the console repository.',
    51                  trim: true)
    52          booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true)
    53          string (name: 'TAGGED_TESTS',
    54                  defaultValue: '',
    55                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    56                  trim: true)
    57          string (name: 'INCLUDED_TESTS',
    58                  defaultValue: '.*',
    59                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    60                  trim: true)
    61          string (name: 'EXCLUDED_TESTS',
    62                  defaultValue: '_excluded_test',
    63                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    64                  trim: true)
    65          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    66          booleanParam (description: 'Whether to Post-Update1 Verify, Infra, and Acceptance Tests', name: 'RUN_POST_UPDATE1', defaultValue: false)
    67          booleanParam (description: 'Whether to Post-Update2 Verify, Infra, and Acceptance Tests', name: 'RUN_POST_UPDATE2', defaultValue: false)
    68          booleanParam (description: 'Whether to run Infra Tests', name: 'RUN_INFRA_TESTS', defaultValue: false)
    69          booleanParam (description: 'Whether to run Availability Status Updates', name: 'RUN_AVAILABILITY_STATUS', defaultValue: false)
    70          booleanParam (description: 'Whether to run API Conversion Update Tests', name: 'RUN_API_CONVERSION', defaultValue: false)
    71          booleanParam (description: 'Whether to run AuthProxy Update Tests', name: 'RUN_AUTHPROXY', defaultValue: false)
    72          booleanParam (description: 'Whether to run EnvironmentName/DNS/CertManager Update Tests', name: 'RUN_CERT_MANAGER', defaultValue: false)
    73          booleanParam (description: 'Whether to run Nginx/Istio Update Tests', name: 'RUN_NGINX_ISTIO', defaultValue: false)
    74          booleanParam (description: 'Whether to run Fluentd Update Tests', name: 'RUN_FLUENTD', defaultValue: false)
    75          booleanParam (description: 'Whether to run Opensearch Update Tests', name: 'RUN_OPENSEARCH', defaultValue: false)
    76          booleanParam (description: 'Whether to run Jaeger Update Tests', name: 'RUN_JAEGER', defaultValue: false)
    77          booleanParam (description: 'Whether to run Fluent Operator Infra Tests', name: 'RUN_FLUENT_OPERATOR', defaultValue: false)
    78          booleanParam (description: 'Whether to run Post-install overrides Tests', name: 'RUN_POST_INSTALL_OVERRIDES', defaultValue: false)
    79          booleanParam (description: 'Whether to skip the VZ install in the prepare environment script', name: 'SKIP_VERRAZZANO_INSTALL_PARAM', defaultValue: false)
    80          booleanParam (description: 'Whether to run with an external cert-manager and NGINX installed', name: 'EXTERNAL_CERT_MANAGER', defaultValue: false)
    81          string (name: 'CLUSTER_RESOURCE_NAMESPACE',
    82                  defaultValue: 'my-cert-manager',
    83                  description: 'The namespace for cluster-scoped Cert-Manager resources; defaults to where Cert-Manager is installed.',
    84                  trim: true)
    85      }
    86  
    87      environment {
    88          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    89          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    90          GOPATH = '/home/opc/go'
    91          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    92          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    93          DOCKER_EMAIL = credentials('github-packages-email')
    94          DOCKER_REPO = 'ghcr.io'
    95          DOCKER_NAMESPACE = 'verrazzano'
    96          NETRC_FILE = credentials('netrc')
    97          CLUSTER_NAME = 'verrazzano'
    98          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    99          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
   100          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
   101          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
   102          OCR_CREDS = credentials('ocr-pull-and-push-account')
   103          OCR_REPO = 'container-registry.oracle.com'
   104          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
   105          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-with-persistence.yaml"
   106          INSTALL_PROFILE = "dev"
   107          VZ_ENVIRONMENT_NAME = "default"
   108          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
   109          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
   110  
   111          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test
   112          DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test
   113  
   114          // Environment variables required to capture cluster snapshot and bug report on test failure
   115          DUMP_KUBECONFIG="${KUBECONFIG}"
   116          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   117          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
   118          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   119  
   120          // Environment variable for Verrazzano CLI executable
   121          VZ_COMMAND="${GO_REPO_PATH}/vz"
   122  
   123          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   124          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   125  
   126          // used for console artifact capture on failure
   127          JENKINS_READ = credentials('jenkins-auditor')
   128          OCI_CLI_AUTH="instance_principal"
   129          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   130          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   131          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   132          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   133  
   134          // used to emit metrics
   135          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   136          TEST_ENV_LABEL = "kind"
   137          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   138          TEST_ENV = "KIND"
   139          SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
   140          SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
   141          SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
   142  
   143          // used to generate Ginkgo test reports
   144          TEST_REPORT = "test-report.xml"
   145          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   146          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   147  
   148          // how many nodes in the KIND cluster
   149          KIND_NODE_COUNT = 3
   150  
   151          SKIP_VERRAZZANO_INSTALL = "${params.SKIP_VERRAZZANO_INSTALL_PARAM}"
   152  
   153          // external cert-manager
   154          INSTALL_EXTERNAL_CERT_MANAGER = "${params.EXTERNAL_CERT_MANAGER}"
   155          CLUSTER_RESOURCE_NAMESPACE = "${params.CLUSTER_RESOURCE_NAMESPACE}"
   156      }
   157  
   158      stages {
   159          stage('Clean workspace and checkout') {
   160              steps {
   161                  sh """
   162                      echo "${NODE_LABELS}"
   163                  """
   164  
   165                  script {
   166                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   167                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   168                          echo "Specific GIT commit was not specified, use current head"
   169                          def scmInfo = checkout scm
   170                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   171                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   172                      } else {
   173                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   174                          def scmInfo = checkout([
   175                              $class: 'GitSCM',
   176                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   177                              doGenerateSubmoduleConfigurations: false,
   178                              extensions: [],
   179                              submoduleCfg: [],
   180                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   181                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   182                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   183                          // If the commit we were handed is not what the SCM says we are using, fail
   184                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   185                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   186                              exit 1
   187                          }
   188                      }
   189                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   190                  }
   191  
   192                  sh """
   193                      cp -f "${NETRC_FILE}" $HOME/.netrc
   194                      chmod 600 $HOME/.netrc
   195                  """
   196  
   197                  script {
   198                      try {
   199                      sh """
   200                          echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   201                      """
   202                      } catch(error) {
   203                          echo "docker login failed, retrying after sleep"
   204                          retry(4) {
   205                              sleep(30)
   206                              sh """
   207                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   208                              """
   209                          }
   210                      }
   211                  }
   212                  sh """
   213                      rm -rf ${GO_REPO_PATH}/verrazzano
   214                      mkdir -p ${GO_REPO_PATH}/verrazzano
   215                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   216                  """
   217  
   218                  script {
   219                      def props = readProperties file: '.verrazzano-development-version'
   220                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   221                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   222                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   223                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   224                      // update the description with some meaningful info
   225                      setDisplayName()
   226                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   227                  }
   228                  script {
   229                      sh """
   230                          echo "Downloading VZ CLI from object storage"
   231                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   232                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   233                          ${GO_REPO_PATH}/vz version
   234                      """
   235                  }
   236              }
   237          }
   238  
   239          stage('Acceptance Tests') {
   240  
   241              stages {
   242                  stage('VZ CLI Custom Resource Validation Tests') {
   243                      environment {
   244                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   245                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   246                      }
   247                      steps {
   248                          script{
   249                              if (params.SKIP_VERRAZZANO_INSTALL_PARAM == true) {
   250                                  runCustomResourceValidation()
   251                              }
   252                          }
   253                      }
   254                  }
   255                  stage('Prepare AT environment') {
   256                      environment {
   257                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   258                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   259                          SKIP_VERRAZZANO_INSTALL = false
   260                      }
   261                      steps {
   262                          script {
   263                              modifyConfigForInstallation()
   264                              if(params.CRD_API_VERSION == "v1alpha1"){
   265                                  scriptForPrepareATEnvironmentV1alpha1()
   266                              }else{
   267                                  scriptForPrepareATEnvironmentV1beta1()
   268                              }
   269                          }
   270                      }
   271                      post {
   272                          success {
   273                              script {
   274                                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   275                                      dumpK8sCluster('dynamic-update-post-install-cluster-snapshot')
   276                                  }
   277                              }
   278                          }
   279                          failure {
   280                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   281                          }
   282                          always {
   283                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true
   284                              // enable debug logging of Verrazzano api istio proxy
   285                              script {
   286                                  if (params.ENABLE_API_ENVOY_LOGGING) {
   287                                      sh '''
   288                                          vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-api --no-headers -o custom-columns=\":metadata.name\")
   289                                          if [ -z "\$vz_api_pod" ]; then
   290                                            echo "Could not find verrazzano-api pod, not enabling debug logging"
   291                                          else
   292                                            kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug
   293                                          fi
   294                                          nginx_ing_pod=\$(kubectl get pod -n verrazzano-ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\")
   295                                          if [ -z "\$nginx_ing_pod" ]; then
   296                                            echo "Could not find nginx ingress controller pod, not enabling debug logging"
   297                                          else
   298                                            kubectl exec \$nginx_ing_pod -c istio-proxy -n verrazzano-ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug
   299                                          fi
   300                                      '''
   301                                  }
   302                              }
   303                          }
   304                      }
   305                  }
   306  
   307                  // Update1 enables disabled components except WLS/COH
   308                  stage('Verrazzano Update1') {
   309                      steps {
   310                          script {
   311                              modifyConfigForUpdate1()
   312                              if(params.CRD_API_VERSION == "v1alpha1"){
   313                                  vzUpdateComponentsV1alpha1()
   314                              }else{
   315                                  vzUpdateComponentsV1beta1()
   316                              }
   317                          }
   318                      }
   319                      post {
   320                          always {
   321                              dumpVerrazzanoPlatformOperatorLogs("verrazzano-update1")
   322                              archiveArtifacts artifacts: "vz-update1.yaml,**/logs/**", allowEmptyArchive: true
   323                          }
   324                      }
   325                  }
   326  
   327                  stage('Post-Update1 Verify Tests') {
   328                      when { expression {params.RUN_POST_UPDATE1} }
   329                      environment {
   330                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update1-verify-infra"
   331                      }
   332                      steps {
   333                          runGinkgoRandomize('verify-install')
   334                      }
   335                      post {
   336                          always {
   337                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   338                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   339                          }
   340                      }
   341                  }
   342  
   343                  stage('Post-Update1 Infra Tests') {
   344                      when { expression {params.RUN_INFRA_TESTS} }
   345                      environment {
   346                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update1-infra-tests"
   347                      }
   348                      steps {
   349                          script {
   350                              parallel generateVerifyInfraStages("${TEST_DUMP_ROOT}/post-update1-infra-tests")
   351                          }
   352                      }
   353                      post {
   354                          always {
   355                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   356                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   357                          }
   358                      }
   359                  }
   360  
   361                  stage('Post-Update1 Acceptance Tests') {
   362                      when { expression {params.RUN_POST_UPDATE1} }
   363                      environment {
   364                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update1-infra-tests"
   365                      }
   366                      steps {
   367                          script {
   368                              // Run all but the WLS tests, and skip the undeploy step to leave the apps running
   369                              parallel generateNonWLSAcceptanceTestStages("${TEST_DUMP_ROOT}/post-update1-acceptance-tests", 'false', 'true')
   370                          }
   371                      }
   372                      post {
   373                          always {
   374                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   375                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   376                          }
   377                      }
   378                  }
   379  
   380                  // Update2 enables COH/WLS operators
   381                  stage('Verrazzano Update2') {
   382                      steps {
   383                          script {
   384                              modifyConfigForUpdate2()
   385                          }
   386                      }
   387                      post {
   388                          always {
   389                              dumpVerrazzanoPlatformOperatorLogs("update-verrazzano")
   390                              archiveArtifacts artifacts: "vz-update2.yaml,**/logs/**", allowEmptyArchive: true
   391                          }
   392                      }
   393                  }
   394  
   395                  stage('WLS/COH Post-Update2 Acceptance Tests') {
   396                      when { expression {params.RUN_POST_UPDATE2} }
   397                      environment {
   398                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-update"
   399                      }
   400                      steps {
   401                          // Run initial WLS/COH acceptance tests, and leave the apps running
   402                          script {
   403                              parallel generate_wls_tests("${TEST_DUMP_ROOT}/wls-coh-update2-tests", 'false', 'true')
   404                          }
   405                      }
   406                      post {
   407                          always {
   408                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   409                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   410                          }
   411                      }
   412                  }
   413  
   414                  stage('Availability Status Updates') {
   415                      when { expression {params.RUN_AVAILABILITY_STATUS} }
   416                      environment {
   417                          TEST_ENV = "KIND"
   418                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/availability-update"
   419                      }
   420                      steps {
   421                          runGinkgo('update/availability')
   422                      }
   423                      post {
   424                          always {
   425                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   426                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   427                          }
   428                      }
   429                  }
   430  
   431                  stage('API Conversion Update Tests') {
   432                      when { expression {params.RUN_API_CONVERSION} }
   433                      environment {
   434                          TEST_ENV = "KIND"
   435                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/apiconversion-update"
   436                      }
   437                      steps {
   438                          runGinkgo('update/apiconversion')
   439                      }
   440                      post {
   441                          always {
   442                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   443                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   444                          }
   445                      }
   446                  }
   447  
   448                  stage('AuthProxy Update Tests') {
   449                      when { expression {params.RUN_AUTHPROXY} }
   450                      environment {
   451                          TEST_ENV = "KIND"
   452                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/authproxy-update"
   453                      }
   454                      steps {
   455                          runGinkgo('update/authproxy')
   456                      }
   457                      post {
   458                          always {
   459                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   460                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   461                          }
   462                      }
   463                  }
   464  
   465                  stage('EnvironmentName/DNS/CertManager Update Tests') {
   466                      when { expression {params.RUN_CERT_MANAGER} }
   467                      environment {
   468                          TEST_ENV = "KIND"
   469                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/env-dns-cm-update"
   470                      }
   471                      steps {
   472                          sh """
   473                              ${WORKSPACE}/ci/scripts/install_third_party_components.sh
   474                          """
   475                          runGinkgo('update/env-dns-cm')
   476                      }
   477                      post {
   478                          always {
   479                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   480                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   481                          }
   482                      }
   483                  }
   484  
   485                  stage('Nginx/Istio Update Tests') {
   486                      when { expression {params.RUN_NGINX_ISTIO} }
   487                      environment {
   488                          TEST_ENV = "KIND"
   489                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/nginx-istio-update"
   490                      }
   491                      steps {
   492                          runGinkgo('update/nginxistio')
   493                      }
   494                      post {
   495                          always {
   496                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   497                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   498                          }
   499                      }
   500                  }
   501  
   502                  stage('Fluentd Update Tests') {
   503                      when { expression {params.RUN_FLUENTD} }
   504                      environment {
   505                          TEST_ENV = "KIND"
   506                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/fluentd-update"
   507                      }
   508                      steps {
   509                          runGinkgo('update/fluentd')
   510                      }
   511                      post {
   512                          always {
   513                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   514                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   515                          }
   516                      }
   517                  }
   518  
   519  
   520                  stage('Opensearch Update Tests') {
   521                      when { expression {params.RUN_OPENSEARCH} }
   522                      environment {
   523                          TEST_ENV = "KIND"
   524                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-update"
   525                      }
   526                      steps {
   527                          runGinkgo('update/opensearch')
   528                      }
   529                      post {
   530                          always {
   531                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   532                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   533                          }
   534                      }
   535                  }
   536  
   537                  stage('Jaeger Update Tests') {
   538                      when { expression {params.RUN_JAEGER} }
   539                      environment {
   540                          TEST_ENV = "KIND"
   541                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jaeger-update"
   542                      }
   543                      steps {
   544                          runGinkgo('update/jaeger')
   545                      }
   546                      post {
   547                          always {
   548                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   549                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   550                          }
   551                      }
   552                  }
   553  
   554                  stage('Fluent Operator Infra Tests') {
   555                      when { expression {params.RUN_FLUENT_OPERATOR} }
   556                      environment {
   557                          TEST_ENV = "KIND"
   558                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/fluent-operator-infra"
   559                      }
   560                      steps {
   561                          runGinkgo('update/fluent-operator/infra')
   562                      }
   563                      post {
   564                          always {
   565                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   566                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   567                          }
   568                      }
   569                  }
   570  
   571                  stage('Post-install overrides Tests') {
   572                      when { expression {params.RUN_POST_INSTALL_OVERRIDES} }
   573                      environment {
   574                          TEST_ENV = "KIND"
   575                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/overrides"
   576                      }
   577                      steps {
   578                          runGinkgo('update/overrides')
   579                      }
   580                      post {
   581                          always {
   582                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   583                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   584                          }
   585                      }
   586                  }
   587  
   588                  stage('Post-update2 Verify Tests') {
   589                      when { expression {params.RUN_POST_UPDATE2} }
   590                      environment {
   591                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update2-verify-install"
   592                      }
   593                      steps {
   594                          runGinkgoRandomize('verify-install')
   595                      }
   596                      post {
   597                          always {
   598                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   599                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   600                          }
   601                      }
   602                  }
   603  
   604                  stage('Post-Update2 Infra Tests') {
   605                      when { expression {params.RUN_INFRA_TESTS} }
   606                      environment {
   607                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update2-infra-tests"
   608                      }
   609                      steps {
   610                          script {
   611                              parallel generateVerifyInfraStages("${TEST_DUMP_ROOT}/post-update2-infra-tests")
   612                          }
   613                      }
   614                      post {
   615                          always {
   616                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   617                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   618                          }
   619                      }
   620                  }
   621  
   622                  stage('Post-update2 Acceptance Tests') {
   623                      when { expression {params.RUN_POST_UPDATE2} }
   624                      environment {
   625                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-update"
   626                      }
   627                      steps {
   628                          // Run all acceptance tests after the sequence of updates including CM and DNS; undeploy apps after
   629                          script {
   630                              parallel generateAllAcceptanceTestStages("${TEST_DUMP_ROOT}/post-cm-dns-acceptance-tests", 'true', 'false')
   631                          }
   632                      }
   633                      post {
   634                          always {
   635                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   636                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   637                          }
   638                      }
   639                  }
   640              }
   641              post {
   642                  failure {
   643                      script {
   644                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   645                              dumpK8sCluster('dynamic-update-failed-cluster-snapshot')
   646                          }
   647                      }
   648                  }
   649                  success {
   650                      script {
   651                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   652                              dumpK8sCluster('dynamic-update-success-cluster-snapshot')
   653                          }
   654                      }
   655                  }
   656              }
   657          }
   658      }
   659  
   660      post {
   661          always {
   662              script {
   663                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   664                      dumpVerrazzanoSystemPods()
   665                      dumpCattleSystemPods()
   666                      dumpNginxIngressControllerLogs()
   667                      dumpVerrazzanoPlatformOperatorLogs("post-run")
   668                      dumpVerrazzanoApplicationOperatorLogs()
   669                      dumpOamKubernetesRuntimeLogs()
   670                      dumpVerrazzanoApiLogs()
   671                  }
   672              }
   673              sh """
   674                  # Copy the generated test reports to WORKSPACE to archive them
   675                  mkdir -p ${TEST_REPORT_DIR}
   676                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   677                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   678              """
   679              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   680              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   681              deleteCluster()
   682          }
   683          failure {
   684              sh """
   685                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   686              """
   687              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   688              sh """
   689                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   690                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   691                  rm archive.zip
   692              """
   693          }
   694          cleanup {
   695              deleteDir()
   696          }
   697      }
   698  }
   699  
   700  def generateNonWLSAcceptanceTestStages(dumpRoot, skipDeploy = 'false', skipUndeploy = 'false') {
   701      return generate_security_tests(dumpRoot) + generate_nonwls_tests(dumpRoot, skipDeploy, skipUndeploy)
   702  }
   703  
   704  def generateAllAcceptanceTestStages(dumpRoot, skipDeploy = 'false', skipUndeploy = 'false') {
   705      return generate_security_tests(dumpRoot) +
   706          generate_nonwls_tests(dumpRoot, skipDeploy, skipUndeploy) +
   707          generate_wls_tests(dumpRoot, skipDeploy, skipUndeploy)
   708  }
   709  
   710  def generateVerifyInfraStages(dumpRoot) {
   711      return [
   712          "verify-scripts": {
   713              runGinkgo('scripts', '', "${KUBECONFIG}")
   714          },
   715          "verify-infra restapi": {
   716              runGinkgoRandomize('verify-infra/restapi', "${dumpRoot}/verify-infra-restapi")
   717          },
   718          "verify-infra oam": {
   719              runGinkgoRandomize('verify-infra/oam', "${dumpRoot}/verify-infra-oam")
   720          },
   721          "system component metrics": {
   722              runGinkgoRandomize('metrics/syscomponents', "${dumpRoot}/system-component-metrics")
   723          },
   724          "verify-infra vmi": {
   725              runGinkgoRandomize('verify-infra/vmi', "${dumpRoot}/verify-infra-vmi")
   726          },
   727      ]
   728  }
   729  
   730  def generate_security_tests(dumpRoot) {
   731      return [
   732          "istio authorization policy": {
   733              runGinkgo('istio/authz', "${dumpRoot}/istio-authz-policy")
   734          },
   735          "security rbac": {
   736              runGinkgo('security/rbac', "${dumpRoot}/sec-role-based-access")
   737          },
   738          "security network policies": {
   739              if (params.CREATE_CLUSTER_USE_CALICO == true) {
   740                  runGinkgo('security/netpol', "${dumpRoot}/netpol")
   741              } else {
   742                  echo "Calico not enabled, skipping network policies tests"
   743              }
   744          },
   745      ]
   746  }
   747  
   748  def generate_nonwls_tests(dumpRoot, skipDeploy = 'false', skipUndeploy = 'false') {
   749      return [
   750          "deployment metrics": {
   751              runGinkgo('metrics/deploymetrics', "${dumpRoot}/k8sdeploy-workload-metrics")
   752          },
   753          "examples logging helidon": {
   754              runGinkgo('logging/helidon', "${dumpRoot}/examples-logging-helidon")
   755          },
   756          "examples springboot": {
   757              runGinkgoAppTest('examples/springboot', "springboot", "${dumpRoot}/examples-spring", skipDeploy, skipUndeploy)
   758          },
   759          "examples helidon": {
   760              runGinkgoAppTest('examples/helidon', "hello-helidon", "${dumpRoot}/examples-helidon", skipDeploy, skipUndeploy)
   761          },
   762          "examples helidon-config": {
   763              runGinkgoAppTest('examples/helidonconfig', "helidon-config", "${dumpRoot}/examples-helidon-config", skipDeploy, skipUndeploy)
   764          },
   765      ]
   766  }
   767  
   768  def generate_wls_tests(dumpRoot, skipDeploy = 'false', skipUndeploy = 'false') {
   769      return [
   770          "weblogic workload": {
   771              runGinkgoAppTest('workloads/weblogic', "hello-wls", "${dumpRoot}/weblogic-workload", skipDeploy, skipUndeploy)
   772          },
   773          "coherence workload": {
   774              runGinkgoAppTest('workloads/coherence', "hello-coherence", "${dumpRoot}/coherence-workload", skipDeploy, skipUndeploy)
   775          },
   776          "console ingress": {
   777              // doesn't work with the deployement hooks
   778              runGinkgo('ingress/console', "wls-console", "${dumpRoot}/console-ingress")
   779          },
   780      ]
   781  }
   782  
   783  def runGinkgoRandomize(testSuitePath, dumpDir = '') {
   784      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   785          sh """
   786              if [ ! -z "${dumpDir}" ]; then
   787                  export DUMP_DIRECTORY=${dumpDir}
   788              fi
   789              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   790              ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   791          """
   792      }
   793  }
   794  
   795  def runGinkgo(testSuitePath, dumpDir = '', kubeconfig = '') {
   796      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   797          sh """
   798              if [ ! -z "${dumpDir}" ]; then
   799                  export DUMP_DIRECTORY=${dumpDir}
   800              fi
   801              if [ ! -z "${kubeConfig}" ]; then
   802                  export KUBECONFIG="${kubeConfig}"
   803              fi
   804              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   805              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   806          """
   807      }
   808  }
   809  
   810  def runGinkgoAppTest(testSuitePath, namespace, dumpDir = '', skipDeploy = 'false', skipUndeploy = 'false') {
   811      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   812          sh """
   813              if [ ! -z "${dumpDir}" ]; then
   814                  export DUMP_DIRECTORY=${dumpDir}
   815              fi
   816              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   817              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace}
   818          """
   819      }
   820  }
   821  
   822  def dumpK8sCluster(dumpDirectory) {
   823      sh """
   824          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   825      """
   826  }
   827  
   828  def dumpVerrazzanoSystemPods() {
   829      sh """
   830          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   831          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   832          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   833          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   834          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   835          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   836          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   837          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   838          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   839      """
   840  }
   841  
   842  def dumpCattleSystemPods() {
   843      sh """
   844          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   845          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   846          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   847          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   848          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   849      """
   850  }
   851  
   852  def dumpNginxIngressControllerLogs() {
   853      sh """
   854          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   855          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   856          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   857      """
   858  }
   859  
   860  def dumpVerrazzanoPlatformOperatorLogs(stage) {
   861      sh """
   862          ## dump out verrazzano-platform-operator logs
   863          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   864          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   865          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   866          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   867          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   868          echo "------------------------------------------"
   869      """
   870  }
   871  
   872  def dumpVerrazzanoApplicationOperatorLogs() {
   873      sh """
   874          ## dump out verrazzano-application-operator logs
   875          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   876          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   877          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   878          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   879          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   880          echo "------------------------------------------"
   881      """
   882  }
   883  
   884  def dumpOamKubernetesRuntimeLogs() {
   885      sh """
   886          ## dump out oam-kubernetes-runtime logs
   887          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   888          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   889          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   890          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   891          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   892          echo "------------------------------------------"
   893      """
   894  }
   895  
   896  def dumpVerrazzanoApiLogs() {
   897      sh """
   898          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   899          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-api.log"
   900          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-api-*" -m "verrazzano api" -c verrazzano-api -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   901      """
   902  }
   903  
   904  def getEffectiveDumpOnSuccess() {
   905      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   906      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   907          effectiveValue = true
   908          echo "Forcing dump on success based on global override setting"
   909      }
   910      return effectiveValue
   911  }
   912  
   913  // Called in parallel Stage console of Stage Run Acceptance Tests
   914  def acceptanceTestsConsole(dumpRoot) {
   915      catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
   916          try {
   917              sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh"
   918          } catch (err) {
   919              saveConsoleScreenShots()
   920              error "${err}"
   921          }
   922      }
   923  }
   924  
   925  def saveConsoleScreenShots() {
   926      sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh"
   927  }
   928  
   929  def deleteCluster() {
   930      sh """
   931          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   932          make delete-cluster
   933          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   934            echo "Failures seen during dumping of artifacts, treat post as failed"
   935            exit 1
   936          fi
   937      """
   938  }
   939  
   940  def setDisplayName() {
   941      echo "Start setDisplayName"
   942      def causes = currentBuild.getBuildCauses()
   943      echo "causes: " + causes.toString()
   944      for (cause in causes) {
   945          def causeString = cause.toString()
   946          echo "current cause: " + causeString
   947          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   948               echo "This job was caused by " + causeString
   949               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   950                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   951               } else if (causeString.contains("verrazzano-flaky-tests")) {
   952                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   953               }
   954           }
   955      }
   956      echo "End setDisplayName"
   957  }
   958  
   959  // Update1 enables disabled components except WLS/COH
   960  def vzUpdateComponentsV1alpha1(){
   961  
   962      sh """
   963          cd ${GO_REPO_PATH}/verrazzano
   964          yq -i eval '.spec.components.ingress.enabled = true' ${WORKSPACE}/vz-update1.yaml
   965          yq -i eval '.spec.components.elasticsearch.enabled = true' ${WORKSPACE}/vz-update1.yaml
   966          yq -i eval '.spec.components.kibana.enabled = true' ${WORKSPACE}/vz-update1.yaml
   967          kubectl apply -f ${WORKSPACE}/vz-update1.yaml
   968          ${TEST_SCRIPTS_DIR}/wait-for-component-ready.sh
   969      """
   970  }
   971  
   972  // Update1 enables disabled components except WLS/COH
   973  def vzUpdateComponentsV1beta1(){
   974  
   975      sh """
   976          cd ${GO_REPO_PATH}/verrazzano
   977          yq -i eval '.spec.components.ingressNGINX.enabled = true' ${WORKSPACE}/vz-update1.yaml
   978          yq -i eval '.spec.components.opensearch.enabled = true' ${WORKSPACE}/vz-update1.yaml
   979          yq -i eval '.spec.components.opensearchDashboards.enabled = true' ${WORKSPACE}/vz-update1.yaml
   980          kubectl apply -f ${WORKSPACE}/vz-update1.yaml
   981          ${TEST_SCRIPTS_DIR}/wait-for-component-ready.sh
   982      """
   983  }
   984  
   985  def scriptForPrepareATEnvironmentV1beta1(){
   986  
   987      sh """
   988          cd ${GO_REPO_PATH}/verrazzano
   989          yq -i eval '.spec.components.ingressNGINX.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   990          yq -i eval '.spec.components.opensearch.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   991          yq -i eval '.spec.components.opensearchDashboards.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   992          ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
   993      """
   994  }
   995  
   996  def scriptForPrepareATEnvironmentV1alpha1(){
   997  
   998      sh """
   999          cd ${GO_REPO_PATH}/verrazzano
  1000          yq -i eval '.spec.components.ingress.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1001          yq -i eval '.spec.components.elasticsearch.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1002          yq -i eval '.spec.components.kibana.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1003          ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
  1004      """
  1005  }
  1006  
  1007  def runCustomResourceValidation(){
  1008      sh """
  1009         cd ${GO_REPO_PATH}/verrazzano
  1010         ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
  1011         oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-vpo.yaml
  1012         ci/scripts/validate_custom_resource.sh ${WORKSPACE}/ci/dynamic-updates/invalidCR.yaml ${WORKSPACE}/downloaded-vpo.yaml
  1013      """
  1014  }
  1015  
  1016  def modifyConfigForInstallation() {
  1017      sh """
  1018          cd ${GO_REPO_PATH}/verrazzano
  1019          yq -i eval '.spec.components.weblogicOperator.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1020          yq -i eval '.spec.components.coherenceOperator.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1021          yq -i eval '.spec.components.kiali.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1022          yq -i eval '.spec.components.rancher.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1023          yq -i eval '.spec.components.istio.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1024          yq -i eval '.spec.components.keycloak.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1025          yq -i eval '.spec.components.applicationOperator.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1026          yq -i eval '.spec.components.oam.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1027          yq -i eval '.spec.components.fluentd.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1028          yq -i eval '.spec.components.prometheus.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1029          yq -i eval '.spec.components.grafana.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1030          yq -i eval '.spec.components.console.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1031          yq -i eval '.spec.components.verrazzano.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1032  
  1033          if [ ${params.EXTERNAL_CERT_MANAGER} == true ]; then
  1034            yq -i eval '.spec.components.certManager.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1035            yq -i eval '.spec.components.clusterIssuer.clusterResourceNamespace = "${CLUSTER_RESOURCE_NAMESPACE}"' ${INSTALL_CONFIG_FILE_KIND}
  1036          else
  1037            yq -i eval '.spec.components.certManager.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1038          fi
  1039      """
  1040  }
  1041  
  1042  def modifyConfigForUpdate1() {
  1043      sh """
  1044          cd ${GO_REPO_PATH}/verrazzano
  1045          cp ${INSTALL_CONFIG_FILE_KIND} ${WORKSPACE}/vz-update1.yaml
  1046          yq -i eval '.spec.components.kiali.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1047          yq -i eval '.spec.components.rancher.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1048          yq -i eval '.spec.components.istio.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1049          yq -i eval '.spec.components.keycloak.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1050          yq -i eval '.spec.components.applicationOperator.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1051          yq -i eval '.spec.components.oam.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1052          yq -i eval '.spec.components.fluentd.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1053          yq -i eval '.spec.components.prometheus.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1054          yq -i eval '.spec.components.grafana.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1055          yq -i eval '.spec.components.console.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1056          yq -i eval '.spec.components.verrazzano.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1057          yq -i eval '.spec.components.argoCD.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1058  
  1059          if [ ${params.EXTERNAL_CERT_MANAGER} == true ]; then
  1060            yq -i eval '.spec.components.certManager.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1061            yq -i eval '.spec.components.clusterIssuer.clusterResourceNamespace = "${CLUSTER_RESOURCE_NAMESPACE}"' ${INSTALL_CONFIG_FILE_KIND}
  1062          else
  1063            yq -i eval '.spec.components.certManager.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1064          fi
  1065      """
  1066  }
  1067  
  1068  def modifyConfigForUpdate2() {
  1069      sh """
  1070          cd ${GO_REPO_PATH}/verrazzano
  1071          cp ${WORKSPACE}/vz-update1.yaml ${WORKSPACE}/vz-update2.yaml
  1072          yq -i eval '.spec.components.weblogicOperator.enabled = true' ${WORKSPACE}/vz-update2.yaml
  1073          yq -i eval '.spec.components.coherenceOperator.enabled = true' ${WORKSPACE}/vz-update2.yaml
  1074          kubectl apply -f ${WORKSPACE}/vz-update2.yaml
  1075          ${TEST_SCRIPTS_DIR}/wait-for-component-ready.sh
  1076      """
  1077  }