github.com/verrazzano/verrazzano@v1.7.1/ci/dynamic-updates/Jenkinsfile (about)

     1  // Copyright (c) 2021, 2024, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
     6  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
     7  
     8  pipeline {
     9      options {
    10          skipDefaultCheckout true
    11          timestamps ()
    12      }
    13  
    14      agent {
    15         docker {
    16              image "${RUNNER_DOCKER_IMAGE}"
    17              args "${RUNNER_DOCKER_ARGS}"
    18              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    19              registryCredentialsId 'ocir-pull-and-push-account'
    20              label "${agentLabel}"
    21          }
    22      }
    23  
    24      parameters {
    25          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    26                  description: 'Kubernetes Version for KinD Cluster',
    27                  // 1st choice is the default value
    28                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    29          string (name: 'GIT_COMMIT_TO_USE',
    30                          defaultValue: 'NONE',
    31                          description: 'This is the full git commit hash from the source build to be used for all jobs',
    32                          trim: true)
    33          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    34                          defaultValue: 'NONE',
    35                          description: 'Verrazzano platform operator image name (in ghcr.io repo).  If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator',
    36                          trim: true)
    37          choice (name: 'WILDCARD_DNS_DOMAIN',
    38                  description: 'This is the wildcard DNS domain',
    39                  // 1st choice is the default value
    40                  choices: [ "nip.io", "sslip.io"])
    41          choice (name: 'CRD_API_VERSION',
    42                  description: 'This is the API crd version.',
    43                  // 1st choice is the default value
    44                  choices: [ "v1beta1", "v1alpha1"])
    45          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    46          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    47          booleanParam (description: 'Whether to include the slow tests in the acceptance tests', name: 'RUN_SLOW_TESTS', defaultValue: false)
    48          string (name: 'CONSOLE_REPO_BRANCH',
    49                  defaultValue: '',
    50                  description: 'The branch to check out after cloning the console repository.',
    51                  trim: true)
    52          booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true)
    53          string (name: 'TAGGED_TESTS',
    54                  defaultValue: '',
    55                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    56                  trim: true)
    57          string (name: 'INCLUDED_TESTS',
    58                  defaultValue: '.*',
    59                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    60                  trim: true)
    61          string (name: 'EXCLUDED_TESTS',
    62                  defaultValue: '_excluded_test',
    63                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    64                  trim: true)
    65          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    66          booleanParam (description: 'Whether to Post-Update1 Verify, Infra, and Acceptance Tests', name: 'RUN_POST_UPDATE1', defaultValue: false)
    67          booleanParam (description: 'Whether to Post-Update2 Verify, Infra, and Acceptance Tests', name: 'RUN_POST_UPDATE2', defaultValue: false)
    68          booleanParam (description: 'Whether to run Infra Tests', name: 'RUN_INFRA_TESTS', defaultValue: false)
    69          booleanParam (description: 'Whether to run Availability Status Updates', name: 'RUN_AVAILABILITY_STATUS', defaultValue: false)
    70          booleanParam (description: 'Whether to run API Conversion Update Tests', name: 'RUN_API_CONVERSION', defaultValue: false)
    71          booleanParam (description: 'Whether to run AuthProxy Update Tests', name: 'RUN_AUTHPROXY', defaultValue: false)
    72          booleanParam (description: 'Whether to run EnvironmentName/DNS/CertManager Update Tests', name: 'RUN_CERT_MANAGER', defaultValue: false)
    73          booleanParam (description: 'Whether to run Nginx/Istio Update Tests', name: 'RUN_NGINX_ISTIO', defaultValue: false)
    74          booleanParam (description: 'Whether to run Fluentd Update Tests', name: 'RUN_FLUENTD', defaultValue: false)
    75          booleanParam (description: 'Whether to run Opensearch Update Tests', name: 'RUN_OPENSEARCH', defaultValue: false)
    76          booleanParam (description: 'Whether to run Jaeger Update Tests', name: 'RUN_JAEGER', defaultValue: false)
    77          booleanParam (description: 'Whether to run Fluent Operator Infra Tests', name: 'RUN_FLUENT_OPERATOR', defaultValue: false)
    78          booleanParam (description: 'Whether to run Post-install overrides Tests', name: 'RUN_POST_INSTALL_OVERRIDES', defaultValue: false)
    79          booleanParam (description: 'Whether to skip the VZ install in the prepare environment script', name: 'SKIP_VERRAZZANO_INSTALL_PARAM', defaultValue: false)
    80          booleanParam (description: 'Whether to run with an external cert-manager and NGINX installed', name: 'EXTERNAL_CERT_MANAGER', defaultValue: false)
    81          string (name: 'CLUSTER_RESOURCE_NAMESPACE',
    82                  defaultValue: 'my-cert-manager',
    83                  description: 'The namespace for cluster-scoped Cert-Manager resources; defaults to where Cert-Manager is installed.',
    84                  trim: true)
    85      }
    86  
    87      environment {
    88          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    89          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    90          GOPATH = '/home/opc/go'
    91          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    92          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    93          DOCKER_EMAIL = credentials('github-packages-email')
    94          DOCKER_REPO = 'ghcr.io'
    95          DOCKER_NAMESPACE = 'verrazzano'
    96          NETRC_FILE = credentials('netrc')
    97          CLUSTER_NAME = 'verrazzano'
    98          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    99          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
   100          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
   101          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
   102          OCR_CREDS = credentials('ocr-pull-and-push-account')
   103          OCR_REPO = 'container-registry.oracle.com'
   104          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
   105          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-with-persistence.yaml"
   106          INSTALL_PROFILE = "dev"
   107          VZ_ENVIRONMENT_NAME = "default"
   108          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
   109          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
   110  
   111          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test
   112          DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test
   113  
   114          // Environment variables required to capture cluster snapshot and bug report on test failure
   115          DUMP_KUBECONFIG="${KUBECONFIG}"
   116          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   117          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots"
   118          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   119  
   120          // Environment variable for Verrazzano CLI executable
   121          VZ_COMMAND="${GO_REPO_PATH}/vz"
   122  
   123          VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs"
   124          VERRAZZANO_INSTALL_LOG="verrazzano-install.log"
   125  
   126          // used for console artifact capture on failure
   127          JENKINS_READ = credentials('jenkins-auditor')
   128          OCI_CLI_AUTH="instance_principal"
   129          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   130          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   131          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   132  
   133          // used to emit metrics
   134          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   135          TEST_ENV_LABEL = "kind"
   136          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   137          TEST_ENV = "KIND"
   138  
   139          // used to generate Ginkgo test reports
   140          TEST_REPORT = "test-report.xml"
   141          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   142          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   143  
   144          // how many nodes in the KIND cluster
   145          KIND_NODE_COUNT = 3
   146  
   147          SKIP_VERRAZZANO_INSTALL = "${params.SKIP_VERRAZZANO_INSTALL_PARAM}"
   148  
   149          // external cert-manager
   150          INSTALL_EXTERNAL_CERT_MANAGER = "${params.EXTERNAL_CERT_MANAGER}"
   151          CLUSTER_RESOURCE_NAMESPACE = "${params.CLUSTER_RESOURCE_NAMESPACE}"
   152      }
   153  
   154      stages {
   155          stage('Clean workspace and checkout') {
   156              steps {
   157                  sh """
   158                      echo "${NODE_LABELS}"
   159                  """
   160  
   161                  script {
   162                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   163                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   164                          echo "Specific GIT commit was not specified, use current head"
   165                          def scmInfo = checkout scm
   166                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   167                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   168                      } else {
   169                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   170                          def scmInfo = checkout([
   171                              $class: 'GitSCM',
   172                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   173                              doGenerateSubmoduleConfigurations: false,
   174                              extensions: [],
   175                              submoduleCfg: [],
   176                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   177                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   178                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   179                          // If the commit we were handed is not what the SCM says we are using, fail
   180                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   181                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   182                              exit 1
   183                          }
   184                      }
   185                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   186                  }
   187  
   188                  sh """
   189                      cp -f "${NETRC_FILE}" $HOME/.netrc
   190                      chmod 600 $HOME/.netrc
   191                  """
   192  
   193                  script {
   194                      try {
   195                      sh """
   196                          echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   197                      """
   198                      } catch(error) {
   199                          echo "docker login failed, retrying after sleep"
   200                          retry(4) {
   201                              sleep(30)
   202                              sh """
   203                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   204                              """
   205                          }
   206                      }
   207                  }
   208                  sh """
   209                      rm -rf ${GO_REPO_PATH}/verrazzano
   210                      mkdir -p ${GO_REPO_PATH}/verrazzano
   211                      tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   212                  """
   213  
   214                  script {
   215                      def props = readProperties file: '.verrazzano-development-version'
   216                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   217                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   218                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   219                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   220                      // update the description with some meaningful info
   221                      setDisplayName()
   222                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   223                  }
   224                  script {
   225                      sh """
   226                          echo "Downloading VZ CLI from object storage"
   227                          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   228                          tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   229                          ${GO_REPO_PATH}/vz version
   230                      """
   231                  }
   232              }
   233          }
   234  
   235          stage('Acceptance Tests') {
   236  
   237              stages {
   238                  stage('VZ CLI Custom Resource Validation Tests') {
   239                      environment {
   240                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   241                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   242                      }
   243                      steps {
   244                          script{
   245                              if (params.SKIP_VERRAZZANO_INSTALL_PARAM == true) {
   246                                  runCustomResourceValidation()
   247                              }
   248                          }
   249                      }
   250                  }
   251                  stage('Prepare AT environment') {
   252                      environment {
   253                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   254                          OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
   255                          SKIP_VERRAZZANO_INSTALL = false
   256                      }
   257                      steps {
   258                          script {
   259                              modifyConfigForInstallation()
   260                              if(params.CRD_API_VERSION == "v1alpha1"){
   261                                  scriptForPrepareATEnvironmentV1alpha1()
   262                              }else{
   263                                  scriptForPrepareATEnvironmentV1beta1()
   264                              }
   265                          }
   266                      }
   267                      post {
   268                          success {
   269                              script {
   270                                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   271                                      dumpK8sCluster('dynamic-update-post-install-cluster-snapshot')
   272                                  }
   273                              }
   274                          }
   275                          failure {
   276                              archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true
   277                          }
   278                          always {
   279                              archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true
   280                              // enable debug logging of Verrazzano api istio proxy
   281                              script {
   282                                  if (params.ENABLE_API_ENVOY_LOGGING) {
   283                                      sh '''
   284                                          vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-api --no-headers -o custom-columns=\":metadata.name\")
   285                                          if [ -z "\$vz_api_pod" ]; then
   286                                            echo "Could not find verrazzano-api pod, not enabling debug logging"
   287                                          else
   288                                            kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug
   289                                          fi
   290                                          nginx_ing_pod=\$(kubectl get pod -n verrazzano-ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\")
   291                                          if [ -z "\$nginx_ing_pod" ]; then
   292                                            echo "Could not find nginx ingress controller pod, not enabling debug logging"
   293                                          else
   294                                            kubectl exec \$nginx_ing_pod -c istio-proxy -n verrazzano-ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug
   295                                          fi
   296                                      '''
   297                                  }
   298                              }
   299                          }
   300                      }
   301                  }
   302  
   303                  // Update1 enables disabled components except WLS/COH
   304                  stage('Verrazzano Update1') {
   305                      steps {
   306                          script {
   307                              modifyConfigForUpdate1()
   308                              if(params.CRD_API_VERSION == "v1alpha1"){
   309                                  vzUpdateComponentsV1alpha1()
   310                              }else{
   311                                  vzUpdateComponentsV1beta1()
   312                              }
   313                          }
   314                      }
   315                      post {
   316                          always {
   317                              dumpVerrazzanoPlatformOperatorLogs("verrazzano-update1")
   318                              archiveArtifacts artifacts: "vz-update1.yaml,**/logs/**", allowEmptyArchive: true
   319                          }
   320                      }
   321                  }
   322  
   323                  stage('Post-Update1 Verify Tests') {
   324                      when { expression {params.RUN_POST_UPDATE1} }
   325                      environment {
   326                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update1-verify-infra"
   327                      }
   328                      steps {
   329                          runGinkgoRandomize('verify-install')
   330                      }
   331                      post {
   332                          always {
   333                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   334                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   335                          }
   336                      }
   337                  }
   338  
   339                  stage('Post-Update1 Infra Tests') {
   340                      when { expression {params.RUN_INFRA_TESTS} }
   341                      environment {
   342                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update1-infra-tests"
   343                      }
   344                      steps {
   345                          script {
   346                              parallel generateVerifyInfraStages("${TEST_DUMP_ROOT}/post-update1-infra-tests")
   347                          }
   348                      }
   349                      post {
   350                          always {
   351                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   352                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   353                          }
   354                      }
   355                  }
   356  
   357                  stage('Post-Update1 Acceptance Tests') {
   358                      when { expression {params.RUN_POST_UPDATE1} }
   359                      environment {
   360                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update1-infra-tests"
   361                      }
   362                      steps {
   363                          script {
   364                              // Run all but the WLS tests, and skip the undeploy step to leave the apps running
   365                              parallel generateNonWLSAcceptanceTestStages("${TEST_DUMP_ROOT}/post-update1-acceptance-tests", 'false', 'true')
   366                          }
   367                      }
   368                      post {
   369                          always {
   370                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   371                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   372                          }
   373                      }
   374                  }
   375  
   376                  // Update2 enables COH/WLS operators
   377                  stage('Verrazzano Update2') {
   378                      steps {
   379                          script {
   380                              modifyConfigForUpdate2()
   381                          }
   382                      }
   383                      post {
   384                          always {
   385                              dumpVerrazzanoPlatformOperatorLogs("update-verrazzano")
   386                              archiveArtifacts artifacts: "vz-update2.yaml,**/logs/**", allowEmptyArchive: true
   387                          }
   388                      }
   389                  }
   390  
   391                  stage('WLS/COH Post-Update2 Acceptance Tests') {
   392                      when { expression {params.RUN_POST_UPDATE2} }
   393                      environment {
   394                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-update"
   395                      }
   396                      steps {
   397                          // Run initial WLS/COH acceptance tests, and leave the apps running
   398                          script {
   399                              parallel generate_wls_tests("${TEST_DUMP_ROOT}/wls-coh-update2-tests", 'false', 'true')
   400                          }
   401                      }
   402                      post {
   403                          always {
   404                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   405                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   406                          }
   407                      }
   408                  }
   409  
   410                  stage('Availability Status Updates') {
   411                      when { expression {params.RUN_AVAILABILITY_STATUS} }
   412                      environment {
   413                          TEST_ENV = "KIND"
   414                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/availability-update"
   415                      }
   416                      steps {
   417                          runGinkgo('update/availability')
   418                      }
   419                      post {
   420                          always {
   421                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   422                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   423                          }
   424                      }
   425                  }
   426  
   427                  stage('API Conversion Update Tests') {
   428                      when { expression {params.RUN_API_CONVERSION} }
   429                      environment {
   430                          TEST_ENV = "KIND"
   431                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/apiconversion-update"
   432                      }
   433                      steps {
   434                          runGinkgo('update/apiconversion')
   435                      }
   436                      post {
   437                          always {
   438                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   439                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   440                          }
   441                      }
   442                  }
   443  
   444                  stage('AuthProxy Update Tests') {
   445                      when { expression {params.RUN_AUTHPROXY} }
   446                      environment {
   447                          TEST_ENV = "KIND"
   448                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/authproxy-update"
   449                      }
   450                      steps {
   451                          runGinkgo('update/authproxy')
   452                      }
   453                      post {
   454                          always {
   455                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   456                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   457                          }
   458                      }
   459                  }
   460  
   461                  stage('EnvironmentName/DNS/CertManager Update Tests') {
   462                      when { expression {params.RUN_CERT_MANAGER} }
   463                      environment {
   464                          TEST_ENV = "KIND"
   465                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/env-dns-cm-update"
   466                      }
   467                      steps {
   468                          sh """
   469                              ${WORKSPACE}/ci/scripts/install_third_party_components.sh
   470                          """
   471                          runGinkgo('update/env-dns-cm')
   472                      }
   473                      post {
   474                          always {
   475                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   476                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   477                          }
   478                      }
   479                  }
   480  
   481                  stage('Nginx/Istio Update Tests') {
   482                      when { expression {params.RUN_NGINX_ISTIO} }
   483                      environment {
   484                          TEST_ENV = "KIND"
   485                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/nginx-istio-update"
   486                      }
   487                      steps {
   488                          runGinkgo('update/nginxistio')
   489                      }
   490                      post {
   491                          always {
   492                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   493                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   494                          }
   495                      }
   496                  }
   497  
   498                  stage('Fluentd Update Tests') {
   499                      when { expression {params.RUN_FLUENTD} }
   500                      environment {
   501                          TEST_ENV = "KIND"
   502                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/fluentd-update"
   503                      }
   504                      steps {
   505                          runGinkgo('update/fluentd')
   506                      }
   507                      post {
   508                          always {
   509                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   510                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   511                          }
   512                      }
   513                  }
   514  
   515  
   516                  stage('Opensearch Update Tests') {
   517                      when { expression {params.RUN_OPENSEARCH} }
   518                      environment {
   519                          TEST_ENV = "KIND"
   520                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/opensearch-update"
   521                      }
   522                      steps {
   523                          runGinkgo('update/opensearch')
   524                      }
   525                      post {
   526                          always {
   527                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   528                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   529                          }
   530                      }
   531                  }
   532  
   533                  stage('Jaeger Update Tests') {
   534                      when { expression {params.RUN_JAEGER} }
   535                      environment {
   536                          TEST_ENV = "KIND"
   537                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/jaeger-update"
   538                      }
   539                      steps {
   540                          runGinkgo('update/jaeger')
   541                      }
   542                      post {
   543                          always {
   544                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   545                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   546                          }
   547                      }
   548                  }
   549  
   550                  stage('Fluent Operator Infra Tests') {
   551                      when { expression {params.RUN_FLUENT_OPERATOR} }
   552                      environment {
   553                          TEST_ENV = "KIND"
   554                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/fluent-operator-infra"
   555                      }
   556                      steps {
   557                          runGinkgo('update/fluent-operator/infra')
   558                      }
   559                      post {
   560                          always {
   561                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   562                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   563                          }
   564                      }
   565                  }
   566  
   567                  stage('Post-install overrides Tests') {
   568                      when { expression {params.RUN_POST_INSTALL_OVERRIDES} }
   569                      environment {
   570                          TEST_ENV = "KIND"
   571                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/overrides"
   572                      }
   573                      steps {
   574                          runGinkgo('update/overrides')
   575                      }
   576                      post {
   577                          always {
   578                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   579                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   580                          }
   581                      }
   582                  }
   583  
   584                  stage('Post-update2 Verify Tests') {
   585                      when { expression {params.RUN_POST_UPDATE2} }
   586                      environment {
   587                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update2-verify-install"
   588                      }
   589                      steps {
   590                          runGinkgoRandomize('verify-install')
   591                      }
   592                      post {
   593                          always {
   594                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   595                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   596                          }
   597                      }
   598                  }
   599  
   600                  stage('Post-Update2 Infra Tests') {
   601                      when { expression {params.RUN_INFRA_TESTS} }
   602                      environment {
   603                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-update2-infra-tests"
   604                      }
   605                      steps {
   606                          script {
   607                              parallel generateVerifyInfraStages("${TEST_DUMP_ROOT}/post-update2-infra-tests")
   608                          }
   609                      }
   610                      post {
   611                          always {
   612                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   613                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   614                          }
   615                      }
   616                  }
   617  
   618                  stage('Post-update2 Acceptance Tests') {
   619                      when { expression {params.RUN_POST_UPDATE2} }
   620                      environment {
   621                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-update"
   622                      }
   623                      steps {
   624                          // Run all acceptance tests after the sequence of updates including CM and DNS; undeploy apps after
   625                          script {
   626                              parallel generateAllAcceptanceTestStages("${TEST_DUMP_ROOT}/post-cm-dns-acceptance-tests", 'true', 'false')
   627                          }
   628                      }
   629                      post {
   630                          always {
   631                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   632                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   633                          }
   634                      }
   635                  }
   636              }
   637              post {
   638                  failure {
   639                      script {
   640                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   641                              dumpK8sCluster('dynamic-update-failed-cluster-snapshot')
   642                          }
   643                      }
   644                  }
   645                  success {
   646                      script {
   647                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   648                              dumpK8sCluster('dynamic-update-success-cluster-snapshot')
   649                          }
   650                      }
   651                  }
   652              }
   653          }
   654      }
   655  
   656      post {
   657          always {
   658              script {
   659                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   660                      dumpVerrazzanoSystemPods()
   661                      dumpCattleSystemPods()
   662                      dumpNginxIngressControllerLogs()
   663                      dumpVerrazzanoPlatformOperatorLogs("post-run")
   664                      dumpVerrazzanoApplicationOperatorLogs()
   665                      dumpOamKubernetesRuntimeLogs()
   666                      dumpVerrazzanoApiLogs()
   667                  }
   668              }
   669              sh """
   670                  # Copy the generated test reports to WORKSPACE to archive them
   671                  mkdir -p ${TEST_REPORT_DIR}
   672                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   673                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   674              """
   675              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true
   676              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   677              deleteCluster()
   678          }
   679          failure {
   680              sh """
   681                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   682              """
   683              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   684              sh """
   685                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   686                  oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   687                  rm archive.zip
   688              """
   689          }
   690          cleanup {
   691              deleteDir()
   692          }
   693      }
   694  }
   695  
   696  def generateNonWLSAcceptanceTestStages(dumpRoot, skipDeploy = 'false', skipUndeploy = 'false') {
   697      return generate_security_tests(dumpRoot) + generate_nonwls_tests(dumpRoot, skipDeploy, skipUndeploy)
   698  }
   699  
   700  def generateAllAcceptanceTestStages(dumpRoot, skipDeploy = 'false', skipUndeploy = 'false') {
   701      return generate_security_tests(dumpRoot) +
   702          generate_nonwls_tests(dumpRoot, skipDeploy, skipUndeploy) +
   703          generate_wls_tests(dumpRoot, skipDeploy, skipUndeploy)
   704  }
   705  
   706  def generateVerifyInfraStages(dumpRoot) {
   707      return [
   708          "verify-scripts": {
   709              runGinkgo('scripts', '', "${KUBECONFIG}")
   710          },
   711          "verify-infra restapi": {
   712              runGinkgoRandomize('verify-infra/restapi', "${dumpRoot}/verify-infra-restapi")
   713          },
   714          "verify-infra oam": {
   715              runGinkgoRandomize('verify-infra/oam', "${dumpRoot}/verify-infra-oam")
   716          },
   717          "system component metrics": {
   718              runGinkgoRandomize('metrics/syscomponents', "${dumpRoot}/system-component-metrics")
   719          },
   720          "verify-infra vmi": {
   721              runGinkgoRandomize('verify-infra/vmi', "${dumpRoot}/verify-infra-vmi")
   722          },
   723      ]
   724  }
   725  
   726  def generate_security_tests(dumpRoot) {
   727      return [
   728          "istio authorization policy": {
   729              runGinkgo('istio/authz', "${dumpRoot}/istio-authz-policy")
   730          },
   731          "security rbac": {
   732              runGinkgo('security/rbac', "${dumpRoot}/sec-role-based-access")
   733          },
   734          "security network policies": {
   735              if (params.CREATE_CLUSTER_USE_CALICO == true) {
   736                  runGinkgo('security/netpol', "${dumpRoot}/netpol")
   737              } else {
   738                  echo "Calico not enabled, skipping network policies tests"
   739              }
   740          },
   741      ]
   742  }
   743  
   744  def generate_nonwls_tests(dumpRoot, skipDeploy = 'false', skipUndeploy = 'false') {
   745      return [
   746          "deployment metrics": {
   747              runGinkgo('metrics/deploymetrics', "${dumpRoot}/k8sdeploy-workload-metrics")
   748          },
   749          "examples logging helidon": {
   750              runGinkgo('logging/helidon', "${dumpRoot}/examples-logging-helidon")
   751          },
   752          "examples springboot": {
   753              runGinkgoAppTest('examples/springboot', "springboot", "${dumpRoot}/examples-spring", skipDeploy, skipUndeploy)
   754          },
   755          "examples helidon": {
   756              runGinkgoAppTest('examples/helidon', "hello-helidon", "${dumpRoot}/examples-helidon", skipDeploy, skipUndeploy)
   757          },
   758          "examples helidon-config": {
   759              runGinkgoAppTest('examples/helidonconfig', "helidon-config", "${dumpRoot}/examples-helidon-config", skipDeploy, skipUndeploy)
   760          },
   761      ]
   762  }
   763  
   764  def generate_wls_tests(dumpRoot, skipDeploy = 'false', skipUndeploy = 'false') {
   765      return [
   766          "weblogic workload": {
   767              runGinkgoAppTest('workloads/weblogic', "hello-wls", "${dumpRoot}/weblogic-workload", skipDeploy, skipUndeploy)
   768          },
   769          "coherence workload": {
   770              runGinkgoAppTest('workloads/coherence', "hello-coherence", "${dumpRoot}/coherence-workload", skipDeploy, skipUndeploy)
   771          },
   772          "console ingress": {
   773              // doesn't work with the deployement hooks
   774              runGinkgo('ingress/console', "wls-console", "${dumpRoot}/console-ingress")
   775          },
   776      ]
   777  }
   778  
   779  def runGinkgoRandomize(testSuitePath, dumpDir = '') {
   780      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   781          sh """
   782              if [ ! -z "${dumpDir}" ]; then
   783                  export DUMP_DIRECTORY=${dumpDir}
   784              fi
   785              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   786              ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   787          """
   788      }
   789  }
   790  
   791  def runGinkgo(testSuitePath, dumpDir = '', kubeconfig = '') {
   792      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   793          sh """
   794              if [ ! -z "${dumpDir}" ]; then
   795                  export DUMP_DIRECTORY=${dumpDir}
   796              fi
   797              if [ ! -z "${kubeConfig}" ]; then
   798                  export KUBECONFIG="${kubeConfig}"
   799              fi
   800              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   801              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   802          """
   803      }
   804  }
   805  
   806  def runGinkgoAppTest(testSuitePath, namespace, dumpDir = '', skipDeploy = 'false', skipUndeploy = 'false') {
   807      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   808          sh """
   809              if [ ! -z "${dumpDir}" ]; then
   810                  export DUMP_DIRECTORY=${dumpDir}
   811              fi
   812              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   813              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace}
   814          """
   815      }
   816  }
   817  
   818  def dumpK8sCluster(dumpDirectory) {
   819      sh """
   820          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   821      """
   822  }
   823  
   824  def dumpVerrazzanoSystemPods() {
   825      sh """
   826          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   827          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log"
   828          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   829          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log"
   830          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   831          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log"
   832          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   833          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log"
   834          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   835      """
   836  }
   837  
   838  def dumpCattleSystemPods() {
   839      sh """
   840          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   841          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log"
   842          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   843          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log"
   844          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   845      """
   846  }
   847  
   848  def dumpNginxIngressControllerLogs() {
   849      sh """
   850          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   851          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log"
   852          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   853      """
   854  }
   855  
   856  def dumpVerrazzanoPlatformOperatorLogs(stage) {
   857      sh """
   858          ## dump out verrazzano-platform-operator logs
   859          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   860          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   861          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   862          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   863          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   864          echo "------------------------------------------"
   865      """
   866  }
   867  
   868  def dumpVerrazzanoApplicationOperatorLogs() {
   869      sh """
   870          ## dump out verrazzano-application-operator logs
   871          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   872          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   873          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   874          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   875          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   876          echo "------------------------------------------"
   877      """
   878  }
   879  
   880  def dumpOamKubernetesRuntimeLogs() {
   881      sh """
   882          ## dump out oam-kubernetes-runtime logs
   883          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   884          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   885          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   886          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   887          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   888          echo "------------------------------------------"
   889      """
   890  }
   891  
   892  def dumpVerrazzanoApiLogs() {
   893      sh """
   894          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   895          export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-api.log"
   896          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-api-*" -m "verrazzano api" -c verrazzano-api -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   897      """
   898  }
   899  
   900  def getEffectiveDumpOnSuccess() {
   901      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   902      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   903          effectiveValue = true
   904          echo "Forcing dump on success based on global override setting"
   905      }
   906      return effectiveValue
   907  }
   908  
   909  // Called in parallel Stage console of Stage Run Acceptance Tests
   910  def acceptanceTestsConsole(dumpRoot) {
   911      catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
   912          try {
   913              sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh"
   914          } catch (err) {
   915              saveConsoleScreenShots()
   916              error "${err}"
   917          }
   918      }
   919  }
   920  
   921  def saveConsoleScreenShots() {
   922      sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh"
   923  }
   924  
   925  def deleteCluster() {
   926      sh """
   927          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   928          make delete-cluster
   929          if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   930            echo "Failures seen during dumping of artifacts, treat post as failed"
   931            exit 1
   932          fi
   933      """
   934  }
   935  
   936  def setDisplayName() {
   937      echo "Start setDisplayName"
   938      def causes = currentBuild.getBuildCauses()
   939      echo "causes: " + causes.toString()
   940      for (cause in causes) {
   941          def causeString = cause.toString()
   942          echo "current cause: " + causeString
   943          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   944               echo "This job was caused by " + causeString
   945               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   946                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   947               } else if (causeString.contains("verrazzano-flaky-tests")) {
   948                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   949               }
   950           }
   951      }
   952      echo "End setDisplayName"
   953  }
   954  
   955  // Update1 enables disabled components except WLS/COH
   956  def vzUpdateComponentsV1alpha1(){
   957  
   958      sh """
   959          cd ${GO_REPO_PATH}/verrazzano
   960          yq -i eval '.spec.components.ingress.enabled = true' ${WORKSPACE}/vz-update1.yaml
   961          yq -i eval '.spec.components.elasticsearch.enabled = true' ${WORKSPACE}/vz-update1.yaml
   962          yq -i eval '.spec.components.kibana.enabled = true' ${WORKSPACE}/vz-update1.yaml
   963          kubectl apply -f ${WORKSPACE}/vz-update1.yaml
   964          ${TEST_SCRIPTS_DIR}/wait-for-component-ready.sh
   965      """
   966  }
   967  
   968  // Update1 enables disabled components except WLS/COH
   969  def vzUpdateComponentsV1beta1(){
   970  
   971      sh """
   972          cd ${GO_REPO_PATH}/verrazzano
   973          yq -i eval '.spec.components.ingressNGINX.enabled = true' ${WORKSPACE}/vz-update1.yaml
   974          yq -i eval '.spec.components.opensearch.enabled = true' ${WORKSPACE}/vz-update1.yaml
   975          yq -i eval '.spec.components.opensearchDashboards.enabled = true' ${WORKSPACE}/vz-update1.yaml
   976          kubectl apply -f ${WORKSPACE}/vz-update1.yaml
   977          ${TEST_SCRIPTS_DIR}/wait-for-component-ready.sh
   978      """
   979  }
   980  
   981  def scriptForPrepareATEnvironmentV1beta1(){
   982  
   983      sh """
   984          cd ${GO_REPO_PATH}/verrazzano
   985          yq -i eval '.spec.components.ingressNGINX.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   986          yq -i eval '.spec.components.opensearch.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   987          yq -i eval '.spec.components.opensearchDashboards.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   988          ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
   989      """
   990  }
   991  
   992  def scriptForPrepareATEnvironmentV1alpha1(){
   993  
   994      sh """
   995          cd ${GO_REPO_PATH}/verrazzano
   996          yq -i eval '.spec.components.ingress.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   997          yq -i eval '.spec.components.elasticsearch.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   998          yq -i eval '.spec.components.kibana.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
   999          ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
  1000      """
  1001  }
  1002  
  1003  def runCustomResourceValidation(){
  1004      sh """
  1005         cd ${GO_REPO_PATH}/verrazzano
  1006         ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN}
  1007         oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-vpo.yaml
  1008         ci/scripts/validate_custom_resource.sh ${WORKSPACE}/ci/dynamic-updates/invalidCR.yaml ${WORKSPACE}/downloaded-vpo.yaml
  1009      """
  1010  }
  1011  
  1012  def modifyConfigForInstallation() {
  1013      sh """
  1014          cd ${GO_REPO_PATH}/verrazzano
  1015          yq -i eval '.spec.components.weblogicOperator.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1016          yq -i eval '.spec.components.coherenceOperator.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1017          yq -i eval '.spec.components.kiali.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1018          yq -i eval '.spec.components.rancher.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1019          yq -i eval '.spec.components.istio.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1020          yq -i eval '.spec.components.keycloak.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1021          yq -i eval '.spec.components.applicationOperator.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1022          yq -i eval '.spec.components.oam.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1023          yq -i eval '.spec.components.fluentd.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1024          yq -i eval '.spec.components.prometheus.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1025          yq -i eval '.spec.components.grafana.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1026          yq -i eval '.spec.components.console.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1027          yq -i eval '.spec.components.verrazzano.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1028  
  1029          if [ ${params.EXTERNAL_CERT_MANAGER} == true ]; then
  1030            yq -i eval '.spec.components.certManager.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1031            yq -i eval '.spec.components.clusterIssuer.clusterResourceNamespace = "${CLUSTER_RESOURCE_NAMESPACE}"' ${INSTALL_CONFIG_FILE_KIND}
  1032          else
  1033            yq -i eval '.spec.components.certManager.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1034          fi
  1035      """
  1036  }
  1037  
  1038  def modifyConfigForUpdate1() {
  1039      sh """
  1040          cd ${GO_REPO_PATH}/verrazzano
  1041          cp ${INSTALL_CONFIG_FILE_KIND} ${WORKSPACE}/vz-update1.yaml
  1042          yq -i eval '.spec.components.kiali.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1043          yq -i eval '.spec.components.rancher.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1044          yq -i eval '.spec.components.istio.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1045          yq -i eval '.spec.components.keycloak.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1046          yq -i eval '.spec.components.applicationOperator.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1047          yq -i eval '.spec.components.oam.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1048          yq -i eval '.spec.components.fluentd.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1049          yq -i eval '.spec.components.prometheus.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1050          yq -i eval '.spec.components.grafana.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1051          yq -i eval '.spec.components.console.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1052          yq -i eval '.spec.components.verrazzano.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1053          yq -i eval '.spec.components.argoCD.enabled = true' ${WORKSPACE}/vz-update1.yaml
  1054  
  1055          if [ ${params.EXTERNAL_CERT_MANAGER} == true ]; then
  1056            yq -i eval '.spec.components.certManager.enabled = false' ${INSTALL_CONFIG_FILE_KIND}
  1057            yq -i eval '.spec.components.clusterIssuer.clusterResourceNamespace = "${CLUSTER_RESOURCE_NAMESPACE}"' ${INSTALL_CONFIG_FILE_KIND}
  1058          else
  1059            yq -i eval '.spec.components.certManager.enabled = true' ${INSTALL_CONFIG_FILE_KIND}
  1060          fi
  1061      """
  1062  }
  1063  
  1064  def modifyConfigForUpdate2() {
  1065      sh """
  1066          cd ${GO_REPO_PATH}/verrazzano
  1067          cp ${WORKSPACE}/vz-update1.yaml ${WORKSPACE}/vz-update2.yaml
  1068          yq -i eval '.spec.components.weblogicOperator.enabled = true' ${WORKSPACE}/vz-update2.yaml
  1069          yq -i eval '.spec.components.coherenceOperator.enabled = true' ${WORKSPACE}/vz-update2.yaml
  1070          kubectl apply -f ${WORKSPACE}/vz-update2.yaml
  1071          ${TEST_SCRIPTS_DIR}/wait-for-component-ready.sh
  1072      """
  1073  }