github.com/verrazzano/verrazzano@v1.7.0/ci/olcne/Jenkinsfile (about)

     1  // Copyright (c) 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  // This pipeline is configured to test Verrazzano using OCNE environment
     5  // -> Setup OCNE environment with Kubernetes, Helm and OCI CCM modules
     6  // -> Install Verrazzano
     7  // -> Deploy Verrazzano test applications
     8  // -> Run validations
     9  // -> Undeploy the apps
    10  // -> Uninstall Verrazzano
    11  // -> Cleanup resources
    12  
    13  def testEnvironments=["OCNE"]
    14  def agentLabel=env.JOB_NAME.contains('-ocne') ? "" : env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
    15  // pulling "ap-*" from the test regions given discovery of image pull issues
    16  def availableRegions=["us-ashburn-1"]
    17  def availableDomains=["hXgQ:US-ASHBURN-AD-1", "hXgQ:US-ASHBURN-AD-2", "hXgQ:US-ASHBURN-AD-3"]
    18  Collections.shuffle(availableDomains)
    19  def ocneVersions=[ "1.7", "1.6", "1.5.13", "1.5.12"]
    20  def osVersions=['8', '7.9']
    21  def uniquePrefix=UUID.randomUUID().toString().substring(0,4).replace('-','')
    22  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS=false
    23  
    24  pipeline {
    25      options {
    26          skipDefaultCheckout true
    27          copyArtifactPermission('*');
    28          timestamps ()
    29      }
    30  
    31      agent {
    32          docker {
    33              image "${RUNNER_DOCKER_IMAGE}"
    34              args "${RUNNER_DOCKER_ARGS} --cap-add=NET_ADMIN"
    35              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    36              label "$agentLabel"
    37          }
    38      }
    39  
    40      parameters {
    41          // Cluster type
    42          choice (
    43              name: 'TEST_ENV',
    44              description: 'Verrazzano Test Environment',
    45              choices: testEnvironments)
    46          // OCNE configuration
    47          choice (
    48              name: 'OCNE_VERSION',
    49              description: 'OCNE Version - OCNE-1.6(k8s-1.25.11), OCNE-1.5.13(k8s-1.24.15), OCNE-1.5.12(k8s-1.24.8), OCNE-1.7(k8s-1.26.6)',
    50              choices: ocneVersions)
    51          choice (
    52              name: 'OCNE_CLUSTER_REGION',
    53              description: 'OCI region to launch OCNE clusters in.',
    54              choices: availableRegions)
    55          choice (
    56              name: 'OCNE_CLUSTER_AVAILABILITY_DOMAIN',
    57              description: 'Availability domain for the cluster region.',
    58              choices: availableDomains)
    59          booleanParam (
    60              name: 'PRIVATE_K8S_API_ENDPOINT',
    61              description: 'Whether to setup Kubernetes API server endpoint on a virtual IP address instead of a LB',
    62              defaultValue: true)
    63          string (
    64              name: "CONTROL_PLANE_NODE_COUNT",
    65              defaultValue: '1',
    66              description: 'Number of OCNE control plane nodes',
    67              trim: true)
    68          string (
    69              name: "WORKER_NODE_COUNT",
    70              defaultValue: '2',
    71              description: 'Number of OCNE worker nodes',
    72              trim: true)
    73          choice (
    74              name: "OCNE_OS_VERSION",
    75              description: 'OracleLinux OS version for OCNE nodes',
    76              choices: osVersions)
    77          choice (name: 'CRD_API_VERSION',
    78                  description: 'This is the API crd version.',
    79                  // 1st choice is the default value
    80                  choices: [ "v1beta1", "v1alpha1"])
    81          string (
    82              name: "INSTALL_PROFILE",
    83              defaultValue: 'prod',
    84              description: 'Verrazzano install profile name',
    85              trim: true)
    86          string (
    87              name: 'GIT_COMMIT_TO_USE',
    88              description: 'This is the full git commit hash from the source build to be used for all jobs',
    89              defaultValue: 'NONE',
    90              trim: true)
    91          string (name: 'CONSOLE_REPO_BRANCH',
    92                  defaultValue: '',
    93                  description: 'The branch to check out after cloning the console repository.',
    94                  trim: true)
    95          booleanParam (
    96              name: 'DUMP_K8S_CLUSTER_ON_SUCCESS',
    97              description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)',
    98              defaultValue: false)
    99          string (name: 'TAGGED_TESTS',
   100                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
   101                  defaultValue: '',
   102                  trim: true)
   103          string (name: 'INCLUDED_TESTS',
   104                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
   105                  defaultValue: '.*',
   106                  trim: true)
   107          string (name: 'EXCLUDED_TESTS',
   108                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
   109                  defaultValue: '_excluded_test',
   110                  trim: true)
   111          booleanParam (
   112              name: 'KEEP_RESOURCES_AFTER_RUN',
   113              description: 'Whether to keep the k8s cluster and the other OCI resources alive after the run',
   114              defaultValue: false)
   115      }
   116  
   117      environment {
   118          // Repositories and credentials
   119          GHCR_REPO='ghcr.io'
   120          OCR_REPO='container-registry.oracle.com'
   121          OCR_CREDS=credentials('ocr-pull-and-push-account')
   122          NETRC_FILE=credentials('netrc')
   123          GITHUB_PKGS_CREDS=credentials('github-packages-credentials-rw')
   124          WEBLOGIC_PSW=credentials('weblogic-example-domain-password') // needed by install_todo.sh OAM example test
   125          DATABASE_PSW=credentials('todo-mysql-password') // needed by install_todo.sh OAM example test
   126          IMAGE_PULL_SECRET='verrazzano-container-registry'
   127  
   128          // Verrazzano variables
   129          TEST_ENV="${params.TEST_ENV}"
   130          VZ_ENVIRONMENT_NAME="${params.TEST_ENV.toLowerCase()}"
   131          VZ_PLATFORM_OPERTOR_YAML="$WORKSPACE/platform-operator.yaml"
   132          CLUSTER_NAME="ocne-" + "$uniquePrefix"
   133          INSTALL_PROFILE="${params.INSTALL_PROFILE}"
   134          GOPATH="$HOME/go"
   135          GO_REPO_PATH="$GOPATH/src/github.com/verrazzano"
   136          TF_REPO_PATH="$GOPATH/src/github.com/terraform-oci-ocne"
   137          KUBECONFIG="$WORKSPACE/kubeconfig"
   138          VZ_COMMAND="$GO_REPO_PATH/vz"
   139          INSTALL_CONFIG_FILE_OCNE="$GO_REPO_PATH/verrazzano/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-ocne.yaml"
   140          OCNE_DUMP_DIR="$WORKSPACE/$VZ_ENVIRONMENT_NAME"
   141          OCNE_TERRAFORM_GIT_URL="https://github.com/oracle-terraform-modules/terraform-oci-olcne.git"
   142          OCNE_BRANCH_TO_USE="origin/main"
   143          SSHUTTLE_PID_FILE="$WORKSPACE/sshuttle.pid"
   144  
   145          // Terraform variables
   146          TF_VAR_tenancy_id=credentials('oci-tenancy')
   147          TF_VAR_compartment_id=credentials('oci-tiburon-dev-compartment-ocid')
   148          TF_VAR_user_id=credentials('oci-user-ocid')
   149          TF_VAR_fingerprint=credentials('oci-api-key-fingerprint')
   150          TF_VAR_api_private_key_path=credentials('oci-api-key')
   151          TF_VAR_region="${params.OCNE_CLUSTER_REGION}"
   152          TF_VAR_availability_domain_id="${params.OCNE_CLUSTER_AVAILABILITY_DOMAIN}"
   153          TF_VAR_deploy_networking="true"
   154          TF_VAR_enable_bastion="true"
   155          TF_VAR_bastion_private_key_path="$OCNE_DUMP_DIR/id_rsa"
   156          TF_VAR_prefix="$CLUSTER_NAME"
   157          TF_VAR_use_vault="false"
   158          TF_VAR_ssh_public_key_path="$OCNE_DUMP_DIR/id_rsa.pub"
   159          TF_VAR_ssh_private_key_path="$OCNE_DUMP_DIR/id_rsa"
   160          TF_VAR_worker_node_count="${params.WORKER_NODE_COUNT}"
   161          TF_VAR_control_plane_node_count="${params.CONTROL_PLANE_NODE_COUNT}"
   162          TF_VAR_os_version="${params.OCNE_OS_VERSION}"
   163          TF_VAR_s3_bucket_access_key=credentials('oci-s3-bucket-access-key')
   164          TF_VAR_s3_bucket_secret_key=credentials('oci-s3-bucket-secret-key')
   165          TF_VAR_instance_shape="{shape:\"VM.Standard.E4.Flex\",ocpus:4,memory:64,boot_volume_size:100}"
   166          TF_VAR_bastion_shape="{shape:\"VM.Standard.E4.Flex\",ocpus:1,memory:4,boot_volume_size:50}"
   167          TF_VAR_load_balancer_shape="{shape:\"flexible\",flex_min:10,flex_max:50}"
   168          TF_VAR_provision_mode="OCNE"
   169          TF_VAR_config_file_path="config-file_ociccm.yaml"
   170          TF_VAR_debug="true"
   171          TF_VAR_virtual_ip="${params.PRIVATE_K8S_API_ENDPOINT}"
   172          OCNE_ENVNAME="myenv-${params.OCNE_VERSION}"
   173          OCNE_K8SNAME="k8s"
   174          OCNE_VERSION="${params.OCNE_VERSION}"
   175          KUBE_VERSION=ocneKubeVersion()
   176  
   177          // OCI variables
   178          OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING='True'
   179          OCI_CLI_REGION="${params.OCNE_CLUSTER_REGION}"
   180          OCI_CLI_TENANCY=credentials('oci-tenancy')
   181          OCI_CLI_USER=credentials('oci-user-ocid')
   182          OCI_CLI_FINGERPRINT=credentials('oci-api-key-fingerprint')
   183          OCI_CLI_KEY_FILE=credentials('oci-api-key')
   184  
   185          // used for cluster dump
   186          TEST_DUMP_ROOT="$WORKSPACE/test-cluster-snapshots"
   187          POST_DUMP_FAILED='false'
   188          POST_DUMP_FAILED_FILE="$WORKSPACE/post_dump_failed_file.tmp"
   189  
   190          DISABLE_SPINNER=1
   191          TIMESTAMP=sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   192          SHORT_TIME_STAMP=sh(returnStdout: true, script: "date +%m%d%H%M%S").trim()
   193  
   194          // used for console artifact capture on failure
   195          JENKINS_READ=credentials('jenkins-auditor')
   196          OCI_OS_NAMESPACE=credentials('oci-os-namespace')
   197          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   198          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   199          // OCI_COMPARTMENT_ID=credentials('oci-tiburon-dev-compartment-ocid')
   200          // OCI_TELEMETRY_URL=credentials('oci-telemetry-url')
   201          VZ_CLI_TARGZ="vz-linux-amd64.tar.gz"
   202  
   203          // used to emit metrics
   204          PROMETHEUS_CREDENTIALS=credentials('prometheus-credentials')
   205          TEST_ENV_LABEL="${params.TEST_ENV}"
   206          SEARCH_HTTP_ENDPOINT=credentials('search-gw-url')
   207          SEARCH_PASSWORD="$PROMETHEUS_CREDENTIALS_PSW"
   208          SEARCH_USERNAME="$PROMETHEUS_CREDENTIALS_USR"
   209  
   210          // used to generate Ginkgo test reports
   211          TEST_REPORT="test-report.xml"
   212          GINKGO_REPORT_ARGS="--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   213      }
   214  
   215      stages {
   216          stage('Initialize') {
   217              steps {
   218                  script {
   219                      // Proceed with checkout
   220                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS=getEffectiveDumpOnSuccess()
   221                      if (params.GIT_COMMIT_TO_USE == "NONE") {
   222                          echo "[INFO] Specific GIT commit was not specified, use current head"
   223                          def scmInfo=checkout scm
   224                          env.GIT_COMMIT=scmInfo.GIT_COMMIT
   225                          env.GIT_BRANCH=scmInfo.GIT_BRANCH
   226                      } else {
   227                          echo "[INFO] SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   228                          def scmInfo=checkout([
   229                              $class: 'GitSCM',
   230                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   231                              doGenerateSubmoduleConfigurations: false,
   232                              extensions: [],
   233                              submoduleCfg: [],
   234                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   235                          env.GIT_COMMIT=scmInfo.GIT_COMMIT
   236                          env.GIT_BRANCH=scmInfo.GIT_BRANCH
   237                          // If the commit we were handed is not what the SCM says we are using, fail
   238                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   239                              echo "[ERROR] SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   240                              exit 1
   241                          }
   242                      }
   243                      echo "[INFO] SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   244  
   245                      sh """
   246                          rm -rf $GO_REPO_PATH/verrazzano
   247                          mkdir -p $GO_REPO_PATH/verrazzano
   248                          tar cf - . | (cd $GO_REPO_PATH/verrazzano/ ; tar xf -)
   249                          rm -rf $WORKSPACE/*
   250                      """
   251  
   252                      script {
   253                          echo "[INFO] ${params.OCNE_CLUSTER_REGION}"
   254                          echo "[INFO] agentlabel: $agentLabel"
   255                          echo "[INFO] $NODE_LABELS"
   256                          SHORT_COMMIT_HASH=sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   257                          // update the description with some meaningful info
   258                          setDisplayName()
   259                          currentBuild.description=SHORT_COMMIT_HASH + " : " + params.OCNE_CLUSTER_REGION + " : OCNE-" + params.OCNE_VERSION + " : k8s-" + ocneKubeVersion()
   260                          // derive the prefix for the OCNE cluster
   261                          OCNE_CLUSTER_PREFIX=sh(returnStdout: true, script: "$GO_REPO_PATH/verrazzano/ci/scripts/derive_oke_cluster_name.sh").trim()
   262                      }
   263  
   264                      echo "[INFO] SCM checkout of $OCNE_TERRAFORM_GIT_URL $OCNE_BRANCH_TO_USE"
   265                      def scmInfo=checkout([
   266                          $class: 'GitSCM',
   267                          branches: [[name: env.OCNE_BRANCH_TO_USE]],
   268                          doGenerateSubmoduleConfigurations: false,
   269                          extensions: [],
   270                          submoduleCfg: [],
   271                          userRemoteConfigs: [[url: env.OCNE_TERRAFORM_GIT_URL]]])
   272                      env.TF_GIT_COMMIT=scmInfo.GIT_COMMIT
   273                      env.TF_GIT_BRANCH=scmInfo.GIT_BRANCH
   274                      // If the commit we were handed is not what the SCM says we are using, fail
   275                      if (!env.TF_GIT_BRANCH.equals(env.OCNE_BRANCH_TO_USE)) {
   276                          echo "[ERROR] SCM didn't checkout the branch we expected. Expected: ${env.OCNE_BRANCH_TO_USE}, Found: ${scmInfo.GIT_BRANCH}"
   277                          exit 1
   278                      }
   279                      echo "[INFO] SCM checkout of ${env.TF_GIT_BRANCH} at ${env.TF_GIT_COMMIT}"
   280  
   281                      sh """
   282                          rm -rf $TF_REPO_PATH
   283                          mkdir -p $TF_REPO_PATH
   284                          tar cf - . | (cd $TF_REPO_PATH ; tar xf -)
   285                          rm -rf $WORKSPACE/*
   286                      """
   287  
   288                      // Install terraform and generate ssh keys
   289                      sh """
   290                          sudo yum-config-manager --add-repo http://yum.oracle.com/repo/OracleLinux/OL7/developer/x86_64
   291                          sudo yum install -y terraform
   292                          mkdir -p $OCNE_DUMP_DIR
   293                          ssh-keygen -t rsa -b 4096 -q -N "" -f $TF_VAR_ssh_private_key_path
   294                          chmod 600 $TF_VAR_ssh_private_key_path
   295                      """
   296  
   297                      sh """
   298                          cp -f "$NETRC_FILE" $WORKSPACE/.netrc
   299                          chmod 600 $WORKSPACE/.netrc
   300                      """
   301  
   302                      script {
   303                          sh """
   304                              echo "Downloading VZ CLI from object storage"
   305                              oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ}
   306                              tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH}
   307                              $VZ_COMMAND version
   308                          """
   309                      }
   310                  }
   311              }
   312          }
   313  
   314          stage("Create OCNE cluster") {
   315              steps {
   316                  script {
   317                      echo "[INFO] Creating OCNE cluster"
   318                      createOCNECluster()
   319                  }
   320              }
   321          }
   322  
   323          stage("Setup SSH access") {
   324              steps {
   325                  script {
   326                      echo "[INFO] Setting up ssh access"
   327                      setupSSH()
   328                  }
   329              }
   330          }
   331  
   332          stage("Setup Kubeconfig") {
   333              steps {
   334                  script {
   335                      echo "[INFO] Setting up Kubeconfig"
   336                      setupKubeconfig()
   337                  }
   338              }
   339          }
   340  
   341          stage("Setup OCI-CCM") {
   342              steps {
   343                  script {
   344                      echo "[INFO] Setting up OCI-CCM"
   345                      setupOciCCM()
   346                  }
   347              }
   348          }
   349  
   350          stage("Create image pull secrets") {
   351              steps {
   352                  script {
   353                      echo "[INFO] Creating image pull secrets"
   354                      createImagePullSecrets()
   355                  }
   356              }
   357          }
   358  
   359          stage("Update OCI-CCM StorageClass") {
   360              steps {
   361                  script {
   362                      echo "[INFO] Update OCI-CCM StorageClass"
   363                      updateStorageClass()
   364                  }
   365              }
   366          }
   367  
   368          stage("Download Verrazzano Platform Operator") {
   369              environment {
   370                  OCI_CLI_AUTH="instance_principal"
   371              }
   372              steps {
   373                  script {
   374                      echo "[INFO] Download Verrazzano Platform Operator"
   375                      downloadVerrazzanoPlatformOperator()
   376                  }
   377              }
   378              post {
   379                  always {
   380                      archiveArtifacts artifacts: "$VZ_PLATFORM_OPERTOR_YAML", allowEmptyArchive: true
   381                  }
   382              }
   383          }
   384  
   385          stage("Install Verrazzano") {
   386              steps {
   387                  script {
   388                      echo "[INFO] Installing Verrazzano on $TEST_ENV"
   389                      installVerrazzano()
   390                  }
   391              }
   392              post {
   393                  always {
   394                      script {
   395                          dumpVerrazzanoInstallLogs()
   396                      }
   397                  }
   398              }
   399          }
   400  
   401          stage('Post-install Verify Tests') {
   402              steps {
   403                  script {
   404                      parallel generateVerifyInstallStages("${TEST_DUMP_ROOT}/post-install-verify-tests")
   405                  }
   406              }
   407              post {
   408                  always {
   409                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   410                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   411                  }
   412              }
   413          }
   414  
   415          stage('Post-install Infra Tests') {
   416              steps {
   417                  script {
   418                      parallel generateVerifyInfraStages("${TEST_DUMP_ROOT}/post-install-infra-tests")
   419                  }
   420              }
   421              post {
   422                  always {
   423                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   424                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   425                  }
   426              }
   427          }
   428  
   429          stage('Post-install Acceptance Tests') {
   430              steps {
   431                  script {
   432                      parallel generateAllAcceptanceTestStages("${TEST_DUMP_ROOT}/post-install-acceptance-tests", 'false', 'false')
   433                  }
   434              }
   435              post {
   436                  always {
   437                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true
   438                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   439                  }
   440              }
   441          }
   442      }
   443      post {
   444          always {
   445              script {
   446                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true || currentBuild.currentResult == 'FAILURE') {
   447                      dumpK8sCluster('ocne-acceptance-tests-cluster-snapshot')
   448                  }
   449              }
   450  
   451              dumpVerrazzanoSystemPods()
   452              dumpCattleSystemPods()
   453              dumpCertManagerNamespaceLogs()
   454              dumpNginxIngressControllerLogs()
   455              dumpVerrazzanoPlatformOperatorLogs()
   456              dumpVerrazzanoApplicationOperatorLogs()
   457              dumpVerrazzanoApiLogs()
   458  
   459              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/*cluster-snapshot*/**,**/test-cluster-snapshots/**/,**/${TEST_REPORT},**/ocne/**", allowEmptyArchive: true
   460              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   461              sh """
   462                  if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   463                  echo "[INFO] Failures seen during dumping of artifacts, treat post as failed"
   464                  exit 1
   465                  fi
   466              """
   467          }
   468         failure {
   469              sh """
   470                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o $WORKSPACE/build-console-output.log ${BUILD_URL}consoleText
   471              """
   472              archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   473              sh """
   474                  curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   475                  OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   476                  rm archive.zip
   477              """
   478         }
   479         cleanup {
   480              script {
   481                  if (params.KEEP_RESOURCES_AFTER_RUN == false) {
   482                      if (env.installedVZ && env.installedVZ == "true") {
   483                          try {
   484                              echo "[INFO] Deleting Verrazzano"
   485                              deleteVerrazzano()
   486                          } catch (error) {
   487                              echo "Failed to delete Verrazzano"
   488                          }
   489                      }
   490                      if (env.createdOCNECluster && env.createdOCNECluster == "true") {
   491                          try {
   492                              echo "[INFO] Deleting OCNE Cluster"
   493                              deleteOCNECluster()
   494                          } catch (error) {
   495                              echo "Failed to delete OCNE Cluster"
   496                          }
   497                      }
   498                  }
   499              }
   500              deleteDir()
   501          }
   502      }
   503  }
   504  
   505  def generateAllAcceptanceTestStages(dumpRoot, skipDeploy='false', skipUndeploy='false') {
   506      return generateSecurityTests(dumpRoot) +
   507          generateNonWLSTests(dumpRoot, skipDeploy, skipUndeploy) +
   508          generateWLSTests(dumpRoot, skipDeploy, skipUndeploy)
   509  }
   510  
   511  def generateVerifyInstallStages(dumpRoot) {
   512      return [
   513          "verify-install keycloak": {
   514              runGinkgoRandomize('verify-install/keycloak', "${dumpRoot}/verify-install-keycloak")
   515          },
   516          "verify-install kubernetes": {
   517              runGinkgoRandomize('verify-install/kubernetes', "${dumpRoot}/verify-install-kubernetes")
   518          },
   519          "verify-install istio": {
   520              runGinkgoRandomize('verify-install/istio', "${dumpRoot}/verify-install-istio")
   521          },
   522          "verify-install kiali": {
   523              runGinkgoRandomize('verify-install/kiali', "${dumpRoot}/verify-install-kiali")
   524          },
   525          "verify-install verrazzano": {
   526              runGinkgoRandomize('verify-install/verrazzano', "${dumpRoot}/verify-install-verrazzano")
   527          },
   528          "verify-install web": {
   529              runGinkgoRandomize('verify-install/web', "${dumpRoot}/verify-install-web")
   530          },
   531      ]
   532  }
   533  
   534  def generateVerifyInfraStages(dumpRoot) {
   535      return [
   536          "verify-scripts": {
   537              runGinkgo('scripts', '', "$KUBECONFIG")
   538          },
   539          "verify-infra oam": {
   540              runGinkgoRandomize('verify-infra/oam', "${dumpRoot}/verify-infra-oam")
   541          },
   542          "system component metrics": {
   543              runGinkgoRandomize('metrics/syscomponents', "${dumpRoot}/system-component-metrics")
   544          },
   545          "console": {
   546              acceptanceTestsConsole("${dumpRoot}")
   547          },
   548      ]
   549  }
   550  
   551  def generateSecurityTests(dumpRoot) {
   552      return [
   553          "istio authorization policy": {
   554              runGinkgo('istio/authz', "${dumpRoot}/istio-authz-policy")
   555          },
   556          "security rbac": {
   557              runGinkgo('security/rbac', "${dumpRoot}/sec-role-based-access")
   558          },
   559          "security network policies": {
   560              if (params.CREATE_CLUSTER_USE_CALICO == true) {
   561                  runGinkgo('security/netpol', "${dumpRoot}/netpol")
   562              } else {
   563                  echo "[INFO] Calico not enabled, skipping network policies tests"
   564              }
   565          },
   566      ]
   567  }
   568  
   569  def generateNonWLSTests(dumpRoot, skipDeploy='false', skipUndeploy='false') {
   570      return [
   571          "deployment metrics": {
   572              runGinkgo('metrics/deploymetrics', "${dumpRoot}/k8sdeploy-workload-metrics")
   573          },
   574          "examples logging helidon": {
   575              runGinkgo('logging/helidon', "${dumpRoot}/examples-logging-helidon")
   576          },
   577          "examples springboot": {
   578              runGinkgoAppTest('examples/springboot', "springboot", "${dumpRoot}/examples-spring", skipDeploy, skipUndeploy)
   579          },
   580          "examples helidon": {
   581              runGinkgoAppTest('examples/helidon', "hello-helidon", "${dumpRoot}/examples-helidon", skipDeploy, skipUndeploy)
   582          },
   583          "examples helidon-config": {
   584              runGinkgoAppTest('examples/helidonconfig', "helidon-config", "${dumpRoot}/examples-helidon-config", skipDeploy, skipUndeploy)
   585          },
   586      ]
   587  }
   588  
   589  def generateWLSTests(dumpRoot, skipDeploy='false', skipUndeploy='false') {
   590      return [
   591          "weblogic workload": {
   592              runGinkgoAppTest('workloads/weblogic', "hello-wls", "${dumpRoot}/weblogic-workload", skipDeploy, skipUndeploy)
   593          },
   594          "coherence workload": {
   595              runGinkgoAppTest('workloads/coherence', "hello-coherence", "${dumpRoot}/coherence-workload", skipDeploy, skipUndeploy)
   596          },
   597          "console ingress": {
   598              // doesn't work with the deployment hooks
   599              runGinkgo('ingress/console', "wls-console")
   600          },
   601      ]
   602  }
   603  
   604  // Called in parallel Stage console of Stage Run Acceptance Tests
   605  def acceptanceTestsConsole(dumpRoot) {
   606      catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
   607          try {
   608              sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} $GO_REPO_PATH/verrazzano/ci/scripts/run_console_tests.sh"
   609          } catch (err) {
   610              saveConsoleScreenShots()
   611              error "${err}"
   612          }
   613      }
   614  }
   615  
   616  def saveConsoleScreenShots() {
   617      sh "$GO_REPO_PATH/verrazzano/ci/scripts/save_console_test_artifacts.sh"
   618  }
   619  
   620  def runGinkgoRandomize(testSuitePath, dumpDir='') {
   621      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   622          sh """
   623              if [ ! -z "${dumpDir}" ]; then
   624                  export DUMP_DIRECTORY=${dumpDir}
   625              fi
   626              cd $GO_REPO_PATH/verrazzano/tests/e2e
   627              ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   628          """
   629      }
   630  }
   631  
   632  def runGinkgo(testSuitePath, dumpDir='', kubeconfig='') {
   633      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   634          sh """
   635              if [ ! -z "${dumpDir}" ]; then
   636                  export DUMP_DIRECTORY=${dumpDir}
   637              fi
   638              if [ ! -z "${kubeconfig}" ]; then
   639                  export KUBECONFIG="$kubeconfig"
   640              fi
   641              cd $GO_REPO_PATH/verrazzano/tests/e2e
   642              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   643          """
   644      }
   645  }
   646  
   647  def runGinkgoFailFast(testSuitePath, dumpDir='') {
   648      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   649          sh """
   650              if [! -z "${dumpDir}" ]; then
   651                  export DUMP_DIRECTORY=${dumpDir}
   652              fi
   653              cd $GO_REPO_PATH/verrazzano/tests/e2e
   654              ginkgo -v --fail-fast --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   655          """
   656      }
   657  }
   658  
   659  def runGinkgoAppTest(testSuitePath, namespace, dumpDir='', skipDeploy='false', skipUndeploy='false') {
   660      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   661          sh """
   662              if [ ! -z "${dumpDir}" ]; then
   663                  export DUMP_DIRECTORY=${dumpDir}
   664              fi
   665              cd $GO_REPO_PATH/verrazzano/tests/e2e
   666              ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace}
   667          """
   668      }
   669  }
   670  
   671  def dumpK8sCluster(dumpDirectory) {
   672      sh """
   673          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   674      """
   675  }
   676  
   677  def dumpVerrazzanoSystemPods() {
   678      sh """
   679          export DIAGNOSTIC_LOG="$WORKSPACE/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log"
   680          $GO_REPO_PATH/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   681          export DIAGNOSTIC_LOG="$WORKSPACE/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log"
   682          $GO_REPO_PATH/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   683          export DIAGNOSTIC_LOG="$WORKSPACE/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log"
   684          $GO_REPO_PATH/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   685          export DIAGNOSTIC_LOG="$WORKSPACE/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log"
   686          $GO_REPO_PATH/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   687      """
   688  }
   689  
   690  def dumpCertManagerNamespaceLogs() {
   691      sh """
   692          kubectl logs --selector=app=cert-manager -n cert-manager > $WORKSPACE/platform-operator/scripts/install/build/logs/cert-manager.log || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   693          kubectl logs --selector=app.kubernetes.io/name=external-dns -n cert-manager > $WORKSPACE/platform-operator/scripts/install/build/logs/external-dns.log || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   694      """
   695  }
   696  
   697  def dumpCattleSystemPods() {
   698      sh """
   699          export DIAGNOSTIC_LOG="$WORKSPACE/platform-operator/scripts/install/build/logs/cattle-system-pods.log"
   700          $GO_REPO_PATH/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   701          export DIAGNOSTIC_LOG="$WORKSPACE/platform-operator/scripts/install/build/logs/rancher.log"
   702          $GO_REPO_PATH/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   703      """
   704  }
   705  
   706  def dumpNginxIngressControllerLogs() {
   707      sh """
   708          export DIAGNOSTIC_LOG="$WORKSPACE/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log"
   709          $GO_REPO_PATH/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   710      """
   711  }
   712  
   713  def dumpVerrazzanoPlatformOperatorLogs() {
   714      sh """
   715          ## dump out verrazzano-platform-operator logs
   716          mkdir -p $WORKSPACE/verrazzano-platform-operator/logs
   717          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > $WORKSPACE/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   718          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > $WORKSPACE/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   719          echo "[INFO] verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   720          echo "[INFO] verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   721          echo "[INFO] ------------------------------------------"
   722      """
   723  }
   724  
   725  def dumpVerrazzanoApplicationOperatorLogs() {
   726      sh """
   727          ## dump out verrazzano-application-operator logs
   728          mkdir -p $WORKSPACE/verrazzano-application-operator/logs
   729          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > $WORKSPACE/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   730          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > $WORKSPACE/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   731          echo "[INFO] verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   732          echo "[INFO] verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   733          echo "[INFO] ------------------------------------------"
   734      """
   735  }
   736  
   737  def dumpVerrazzanoApiLogs() {
   738      sh """
   739          export DIAGNOSTIC_LOG="$WORKSPACE/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log"
   740          $GO_REPO_PATH/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "[ERROR] failed" > ${POST_DUMP_FAILED_FILE}
   741      """
   742  }
   743  
   744  def dumpVerrazzanoInstallLogs() {
   745      sh """
   746          ## dump out install logs
   747          mkdir -p $WORKSPACE/platform-operator/scripts/install/build/logs
   748          kubectl -n verrazzano-install logs --selector=job-name=verrazzano-install-my-verrazzano > $WORKSPACE/platform-operator/scripts/install/build/logs/verrazzano-install.log --tail -1
   749          kubectl -n verrazzano-install describe pod --selector=job-name=verrazzano-install-my-verrazzano > $WORKSPACE/platform-operator/scripts/install/build/logs/verrazzano-install-job-pod.out
   750          echo "[INFO] Verrazzano Installation logs dumped to verrazzano-install.log"
   751          echo "[INFO] Verrazzano Install pod description dumped to verrazzano-install-job-pod.out"
   752      """
   753  }
   754  
   755  def getEffectiveDumpOnSuccess() {
   756      def effectiveValue=params.DUMP_K8S_CLUSTER_ON_SUCCESS
   757      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   758          effectiveValue=true
   759          echo "[INFO] Forcing dump on success based on global override setting"
   760      }
   761      return effectiveValue
   762  }
   763  
   764  def setDisplayName() {
   765      echo "[INFO] Start setDisplayName"
   766      def causes=currentBuild.getBuildCauses()
   767      echo "[INFO] causes: " + causes.toString()
   768      for (cause in causes) {
   769          def causeString=cause.toString()
   770          echo "[INFO] current cause: " + causeString
   771          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   772               echo "[INFO] This job was caused by " + causeString
   773               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   774                   currentBuild.displayName=env.BUILD_NUMBER + " : PERIODIC"
   775               } else if (causeString.contains("verrazzano-flaky-tests")) {
   776                   currentBuild.displayName=env.BUILD_NUMBER + " : FLAKY"
   777               }
   778           }
   779      }
   780      echo "[INFO] End setDisplayName"
   781  }
   782  
   783  def createOCNECluster() {
   784      env.createdOCNECluster="true"
   785      sh """
   786          cd $TF_REPO_PATH
   787          cat << EOF > config-file_ociccm.yaml
   788  environments:
   789    - environment-name: ${OCNE_ENVNAME}
   790      globals:
   791        version: ${OCNE_VERSION}
   792      modules:
   793        - module: kubernetes
   794          name: ${OCNE_K8SNAME}
   795          args:
   796            restrict-service-externalip: false
   797        #- module: oci-ccm
   798        #  name: ociccm
   799  EOF
   800          terraform init -input=false
   801          terraform plan -input=false -out=tfplan -no-color
   802          terraform apply -input=false -no-color -auto-approve tfplan
   803          terraform output > $OCNE_DUMP_DIR/terraform.out
   804      """
   805      env.API_SERVER_IP=sh(returnStdout: true, script: """ cd $TF_REPO_PATH && terraform output -raw apiserver_ip """).trim()
   806      sh 'echo "API_SERVER_IP: $API_SERVER_IP"'
   807      env.WORKER_NODES_IP=sh(returnStdout: true, script: """ cd $TF_REPO_PATH && terraform output -json worker_nodes""").replaceAll(/["\[\]]/, "").replaceAll(",", " ").trim()
   808      sh 'echo "WORKER_NODES_IP: $WORKER_NODES_IP"'
   809      env.CONTROL_PLANE_NODES_IP=sh(returnStdout: true, script: """ cd $TF_REPO_PATH && terraform output -json control_plane_nodes """).replaceAll(/["\[\]]/, "").replaceAll(",", " ").trim()
   810      sh 'echo "CONTROL_PLANE_NODES_IP: $CONTROL_PLANE_NODES_IP"'
   811      env.VCN_OCID=sh(returnStdout: true, script: """ cd $TF_REPO_PATH && terraform output -raw vcn_id """).trim()
   812      sh 'echo "VCN_OCID: $VCN_OCID"'
   813      env.VCN_CIDR=sh(returnStdout: true, script: """ oci network vcn get --vcn-id "$VCN_OCID" | jq -r '.data."cidr-block"' """).trim()
   814      sh 'echo "VCN_CIDR: $VCN_CIDR"'
   815      env.NODE_SUBNET_OCID=sh(returnStdout: true, script: """ cd $TF_REPO_PATH && terraform output -raw subnet_id """).trim()
   816      sh 'echo "NODE_SUBNET_OCID: $NODE_SUBNET_OCID"'
   817      env.BASTION_SUBNET_OCID=sh(returnStdout: true, script: """ oci network subnet list -c "$TF_VAR_compartment_id" --vcn-id "$VCN_OCID" --display-name "$TF_VAR_prefix-bastion" | jq -r '.data[0].id' """).trim()
   818      sh 'echo "BASTION_SUBNET_OCID: $BASTION_SUBNET_OCID"'
   819      env.BASTION_IP=sh(returnStdout: true, script: """ cd $TF_REPO_PATH && terraform output -raw bastion_public_ip """).trim()
   820      sh 'echo "BASTION_IP: $BASTION_IP"'
   821      env.BASTION_USER=sh(returnStdout: true, script: """ cd $TF_REPO_PATH && terraform output -raw bastion_user """).trim()
   822      sh 'echo "BASTION_USER: $BASTION_USER"'
   823  }
   824  
   825  def setupSSH() {
   826      env.CONTROL_PLANE_IP=CONTROL_PLANE_NODES_IP.split(" ")[0].replaceAll(" ", "").trim()
   827      echo CONTROL_PLANE_IP
   828      sh '''
   829          sudo yum -y install oracle-epel-release-el7
   830          sudo yum -y install sshuttle
   831          if [ $? -ne 0 ]; then
   832              echo "[ERROR] Failed to install sshuttle"
   833              exit 1
   834          fi
   835          sshuttle -r $BASTION_USER@$BASTION_IP $VCN_CIDR --ssh-cmd 'ssh -o StrictHostKeyChecking=no -i '$TF_VAR_bastion_private_key_path'' --daemon --pidfile=$SSHUTTLE_PID_FILE
   836          if [ $? -ne 0 ]; then
   837              echo "[ERROR] Failed to ssh tunnel to the bastion host $TF_VAR_prefix-bastion at $BASTION_USER@$BASTION_IP"
   838              exit 1
   839          fi
   840          ssh -i "$TF_VAR_ssh_private_key_path" -fN4 -L 6443:$CONTROL_PLANE_IP:6443 "$BASTION_USER"@"$BASTION_IP"
   841      '''
   842  }
   843  
   844  def setupKubeconfig() {
   845      sh """
   846          cp ${TF_REPO_PATH}/kubeconfig "$KUBECONFIG"
   847          yq -i 'del(.clusters[0].cluster."certificate-authority-data")' "$KUBECONFIG"
   848          yq -i eval '.clusters[0].cluster.server = "https://127.0.0.1:6443"' "$KUBECONFIG"
   849          yq -i eval '.clusters[0].cluster."insecure-skip-tls-verify" = true' "$KUBECONFIG"
   850          cp "$KUBECONFIG" $OCNE_DUMP_DIR/kubeconfig
   851          kubectl get nodes
   852          kubectl get po -A
   853      """
   854  }
   855  
   856  def setupOciCCM() {
   857      sh """
   858          cd $TF_REPO_PATH
   859          $GO_REPO_PATH/verrazzano/ci/scripts/ocne_ociccm.sh
   860          kubectl get po -A
   861          kubectl get po -o yaml -n kube-system -l app=csi-oci-controller | grep image:
   862      """
   863  }
   864  
   865  def updateStorageClass() {
   866      env.CONTROL_PLANE_IP=CONTROL_PLANE_NODES_IP.split(" ")[0]
   867      echo CONTROL_PLANE_IP
   868      sh """
   869          # Add annotation to the storageclass created by OCNE oci-ccm module
   870          kubectl annotate storageclass oci-bv "storageclass.kubernetes.io/is-default-class"="true" --overwrite=true
   871          # Create a CSI Driver
   872          cat << EOF | kubectl apply -f -
   873              apiVersion: storage.k8s.io/v1
   874              kind: CSIDriver
   875              metadata:
   876                  name: blockvolume.csi.oraclecloud.com
   877              spec:
   878                  fsGroupPolicy: File
   879  EOF
   880      """
   881  }
   882  
   883  def createImagePullSecrets() {
   884      sh """
   885          $GO_REPO_PATH/verrazzano/tests/e2e/config/scripts/create-image-pull-secret.sh "$IMAGE_PULL_SECRET" "$GHCR_REPO" "$GITHUB_PKGS_CREDS_USR" "$GITHUB_PKGS_CREDS_PSW"
   886          $GO_REPO_PATH/verrazzano/tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "$GHCR_REPO" "$GITHUB_PKGS_CREDS_USR" "$GITHUB_PKGS_CREDS_PSW"
   887          $GO_REPO_PATH/verrazzano/tests/e2e/config/scripts/create-image-pull-secret.sh ocr "$OCR_REPO" "$OCR_CREDS_USR" "$OCR_CREDS_PSW"
   888          if kubectl get ns | grep verrazzano-install > /dev/null 2>&1 ; then
   889              echo "Namespace verrazzano-install already exists (with OCNE-1.7 OCI-CCM)"
   890          else
   891              kubectl create namespace verrazzano-install
   892          fi
   893          $GO_REPO_PATH/verrazzano/tests/e2e/config/scripts/create-image-pull-secret.sh "$IMAGE_PULL_SECRET" "$GHCR_REPO" "$GITHUB_PKGS_CREDS_USR" "$GITHUB_PKGS_CREDS_PSW" "verrazzano-install"
   894      """
   895  }
   896  
   897  def downloadVerrazzanoPlatformOperator() {
   898      sh """
   899          oci --region us-phoenix-1 os object get \
   900             --namespace "$OCI_OS_NAMESPACE" -bn "$OCI_OS_COMMIT_BUCKET" \
   901             --name "ephemeral/${env.BRANCH_NAME}/$SHORT_COMMIT_HASH/operator.yaml" \
   902             --file $VZ_PLATFORM_OPERTOR_YAML
   903      """
   904  }
   905  
   906  def installVerrazzano() {
   907      env.installedVZ="true"
   908      sh """
   909          # Install Verrazzano
   910          $VZ_COMMAND install -f $INSTALL_CONFIG_FILE_OCNE --set profile="$INSTALL_PROFILE" --set environmentName="$VZ_ENVIRONMENT_NAME" --manifests="$VZ_PLATFORM_OPERTOR_YAML"
   911          $VZ_COMMAND status
   912  
   913          # Print out the namespaces
   914          kubectl get namespaces
   915  
   916          # Print out the username and passwords
   917          echo "Verrazzano username: verrazzano"
   918          kubectl get secret --namespace verrazzano-system verrazzano -o jsonpath={.data.password} | base64 --decode; echo
   919          echo "Rancher username: admin"
   920          kubectl get secret --namespace cattle-system rancher-admin-secret -o jsonpath={.data.password} | base64 --decode; echo
   921          echo "Keycloak username: keycloakadmin"
   922          kubectl get secret --namespace keycloak keycloak-http -o jsonpath={.data.password} | base64 --decode; echo
   923      """
   924  }
   925  
   926  def deleteVerrazzano() {
   927      sh """
   928          $VZ_COMMAND uninstall -y --wait
   929      """
   930  }
   931  
   932  def deleteOCNECluster() {
   933      sh '''
   934          cd $TF_REPO_PATH
   935          $GO_REPO_PATH/verrazzano/ci/scripts/ocne_delete_lb_subnet.sh
   936          terraform destroy -auto-approve
   937          for x in {1..3}; do
   938              if [[ $? -ne 0 ]]; then
   939                  echo "[INFO] Failed to destroy terraform resources. Retrying ..."
   940                  terraform destroy -auto-approve
   941              else
   942                  break
   943              fi
   944          done
   945      '''
   946  }
   947  
   948  def ocneKubeVersion() {
   949      if ( params.OCNE_VERSION.equals("1.6") ) {
   950          return "1.25.11"
   951      } else if ( params.OCNE_VERSION.equals("1.5.13") ) {
   952          return "1.24.15"
   953      } else if ( params.OCNE_VERSION.equals("1.5.12") ) {
   954          return "1.24.8"
   955      }
   956      return "1.26.6"
   957  }