github.com/verrazzano/verrazzano@v1.7.0/ci/private-registry/Jenkinsfile (about)

     1  // Copyright (c) 2021, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = params.DISTRIBUTION_VARIANT == "Lite" ? "2.0-large-phx" : "airgap-2.0-large"
     6  def ocirRegion = params.DISTRIBUTION_VARIANT == "Lite" ? "phx" : "fra"
     7  def ociRegionFull = params.DISTRIBUTION_VARIANT == "Lite" ? "us-phoenix-1" : "eu-frankfurt-1"
     8  def ocirRegistry = "${ocirRegion}.ocir.io"
     9  def imageRepoSubPath=""
    10  def baseImageRepo=""
    11  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
    12  def zipFileLoc=""
    13  def installFilePath=""
    14  
    15  pipeline {
    16      options {
    17          skipDefaultCheckout true
    18      }
    19  
    20      agent {
    21         docker {
    22              image getRunnerImage("${RUNNER_DOCKER_IMAGE}")
    23              args "${RUNNER_DOCKER_ARGS}"
    24              registryUrl "https://${ocirRegistry}"
    25              registryCredentialsId 'ocir-pull-and-push-account'
    26              label "${agentLabel}"
    27          }
    28      }
    29  
    30      parameters {
    31          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    32                  description: 'Kubernetes Version for KinD Cluster',
    33                  // 1st choice is the default value
    34                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    35          choice (name: 'DISTRIBUTION_VARIANT',
    36                  description: 'Verrazzano Distribution Variant to use for testing',
    37                  choices: ["Full", "Lite"])
    38          string (name: 'GIT_COMMIT_TO_USE',
    39                          defaultValue: 'NONE',
    40                          description: 'This is the short git commit hash from the source build to be used for downloading from object storage',
    41                          trim: true)
    42          choice (name: 'WILDCARD_DNS_DOMAIN',
    43                  description: 'This is the wildcard DNS domain',
    44                  // 1st choice is the default value
    45                  choices: [ "nip.io", "sslip.io"])
    46          choice (name: 'CRD_API_VERSION',
    47                  description: 'This is the API crd version.',
    48                  // 1st choice is the default value
    49                  choices: [ "v1beta1", "v1alpha1"])
    50          string (name: 'ZIPFILE_LOCATION',
    51                          defaultValue: 'NONE',
    52                          description: 'The bucket location to use for the Zip file download',
    53                          trim: true)
    54          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    55          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    56          booleanParam (description: 'Whether to include the slow tests in the acceptance tests', name: 'RUN_SLOW_TESTS', defaultValue: false)
    57          string (name: 'TAGGED_TESTS',
    58                  defaultValue: '',
    59                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    60                  trim: true)
    61          string (name: 'INCLUDED_TESTS',
    62                  defaultValue: '.*',
    63                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    64                  trim: true)
    65          string (name: 'EXCLUDED_TESTS',
    66                  defaultValue: '_excluded_test',
    67                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    68                  trim: true)
    69          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    70      }
    71  
    72      environment {
    73          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    74          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    75          OCIR_CREDS = credentials('ocir-pull-and-push-account')
    76  
    77          GOPATH = '/home/opc/go'
    78          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    79          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    80          DOCKER_EMAIL = credentials('github-packages-email')
    81          DOCKER_REPO = 'ghcr.io'
    82          DOCKER_NAMESPACE = 'verrazzano'
    83          NETRC_FILE = credentials('netrc')
    84          CLUSTER_NAME = 'verrazzano'
    85          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    86          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    87          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    88          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    89          OCR_CREDS_USR = "${OCIR_CREDS_USR}"
    90          OCR_CREDS_PSW = "${OCIR_CREDS_PSW}"
    91          OCR_REPO = "${ocirRegistry}"
    92          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    93          INSTALL_PROFILE = "dev"
    94          VZ_ENVIRONMENT_NAME = "default"
    95          LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test"
    96          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    97          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    98          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test
    99          DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test
   100          SHORT_COMMIT_HASH = params.GIT_COMMIT_TO_USE.substring(0, 8)
   101  
   102          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   103          OCI_OS_BUCKET="verrazzano-builds"
   104          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   105  
   106          // Variables to set the OCI CLI user context to the CI user
   107          OCI_CLI_TENANCY = credentials('oci-tenancy')
   108          OCI_CLI_USER = credentials('oci-user-ocid')
   109          OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint')
   110          OCI_CLI_KEY_FILE = credentials('oci-api-key')
   111          OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True'
   112          OCI_CLI_REGION = "us-phoenix-1"
   113  
   114          // Environment variables required to capture cluster snapshot and bug report on test failure
   115          DUMP_KUBECONFIG="${KUBECONFIG}"
   116          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   117          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshot"
   118          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   119  
   120          // Environment variable for Verrazzano CLI executable
   121          VZ_COMMAND="${GO_REPO_PATH}/vz"
   122  
   123          // Ideally use the TIBURON-DEV compartment, but we need permissions
   124          REPOSITORY_COMPARTMENT_OCID = credentials('oci-tiburon-dev-compartment-ocid')
   125  
   126          TARBALL_ROOT_DIR = "${WORKSPACE}/vz-tarball"
   127  
   128          // used to emit metrics
   129          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   130          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   131          TEST_ENV = "KIND"
   132          SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
   133          SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
   134          SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
   135  
   136          // used to generate Ginkgo test reports
   137          TEST_REPORT = "test-report.xml"
   138          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   139          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   140  
   141          CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}"
   142          CLEAN_PERIODIC_PREFIX = "${CLEAN_BRANCH_NAME}-last-clean-periodic-test"
   143  
   144          DISTRIBUTION_VARIANT = "${params.DISTRIBUTION_VARIANT}"
   145          AIR_GAPPED = true
   146      }
   147  
   148      stages {
   149          stage('Verify Air-Gapped') {
   150              when {
   151                  expression {
   152                      DISTRIBUTION_VARIANT == "Full"
   153                  }
   154              }
   155              steps {
   156                  script {
   157                      echo "Checking if we are running in an air-gapped environment"
   158                      def exitCode = sh(returnStatus: true, script: 'curl --max-time 20 "http://www.oracle.com"')
   159                      if (exitCode == 0) {
   160                          echo "curl command to internet succeeded, environment is not air-gapped! Exiting"
   161                          AIR_GAPPED = false
   162                          sh "exit 1"
   163                      }
   164                      echo "Verified that we are in an air-gapped environment"
   165                  }
   166              }
   167          }
   168          stage('Initialize workspace and download/extract vz source tarball') {
   169              when {
   170                  expression {
   171                      return runPipeline()
   172                  }
   173              }
   174              steps {
   175                  script {
   176                      echo "Current execution region: $ocirRegion"
   177                      echo "OCIR Repository: ${ocirRegistry}"
   178                  }
   179  
   180                  script {
   181                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   182                      imageRepoSubPath="verrazzano-private-registry/${env.BRANCH_NAME}/b${env.BUILD_NUMBER}".trim().toLowerCase()
   183                      baseImageRepo="${env.OCI_OS_NAMESPACE}/${imageRepoSubPath}".trim().toLowerCase()
   184                      echo "Image Repo Subpath: ${imageRepoSubPath}"
   185                      echo "Base Image Repo: ${baseImageRepo}"
   186  
   187                      srcTar = "ephemeral/${env.BRANCH_NAME}/" + env.SHORT_COMMIT_HASH + "/verrazzano-src.tar.gz"
   188                      echo "source tarball: ${srcTar}"
   189                      downloadLocation = "${WORKSPACE}/vzsrc.tar.gz"
   190                      echo "source tarball download location: ${downloadLocation}"
   191                  }
   192  
   193                  sh """
   194                      cp -f "${NETRC_FILE}" $HOME/.netrc
   195                      chmod 600 $HOME/.netrc
   196                  """
   197                  println("agentlabel: ${agentLabel}")
   198                  sh """
   199                      echo "${NODE_LABELS}"
   200                  """
   201  
   202                  script {
   203                      try {
   204                          sh """
   205                              echo "${OCIR_CREDS_PSW}" | docker login -u ${OCIR_CREDS_USR} ${ocirRegistry} --password-stdin
   206                          """
   207                      } catch(error) {
   208                          echo "OCIR docker login at ${OCIR_REPO} failed, retrying after sleep"
   209                          retry(4) {
   210                              sleep(30)
   211                              sh """
   212                              echo "${OCIR_CREDS_PSW}" | docker login -u ${OCIR_CREDS_USR} ${ocirRegistry} --password-stdin
   213                              """
   214                          }
   215                      }
   216  	            }
   217                  sh """
   218                      rm -rf ${GO_REPO_PATH}/verrazzano
   219                      mkdir -p ${GO_REPO_PATH}/verrazzano
   220  
   221                      echo "Downloading VZ source tarball"
   222                      oci --region ${ociRegionFull} os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ${srcTar} --file ${downloadLocation}
   223  
   224                      echo "Extracting source"
   225                      tar xvf ${downloadLocation}
   226                      tar cf - . | (cd ${GO_REPO_PATH} ; tar xf - ; ls -las)
   227                  """
   228  
   229                  script {
   230                      VERRAZZANO_DEV_VERSION = sh(returnStdout: true, script: "grep verrazzano-development-version ${GO_REPO_PATH}/verrazzano/.verrazzano-development-version | cut -d'=' -f2 | tr -d '\n'")
   231                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   232                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   233                      // update the description with some meaningful info
   234                      setDisplayName()
   235                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + env.SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   236                  }
   237                  script {
   238                      zipFileLoc = params.ZIPFILE_LOCATION
   239                      if (params.ZIPFILE_LOCATION == "NONE") {
   240                          if (params.DISTRIBUTION_VARIANT == "Full") {
   241                              zipFileLoc = "${CLEAN_PERIODIC_PREFIX}/verrazzano-${VERRAZZANO_DEV_VERSION}.zip".trim()
   242                          } else {
   243                              zipFileLoc = "${CLEAN_PERIODIC_PREFIX}/verrazzano-${VERRAZZANO_DEV_VERSION}-lite.zip".trim()
   244                          }
   245                      }
   246                  }
   247              }
   248          }
   249  
   250          stage('Download and extract dist tarball') {
   251              when {
   252                  expression {
   253                      return runPipeline()
   254                  }
   255              }
   256               environment {
   257                   TARBALL_DIR="${TARBALL_ROOT_DIR}/verrazzano-${VERRAZZANO_DEV_VERSION}"
   258               }
   259              steps {
   260                  script {
   261                      downloadLocation = "${WORKSPACE}/${zipFileLoc}"
   262                      echo "download location: $downloadLocation"
   263                      baseFilename = sh (
   264                          script: "basename ${zipFileLoc} .zip",
   265                          returnStdout: true
   266                          ).trim()
   267  
   268                      if (params.DISTRIBUTION_VARIANT == "Lite") {
   269                          tarfileName = "verrazzano-${VERRAZZANO_DEV_VERSION}-linux-amd64.tar.gz"
   270                          checksumFile = "${tarfileName}.sha256"
   271                          echo "base name: $baseFilename, tar name: $tarfileName, checksum name: $checksumFile"
   272                      }
   273  
   274                      zipDir = sh (
   275                          script: "dirname $downloadLocation",
   276                          returnStdout: true
   277                          ).trim()
   278  
   279                      sh """
   280                          mkdir -p $zipDir || true
   281                          oci --region ${ociRegionFull} os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ${zipFileLoc} --file ${downloadLocation}
   282                      """
   283  
   284                      dir(env.TARBALL_ROOT_DIR) {
   285                          unzip zipFile: downloadLocation
   286  
   287                          if (params.DISTRIBUTION_VARIANT == "Lite") {
   288                              sh """
   289                                  sha256sum -c $checksumFile
   290                                  tar xvf $tarfileName
   291  
   292                                  # Installing VZ CLI
   293                                  cp ${TARBALL_DIR}/bin/vz ${GO_REPO_PATH}
   294                                  ${GO_REPO_PATH}/vz version # Verify install
   295                              """
   296                          } else {
   297                              sh """
   298                                  cp ${TARBALL_DIR}/bin/linux-amd64/vz ${GO_REPO_PATH}/vz
   299                                  chmod +x ${GO_REPO_PATH}/vz
   300                                  ${GO_REPO_PATH}/vz version
   301  
   302                                  # Generate components list and save to file
   303                                  chmod +x ${TARBALL_DIR}/bin/vz-registry-image-helper.sh
   304                                  cd ${TARBALL_DIR}/manifests
   305                                  sh ${TARBALL_DIR}/bin/vz-registry-image-helper.sh -d -t ${ocirRegistry} -b ${TARBALL_DIR}/manifests/verrazzano-bom.json -m ${TARBALL_ROOT_DIR}/componentsList.txt
   306                              """
   307                          }
   308                      }
   309                  }
   310              }
   311          }
   312  
   313          stage('verify distribution') {
   314              when {
   315                  expression {
   316                      return runPipeline()
   317                  }
   318              }
   319              environment {
   320                  VERRAZZANO_DEV_VERSION="${VERRAZZANO_DEV_VERSION}"
   321                  TARBALL_DIR="${TARBALL_ROOT_DIR}/verrazzano-${VERRAZZANO_DEV_VERSION}"
   322              }
   323              steps {
   324                  runGinkgoRandomize('verify-distribution')
   325              }
   326          }
   327  
   328          stage('Upload Verrazzano Images') {
   329              when {
   330                  expression {
   331                      return runPipeline()
   332                 }
   333              }
   334              environment {
   335                  TARBALL_DIR="${TARBALL_ROOT_DIR}/verrazzano-${VERRAZZANO_DEV_VERSION}"
   336              }
   337              steps {
   338                  script {
   339                      if (params.DISTRIBUTION_VARIANT == "Full") {
   340                          sh """
   341                              # Create OCIR repos for the images in the tarballs in the test compartment
   342                              sh ${TEST_SCRIPTS_DIR}/create_ocir_repositories.sh -r ${ociRegionFull} -p ${imageRepoSubPath} -d ${TARBALL_DIR}/images -c ${REPOSITORY_COMPARTMENT_OCID}
   343                              echo "Upload OCIR region: $ocirRegion"
   344                              echo "Upload OCIR Repository: ${ocirRegistry}"
   345                              cd ${TARBALL_DIR}
   346                              chmod +x ${TARBALL_DIR}/bin/vz-registry-image-helper.sh
   347                              ${TARBALL_DIR}/bin/vz-registry-image-helper.sh -t ${ocirRegistry} -l ./images -r ${baseImageRepo}
   348                          """
   349                      } else {
   350                          try {
   351                              sh """
   352                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   353                              """
   354                          } catch(error) {
   355                              echo "docker login failed, retrying after sleep"
   356                              retry(4) {
   357                                  sleep(30)
   358                                  sh """
   359                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   360                                  """
   361                              }
   362                          }
   363                          sh """
   364                              # Create OCIR repos for the images in the tarballs in the test compartment
   365                              cd ${TARBALL_DIR}
   366                              sh ${TARBALL_DIR}/bin/vz-registry-image-helper.sh -b ${TARBALL_DIR}/manifests/verrazzano-bom.json -f ${TARBALL_DIR}/images
   367                              cd ${TEST_SCRIPTS_DIR}
   368                              sh ${TEST_SCRIPTS_DIR}/create_ocir_repositories.sh -r ${ociRegionFull} -p ${imageRepoSubPath} -d ${TARBALL_DIR}/images -c ${REPOSITORY_COMPARTMENT_OCID}
   369                              echo "Upload OCIR region: $ocirRegion"
   370                              echo "Upload OCIR Repository: ${ocirRegistry}"
   371                              # Run the image-helper to load the images into the OCIR registry in the runner's region
   372                              cd ${TARBALL_DIR}
   373                              chmod +x ${TARBALL_DIR}/bin/vz-registry-image-helper.sh
   374                              ${TARBALL_DIR}/bin/vz-registry-image-helper.sh -t ${ocirRegistry} -l ./images -r ${baseImageRepo}
   375                          """
   376                      }
   377                  }
   378              }
   379          }
   380          stage('Acceptance Tests') {
   381              when {
   382                  expression {
   383                      return runPipeline()
   384                  }
   385              }
   386              stages {
   387                  stage('Prepare AT environment') {
   388                      environment {
   389                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   390                          PRIVATE_REPO="${baseImageRepo}"
   391                          REGISTRY="${ocirRegistry}"
   392                          PRIVATE_REGISTRY_USR="${OCIR_CREDS_USR}"
   393                          PRIVATE_REGISTRY_PSW="${OCIR_CREDS_PSW}"
   394                          CLUSTER_SNAPSHOT_DIR="${WORKSPACE}/verrazzano/build/resources/pre-install-resources"
   395                          TARBALL_DIR="${TARBALL_ROOT_DIR}/verrazzano-${VERRAZZANO_DEV_VERSION}"
   396                      }
   397                      steps {
   398                          script {
   399                              if (params.CRD_API_VERSION == "v1alpha1") {
   400                                  installFilePath = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1alpha1/install-verrazzano-kind-no-persistence.yaml"
   401                              } else {
   402                                  installFilePath = "${TARBALL_DIR}/manifests/profiles/dev.yaml"
   403                              }
   404                          sh """
   405                              cd ${GO_REPO_PATH}/verrazzano
   406                              chmod +x ci/scripts/prepare_distribution_test_jenkins_at_environment.sh
   407                              ci/scripts/prepare_distribution_test_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${installFilePath}
   408                          """
   409                          }
   410                      }
   411                      post {
   412                          success {
   413                              script {
   414                                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   415                                      dumpK8sCluster('install-success-cluster-snapshot')
   416                                  }
   417                              }
   418                          }
   419                      }
   420                  }
   421                  stage('Load Example Images') {
   422                      steps {
   423                          script {
   424                              exampleImagesTar = "ephemeral/${env.BRANCH_NAME}/" + env.SHORT_COMMIT_HASH + "/verrazzano-${VERRAZZANO_DEV_VERSION}-example-images.tar"
   425                              echo "example image tarball: ${exampleImagesTar}"
   426                              downloadLocation = "${WORKSPACE}/verrazzano-${VERRAZZANO_DEV_VERSION}-example-images.tar"
   427                              echo "example image tarball download location: ${downloadLocation}"
   428                          }
   429                          sh """
   430                              oci --region ${ociRegionFull} os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ${exampleImagesTar} --file ${downloadLocation}
   431                              kind load image-archive ${downloadLocation} --name verrazzano
   432                          """
   433                      }
   434                  }
   435                  stage('Run Acceptance Tests') {
   436                      environment {
   437                          PRIVATE_REPO="${baseImageRepo}"
   438                          REGISTRY="${ocirRegistry}"
   439                      }
   440                      stages {
   441                          stage('verify-install') {
   442                              steps {
   443                                  runGinkgoRandomize('verify-install')
   444                              }
   445                          }
   446  
   447                          stage('run-acceptance-tests') {
   448                              parallel {
   449                                  stage('verify-infra restapi') {
   450                                      steps {
   451                                          runGinkgoRandomize('verify-infra/restapi')
   452                                      }
   453                                  }
   454                                  stage('verify-infra oam') {
   455                                      steps {
   456                                          runGinkgoRandomize('verify-infra/oam')
   457                                      }
   458                                  }
   459                                  stage('verify-infra vmi') {
   460                                      steps {
   461                                          runGinkgoRandomize('verify-infra/vmi')
   462                                      }
   463                                  }
   464                                  stage('security rbac') {
   465                                      environment {
   466                                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/sec-role-based-access"
   467                                      }
   468                                      steps {
   469                                          runGinkgoKeepGoing('security/rbac')
   470                                      }
   471                                  }
   472                                  stage('examples helidon') {
   473                                      environment {
   474                                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon"
   475                                      }
   476                                      steps {
   477                                          runGinkgoKeepGoing('examples/helidon')
   478                                      }
   479                                  }
   480                                  stage('examples todo-list') {
   481                                      environment {
   482                                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-todo"
   483                                      }
   484                                      steps {
   485                                          runGinkgoKeepGoing('examples/todo')
   486                                      }
   487                                  }
   488                              }
   489                              post {
   490                                  always {
   491                                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true
   492                                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   493                                  }
   494                              }
   495                          }
   496                      }
   497                  }
   498                  // Run Verify Registry after the other tests, when we know all the pods are up and ready without
   499                  // having to duplicate the waits.
   500                  stage('verify-registry') {
   501                      environment {
   502                          REGISTRY="${ocirRegistry}"
   503                          PRIVATE_REPO="${baseImageRepo}"
   504                      }
   505                      steps {
   506                          runGinkgo('registry')
   507                      }
   508                      post {
   509                          always {
   510                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true
   511                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   512                          }
   513                      }
   514                  }
   515              }
   516              post {
   517                  failure {
   518                      script {
   519                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   520                              dumpK8sCluster('tests-failure-cluster-snapshot')
   521                          }
   522                      }
   523                  }
   524                  success {
   525                      script {
   526                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   527                              dumpK8sCluster('tests-success-cluster-snapshot')
   528                          }
   529                      }
   530                  }
   531              }
   532          }
   533          stage('Uninstall') {
   534              when {
   535                  expression {
   536                      return runPipeline()
   537                  }
   538              }
   539              options {
   540                  timeout(time: 30, unit: "MINUTES")
   541              }
   542              steps {
   543                  script {
   544                      try {
   545                          sh """
   546                              ${GO_REPO_PATH}/vz uninstall -y --timeout 45m
   547                          """
   548                      } catch (err) {
   549                          currentBuild.result = "FAILURE"
   550                          echo "Caught: ${err}"
   551                          err 'Verrazzano uninstall failed'
   552                      }
   553                  }
   554              }
   555              post {
   556                  always {
   557                      sh """
   558                          echo "Listing all pods in all namespaces after uninstall"
   559                          kubectl get pods --all-namespaces
   560                          echo "-----------------------------------------------------"
   561                      """
   562                      listNamespacesAndPods('after Verrazzano uninstall')
   563                      listHelmReleases('after Verrazzano uninstall')
   564                  }
   565                  failure {
   566                      dumpK8sCluster('uninstall-failure-cluster-snapshot')
   567                  }
   568                  aborted {
   569                      dumpK8sCluster('uninstall-aborted-cluster-snapshot')
   570                  }
   571              }
   572          }
   573          stage("Verify Uninstall") {
   574              when {
   575                  expression {
   576                      return runPipeline()
   577                  }
   578              }
   579              steps {
   580                  catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   581                      sh """
   582                          ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/post-uninstall-resources false
   583                          ${LOOPING_TEST_SCRIPTS_DIR}/verify_uninstall.sh ${WORKSPACE}/verrazzano/build/resources
   584                      """
   585                  }
   586              }
   587              post {
   588                  success {
   589                      script {
   590                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   591                              dumpK8sCluster('verify-uninstall-success-cluster-snapshot')
   592                          }
   593                      }
   594                  }
   595                  failure {
   596                      dumpK8sCluster('verify-uninstall-failed-cluster-snapshot')
   597                  }
   598              }
   599          }
   600      }
   601  
   602      post {
   603          always {
   604              script {
   605                  if (DISTRIBUTION_VARIANT == "Full" && !env.AIR_GAPPED) {
   606                      echo "Airgap check failed, not running post actions."
   607                      sh "exit 0"
   608                  }
   609              }
   610              sh """
   611                  echo "Current execution region: ${ociRegionFull}"
   612                  echo "OCIR Repository: ${ocirRegistry}"
   613                  echo "Image repo subpath: ${imageRepoSubPath}"
   614                  echo "Base image repo: ${baseImageRepo}"
   615                  sh ${TEST_SCRIPTS_DIR}/delete_ocir_repositories.sh -p ${imageRepoSubPath}  -r ${ociRegionFull} -c ${REPOSITORY_COMPARTMENT_OCID} -f -w
   616  
   617                  # Copy the generated test reports to WORKSPACE to archive them
   618                  mkdir -p ${TEST_REPORT_DIR}
   619                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   620                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   621              """
   622  
   623              script {
   624                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   625                      dumpVerrazzanoSystemPods()
   626                      dumpCattleSystemPods()
   627                      dumpNginxIngressControllerLogs()
   628                      dumpVerrazzanoPlatformOperatorLogs()
   629                      dumpVerrazzanoApplicationOperatorLogs()
   630                      dumpOamKubernetesRuntimeLogs()
   631                      dumpVerrazzanoApiLogs()
   632                  }
   633              }
   634              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/${TEST_REPORT}", allowEmptyArchive: true
   635              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   636  
   637              sh """
   638                  cd ${GO_REPO_PATH}/verrazzano/platform-operator
   639                  make delete-cluster
   640                  cd ${WORKSPACE}/verrazzano
   641                  if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   642                    echo "Failures seen during dumping of artifacts, treat post as failed"
   643                    exit 1
   644                  fi
   645              """
   646          }
   647          cleanup {
   648              deleteDir()
   649          }
   650      }
   651  }
   652  
   653  def runPipeline() {
   654      return AIR_GAPPED
   655  }
   656  
   657  def runGinkgoRandomize(testSuitePath) {
   658      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   659          sh """
   660              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   661              ginkgo -p --randomize-all -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   662              ../../build/copy-junit-output.sh ${WORKSPACE}
   663          """
   664      }
   665  }
   666  
   667  def runGinkgoKeepGoing(testSuitePath) {
   668      runGinkgo(testSuitePath, '--keep-going')
   669  }
   670  
   671  def runGinkgoFailFast(testSuitePath) {
   672      runGinkgo(testSuitePath, '--fail-fast')
   673  }
   674  
   675  def runGinkgo(testSuitePath, String... extraArgs) {
   676      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   677          // Concatenate the variable args into a single string
   678          String extraGinkgoArgs = ""
   679          extraArgs.each { extraGinkgoArgs += it + " " }
   680          sh """
   681              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   682              ginkgo -v ${extraGinkgoArgs} --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   683              ../../build/copy-junit-output.sh ${WORKSPACE}
   684          """
   685      }
   686  }
   687  
   688  def dumpK8sCluster(dumpDirectory) {
   689      sh """
   690          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   691      """
   692  }
   693  
   694  def dumpVerrazzanoSystemPods() {
   695      sh """
   696          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   697          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log"
   698          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   699          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log"
   700          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   701          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-opensearchdashboards.log"
   702          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   703          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log"
   704          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   705      """
   706  }
   707  
   708  def dumpCattleSystemPods() {
   709      sh """
   710          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   711          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/cattle-system-pods.log"
   712          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   713          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/rancher.log"
   714          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   715      """
   716  }
   717  
   718  def dumpNginxIngressControllerLogs() {
   719      sh """
   720          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   721          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log"
   722          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   723      """
   724  }
   725  
   726  def dumpVerrazzanoPlatformOperatorLogs() {
   727      sh """
   728          ## dump out verrazzano-platform-operator logs
   729          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   730          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   731          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   732          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   733          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   734          echo "------------------------------------------"
   735      """
   736  }
   737  
   738  def dumpVerrazzanoApplicationOperatorLogs() {
   739      sh """
   740          ## dump out verrazzano-application-operator logs
   741          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   742          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   743          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   744          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   745          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   746          echo "------------------------------------------"
   747      """
   748  }
   749  
   750  def dumpOamKubernetesRuntimeLogs() {
   751      sh """
   752          ## dump out oam-kubernetes-runtime logs
   753          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   754          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   755          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   756          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   757          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   758          echo "------------------------------------------"
   759      """
   760  }
   761  
   762  def dumpVerrazzanoApiLogs() {
   763      sh """
   764          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   765          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log"
   766          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   767      """
   768  }
   769  
   770  def setDisplayName() {
   771      echo "Start setDisplayName"
   772      def causes = currentBuild.getBuildCauses()
   773      echo "causes: " + causes.toString()
   774      for (cause in causes) {
   775          def causeString = cause.toString()
   776          echo "current cause: " + causeString
   777          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   778               echo "This job was caused by " + causeString
   779               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   780                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   781               } else if (causeString.contains("verrazzano-flaky-tests")) {
   782                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   783               }
   784           }
   785      }
   786      echo "End setDisplayName"
   787  }
   788  
   789  def getEffectiveDumpOnSuccess() {
   790      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   791      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   792          effectiveValue = true
   793          echo "Forcing dump on success based on global override setting"
   794      }
   795      return effectiveValue
   796  }
   797  
   798  def listNamespacesAndPods(customMessage) {
   799      sh """
   800          echo "Listing all the namespaces and pods the namespaces ${customMessage}."
   801          kubectl get namespaces
   802          kubectl get pods -A
   803          echo "-----------------------------------------------------"
   804      """
   805  }
   806  
   807  def listHelmReleases(customMessage) {
   808      sh """
   809          echo "Listing the releases across all namespaces ${customMessage}."
   810          helm list -A
   811          echo "-----------------------------------------------------"
   812      """
   813  }
   814  
   815  def getRunnerImage(runnerImageIn) {
   816      if (params.DISTRIBUTION_VARIANT.equals("Full")) {
   817          return runnerImageIn.replaceFirst("phx", "fra")
   818      }
   819      return runnerImageIn
   820  }