github.com/verrazzano/verrazzano@v1.7.1/ci/private-registry/Jenkinsfile (about)

     1  // Copyright (c) 2021, 2024, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  def DOCKER_IMAGE_TAG
     5  def agentLabel = params.DISTRIBUTION_VARIANT == "Lite" ? "2.0-large-phx" : "airgap-2.0-large"
     6  def ocirRegion = params.DISTRIBUTION_VARIANT == "Lite" ? "phx" : "fra"
     7  def ociRegionFull = params.DISTRIBUTION_VARIANT == "Lite" ? "us-phoenix-1" : "eu-frankfurt-1"
     8  def ocirRegistry = "${ocirRegion}.ocir.io"
     9  def imageRepoSubPath=""
    10  def baseImageRepo=""
    11  def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false
    12  def zipFileLoc=""
    13  def installFilePath=""
    14  
    15  pipeline {
    16      options {
    17          skipDefaultCheckout true
    18      }
    19  
    20      agent {
    21         docker {
    22              image getRunnerImage("${RUNNER_DOCKER_IMAGE}")
    23              args "${RUNNER_DOCKER_ARGS}"
    24              registryUrl "https://${ocirRegistry}"
    25              registryCredentialsId 'ocir-pull-and-push-account'
    26              label "${agentLabel}"
    27          }
    28      }
    29  
    30      parameters {
    31          choice (name: 'KUBERNETES_CLUSTER_VERSION',
    32                  description: 'Kubernetes Version for KinD Cluster',
    33                  // 1st choice is the default value
    34                  choices: [ "1.27", "1.26", "1.25", "1.24" ])
    35          choice (name: 'DISTRIBUTION_VARIANT',
    36                  description: 'Verrazzano Distribution Variant to use for testing',
    37                  choices: ["Full", "Lite"])
    38          string (name: 'GIT_COMMIT_TO_USE',
    39                          defaultValue: 'NONE',
    40                          description: 'This is the short git commit hash from the source build to be used for downloading from object storage',
    41                          trim: true)
    42          choice (name: 'WILDCARD_DNS_DOMAIN',
    43                  description: 'This is the wildcard DNS domain',
    44                  // 1st choice is the default value
    45                  choices: [ "nip.io", "sslip.io"])
    46          choice (name: 'CRD_API_VERSION',
    47                  description: 'This is the API crd version.',
    48                  // 1st choice is the default value
    49                  choices: [ "v1beta1", "v1alpha1"])
    50          string (name: 'ZIPFILE_LOCATION',
    51                          defaultValue: 'NONE',
    52                          description: 'The bucket location to use for the Zip file download',
    53                          trim: true)
    54          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    55          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    56          booleanParam (description: 'Whether to include the slow tests in the acceptance tests', name: 'RUN_SLOW_TESTS', defaultValue: false)
    57          string (name: 'TAGGED_TESTS',
    58                  defaultValue: '',
    59                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    60                  trim: true)
    61          string (name: 'INCLUDED_TESTS',
    62                  defaultValue: '.*',
    63                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    64                  trim: true)
    65          string (name: 'EXCLUDED_TESTS',
    66                  defaultValue: '_excluded_test',
    67                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    68                  trim: true)
    69          booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false)
    70      }
    71  
    72      environment {
    73          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    74          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    75          OCIR_CREDS = credentials('ocir-pull-and-push-account')
    76  
    77          GOPATH = '/home/opc/go'
    78          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    79          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    80          DOCKER_EMAIL = credentials('github-packages-email')
    81          DOCKER_REPO = 'ghcr.io'
    82          DOCKER_NAMESPACE = 'verrazzano'
    83          NETRC_FILE = credentials('netrc')
    84          CLUSTER_NAME = 'verrazzano'
    85          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    86          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    87          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    88          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    89          OCR_CREDS_USR = "${OCIR_CREDS_USR}"
    90          OCR_CREDS_PSW = "${OCIR_CREDS_PSW}"
    91          OCR_REPO = "${ocirRegistry}"
    92          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
    93          INSTALL_PROFILE = "dev"
    94          VZ_ENVIRONMENT_NAME = "default"
    95          LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test"
    96          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
    97          VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}"
    98          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test
    99          DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test
   100          SHORT_COMMIT_HASH = params.GIT_COMMIT_TO_USE.substring(0, 8)
   101  
   102          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   103  
   104          // Variables to set the OCI CLI user context to the CI user
   105          OCI_CLI_TENANCY = credentials('oci-tenancy')
   106          OCI_CLI_USER = credentials('oci-user-ocid')
   107          OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint')
   108          OCI_CLI_KEY_FILE = credentials('oci-api-key')
   109          OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True'
   110          OCI_CLI_REGION = "us-phoenix-1"
   111  
   112          // Environment variables required to capture cluster snapshot and bug report on test failure
   113          DUMP_KUBECONFIG="${KUBECONFIG}"
   114          DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh"
   115          TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshot"
   116          CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}"
   117  
   118          // Environment variable for Verrazzano CLI executable
   119          VZ_COMMAND="${GO_REPO_PATH}/vz"
   120  
   121          // Ideally use the TIBURON-DEV compartment, but we need permissions
   122          REPOSITORY_COMPARTMENT_OCID = credentials('oci-tiburon-dev-compartment-ocid')
   123  
   124          TARBALL_ROOT_DIR = "${WORKSPACE}/vz-tarball"
   125  
   126          // used to emit metrics
   127          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   128          K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}"
   129          TEST_ENV = "KIND"
   130  
   131          // used to generate Ginkgo test reports
   132          TEST_REPORT = "test-report.xml"
   133          GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true"
   134          TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e"
   135  
   136          CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}"
   137          CLEAN_PERIODIC_PREFIX = "${CLEAN_BRANCH_NAME}-last-clean-periodic-test"
   138  
   139          DISTRIBUTION_VARIANT = "${params.DISTRIBUTION_VARIANT}"
   140          AIR_GAPPED = true
   141      }
   142  
   143      stages {
   144          stage('Verify Air-Gapped') {
   145              when {
   146                  expression {
   147                      DISTRIBUTION_VARIANT == "Full"
   148                  }
   149              }
   150              steps {
   151                  script {
   152                      echo "Checking if we are running in an air-gapped environment"
   153                      def exitCode = sh(returnStatus: true, script: 'curl --max-time 20 "http://www.oracle.com"')
   154                      if (exitCode == 0) {
   155                          echo "curl command to internet succeeded, environment is not air-gapped! Exiting"
   156                          AIR_GAPPED = false
   157                          sh "exit 1"
   158                      }
   159                      echo "Verified that we are in an air-gapped environment"
   160                  }
   161              }
   162          }
   163          stage('Initialize workspace and download/extract vz source tarball') {
   164              when {
   165                  expression {
   166                      return runPipeline()
   167                  }
   168              }
   169              steps {
   170                  script {
   171                      echo "Current execution region: $ocirRegion"
   172                      echo "OCIR Repository: ${ocirRegistry}"
   173                  }
   174  
   175                  script {
   176                      EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess()
   177                      imageRepoSubPath="verrazzano-private-registry/${env.BRANCH_NAME}/b${env.BUILD_NUMBER}".trim().toLowerCase()
   178                      baseImageRepo="${env.OCI_OS_NAMESPACE}/${imageRepoSubPath}".trim().toLowerCase()
   179                      echo "Image Repo Subpath: ${imageRepoSubPath}"
   180                      echo "Base Image Repo: ${baseImageRepo}"
   181  
   182                      srcTar = "ephemeral/${env.BRANCH_NAME}/" + env.SHORT_COMMIT_HASH + "/verrazzano-src.tar.gz"
   183                      echo "source tarball: ${srcTar}"
   184                      downloadLocation = "${WORKSPACE}/vzsrc.tar.gz"
   185                      echo "source tarball download location: ${downloadLocation}"
   186                  }
   187  
   188                  sh """
   189                      cp -f "${NETRC_FILE}" $HOME/.netrc
   190                      chmod 600 $HOME/.netrc
   191                  """
   192                  println("agentlabel: ${agentLabel}")
   193                  sh """
   194                      echo "${NODE_LABELS}"
   195                  """
   196  
   197                  script {
   198                      try {
   199                          sh """
   200                              echo "${OCIR_CREDS_PSW}" | docker login -u ${OCIR_CREDS_USR} ${ocirRegistry} --password-stdin
   201                          """
   202                      } catch(error) {
   203                          echo "OCIR docker login at ${OCIR_REPO} failed, retrying after sleep"
   204                          retry(4) {
   205                              sleep(30)
   206                              sh """
   207                              echo "${OCIR_CREDS_PSW}" | docker login -u ${OCIR_CREDS_USR} ${ocirRegistry} --password-stdin
   208                              """
   209                          }
   210                      }
   211  	            }
   212                  sh """
   213                      rm -rf ${GO_REPO_PATH}/verrazzano
   214                      mkdir -p ${GO_REPO_PATH}/verrazzano
   215  
   216                      echo "Downloading VZ source tarball"
   217                      oci --region ${ociRegionFull} os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ${srcTar} --file ${downloadLocation}
   218  
   219                      echo "Extracting source"
   220                      tar xvf ${downloadLocation}
   221                      tar cf - . | (cd ${GO_REPO_PATH} ; tar xf - ; ls -las)
   222                  """
   223  
   224                  script {
   225                      VERRAZZANO_DEV_VERSION = sh(returnStdout: true, script: "grep verrazzano-development-version ${GO_REPO_PATH}/verrazzano/.verrazzano-development-version | cut -d'=' -f2 | tr -d '\n'")
   226                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   227                      DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   228                      // update the description with some meaningful info
   229                      setDisplayName()
   230                      currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + env.SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   231                  }
   232                  script {
   233                      zipFileLoc = params.ZIPFILE_LOCATION
   234                      if (params.ZIPFILE_LOCATION == "NONE") {
   235                          if (params.DISTRIBUTION_VARIANT == "Full") {
   236                              zipFileLoc = "${CLEAN_PERIODIC_PREFIX}/verrazzano-${VERRAZZANO_DEV_VERSION}.zip".trim()
   237                          } else {
   238                              zipFileLoc = "${CLEAN_PERIODIC_PREFIX}/verrazzano-${VERRAZZANO_DEV_VERSION}-lite.zip".trim()
   239                          }
   240                      }
   241                  }
   242              }
   243          }
   244  
   245          stage('Download and extract dist tarball') {
   246              when {
   247                  expression {
   248                      return runPipeline()
   249                  }
   250              }
   251               environment {
   252                   TARBALL_DIR="${TARBALL_ROOT_DIR}/verrazzano-${VERRAZZANO_DEV_VERSION}"
   253               }
   254              steps {
   255                  script {
   256                      downloadLocation = "${WORKSPACE}/${zipFileLoc}"
   257                      echo "download location: $downloadLocation"
   258                      baseFilename = sh (
   259                          script: "basename ${zipFileLoc} .zip",
   260                          returnStdout: true
   261                          ).trim()
   262  
   263                      if (params.DISTRIBUTION_VARIANT == "Lite") {
   264                          tarfileName = "verrazzano-${VERRAZZANO_DEV_VERSION}-linux-amd64.tar.gz"
   265                          checksumFile = "${tarfileName}.sha256"
   266                          echo "base name: $baseFilename, tar name: $tarfileName, checksum name: $checksumFile"
   267                      }
   268  
   269                      zipDir = sh (
   270                          script: "dirname $downloadLocation",
   271                          returnStdout: true
   272                          ).trim()
   273  
   274                      sh """
   275                          mkdir -p $zipDir || true
   276                          oci --region ${ociRegionFull} os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ${zipFileLoc} --file ${downloadLocation}
   277                      """
   278  
   279                      dir(env.TARBALL_ROOT_DIR) {
   280                          unzip zipFile: downloadLocation
   281  
   282                          if (params.DISTRIBUTION_VARIANT == "Lite") {
   283                              sh """
   284                                  sha256sum -c $checksumFile
   285                                  tar xvf $tarfileName
   286  
   287                                  # Installing VZ CLI
   288                                  cp ${TARBALL_DIR}/bin/vz ${GO_REPO_PATH}
   289                                  ${GO_REPO_PATH}/vz version # Verify install
   290                              """
   291                          } else {
   292                              sh """
   293                                  cp ${TARBALL_DIR}/bin/linux-amd64/vz ${GO_REPO_PATH}/vz
   294                                  chmod +x ${GO_REPO_PATH}/vz
   295                                  ${GO_REPO_PATH}/vz version
   296  
   297                                  # Generate components list and save to file
   298                                  chmod +x ${TARBALL_DIR}/bin/vz-registry-image-helper.sh
   299                                  cd ${TARBALL_DIR}/manifests
   300                                  sh ${TARBALL_DIR}/bin/vz-registry-image-helper.sh -d -t ${ocirRegistry} -b ${TARBALL_DIR}/manifests/verrazzano-bom.json -m ${TARBALL_ROOT_DIR}/componentsList.txt
   301                              """
   302                          }
   303                      }
   304                  }
   305              }
   306          }
   307  
   308          stage('verify distribution') {
   309              when {
   310                  expression {
   311                      return runPipeline()
   312                  }
   313              }
   314              environment {
   315                  VERRAZZANO_DEV_VERSION="${VERRAZZANO_DEV_VERSION}"
   316                  TARBALL_DIR="${TARBALL_ROOT_DIR}/verrazzano-${VERRAZZANO_DEV_VERSION}"
   317              }
   318              steps {
   319                  runGinkgoRandomize('verify-distribution')
   320              }
   321          }
   322  
   323          stage('Upload Verrazzano Images') {
   324              when {
   325                  expression {
   326                      return runPipeline()
   327                 }
   328              }
   329              environment {
   330                  TARBALL_DIR="${TARBALL_ROOT_DIR}/verrazzano-${VERRAZZANO_DEV_VERSION}"
   331              }
   332              steps {
   333                  script {
   334                      if (params.DISTRIBUTION_VARIANT == "Full") {
   335                          sh """
   336                              # Create OCIR repos for the images in the tarballs in the test compartment
   337                              sh ${TEST_SCRIPTS_DIR}/create_ocir_repositories.sh -r ${ociRegionFull} -p ${imageRepoSubPath} -d ${TARBALL_DIR}/images -c ${REPOSITORY_COMPARTMENT_OCID}
   338                              echo "Upload OCIR region: $ocirRegion"
   339                              echo "Upload OCIR Repository: ${ocirRegistry}"
   340                              cd ${TARBALL_DIR}
   341                              chmod +x ${TARBALL_DIR}/bin/vz-registry-image-helper.sh
   342                              ${TARBALL_DIR}/bin/vz-registry-image-helper.sh -t ${ocirRegistry} -l ./images -r ${baseImageRepo}
   343                          """
   344                      } else {
   345                          try {
   346                              sh """
   347                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   348                              """
   349                          } catch(error) {
   350                              echo "docker login failed, retrying after sleep"
   351                              retry(4) {
   352                                  sleep(30)
   353                                  sh """
   354                                  echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   355                                  """
   356                              }
   357                          }
   358                          sh """
   359                              # Create OCIR repos for the images in the tarballs in the test compartment
   360                              cd ${TARBALL_DIR}
   361                              sh ${TARBALL_DIR}/bin/vz-registry-image-helper.sh -b ${TARBALL_DIR}/manifests/verrazzano-bom.json -f ${TARBALL_DIR}/images
   362                              cd ${TEST_SCRIPTS_DIR}
   363                              sh ${TEST_SCRIPTS_DIR}/create_ocir_repositories.sh -r ${ociRegionFull} -p ${imageRepoSubPath} -d ${TARBALL_DIR}/images -c ${REPOSITORY_COMPARTMENT_OCID}
   364                              echo "Upload OCIR region: $ocirRegion"
   365                              echo "Upload OCIR Repository: ${ocirRegistry}"
   366                              # Run the image-helper to load the images into the OCIR registry in the runner's region
   367                              cd ${TARBALL_DIR}
   368                              chmod +x ${TARBALL_DIR}/bin/vz-registry-image-helper.sh
   369                              ${TARBALL_DIR}/bin/vz-registry-image-helper.sh -t ${ocirRegistry} -l ./images -r ${baseImageRepo}
   370                          """
   371                      }
   372                  }
   373              }
   374          }
   375          stage('Acceptance Tests') {
   376              when {
   377                  expression {
   378                      return runPipeline()
   379                  }
   380              }
   381              stages {
   382                  stage('Prepare AT environment') {
   383                      environment {
   384                          KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}"
   385                          PRIVATE_REPO="${baseImageRepo}"
   386                          REGISTRY="${ocirRegistry}"
   387                          PRIVATE_REGISTRY_USR="${OCIR_CREDS_USR}"
   388                          PRIVATE_REGISTRY_PSW="${OCIR_CREDS_PSW}"
   389                          CLUSTER_SNAPSHOT_DIR="${WORKSPACE}/verrazzano/build/resources/pre-install-resources"
   390                          TARBALL_DIR="${TARBALL_ROOT_DIR}/verrazzano-${VERRAZZANO_DEV_VERSION}"
   391                      }
   392                      steps {
   393                          script {
   394                              if (params.CRD_API_VERSION == "v1alpha1") {
   395                                  installFilePath = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1alpha1/install-verrazzano-kind-no-persistence.yaml"
   396                              } else {
   397                                  installFilePath = "${TARBALL_DIR}/manifests/profiles/dev.yaml"
   398                              }
   399                          sh """
   400                              cd ${GO_REPO_PATH}/verrazzano
   401                              chmod +x ci/scripts/prepare_distribution_test_jenkins_at_environment.sh
   402                              ci/scripts/prepare_distribution_test_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${installFilePath}
   403                          """
   404                          }
   405                      }
   406                      post {
   407                          success {
   408                              script {
   409                                  if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   410                                      dumpK8sCluster('install-success-cluster-snapshot')
   411                                  }
   412                              }
   413                          }
   414                      }
   415                  }
   416                  stage('Load Example Images') {
   417                      steps {
   418                          script {
   419                              exampleImagesTar = "ephemeral/${env.BRANCH_NAME}/" + env.SHORT_COMMIT_HASH + "/verrazzano-${VERRAZZANO_DEV_VERSION}-example-images.tar"
   420                              echo "example image tarball: ${exampleImagesTar}"
   421                              downloadLocation = "${WORKSPACE}/verrazzano-${VERRAZZANO_DEV_VERSION}-example-images.tar"
   422                              echo "example image tarball download location: ${downloadLocation}"
   423                          }
   424                          sh """
   425                              oci --region ${ociRegionFull} os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ${exampleImagesTar} --file ${downloadLocation}
   426                              kind load image-archive ${downloadLocation} --name verrazzano
   427                          """
   428                      }
   429                  }
   430                  stage('Run Acceptance Tests') {
   431                      environment {
   432                          PRIVATE_REPO="${baseImageRepo}"
   433                          REGISTRY="${ocirRegistry}"
   434                      }
   435                      stages {
   436                          stage('verify-install') {
   437                              steps {
   438                                  runGinkgoRandomize('verify-install')
   439                              }
   440                          }
   441  
   442                          stage('run-acceptance-tests') {
   443                              parallel {
   444                                  stage('verify-infra restapi') {
   445                                      steps {
   446                                          runGinkgoRandomize('verify-infra/restapi')
   447                                      }
   448                                  }
   449                                  stage('verify-infra oam') {
   450                                      steps {
   451                                          runGinkgoRandomize('verify-infra/oam')
   452                                      }
   453                                  }
   454                                  stage('verify-infra vmi') {
   455                                      steps {
   456                                          runGinkgoRandomize('verify-infra/vmi')
   457                                      }
   458                                  }
   459                                  stage('security rbac') {
   460                                      environment {
   461                                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/sec-role-based-access"
   462                                      }
   463                                      steps {
   464                                          runGinkgoKeepGoing('security/rbac')
   465                                      }
   466                                  }
   467                                  stage('examples helidon') {
   468                                      environment {
   469                                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon"
   470                                      }
   471                                      steps {
   472                                          runGinkgoKeepGoing('examples/helidon')
   473                                      }
   474                                  }
   475                                  stage('examples todo-list') {
   476                                      environment {
   477                                          DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-todo"
   478                                      }
   479                                      steps {
   480                                          runGinkgoKeepGoing('examples/todo')
   481                                      }
   482                                  }
   483                              }
   484                              post {
   485                                  always {
   486                                      archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true
   487                                      junit testResults: '**/*test-result.xml', allowEmptyResults: true
   488                                  }
   489                              }
   490                          }
   491                      }
   492                  }
   493                  // Run Verify Registry after the other tests, when we know all the pods are up and ready without
   494                  // having to duplicate the waits.
   495                  stage('verify-registry') {
   496                      environment {
   497                          REGISTRY="${ocirRegistry}"
   498                          PRIVATE_REPO="${baseImageRepo}"
   499                      }
   500                      steps {
   501                          runGinkgo('registry')
   502                      }
   503                      post {
   504                          always {
   505                              archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true
   506                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   507                          }
   508                      }
   509                  }
   510              }
   511              post {
   512                  failure {
   513                      script {
   514                          if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   515                              dumpK8sCluster('tests-failure-cluster-snapshot')
   516                          }
   517                      }
   518                  }
   519                  success {
   520                      script {
   521                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) {
   522                              dumpK8sCluster('tests-success-cluster-snapshot')
   523                          }
   524                      }
   525                  }
   526              }
   527          }
   528          stage('Uninstall') {
   529              when {
   530                  expression {
   531                      return runPipeline()
   532                  }
   533              }
   534              options {
   535                  timeout(time: 30, unit: "MINUTES")
   536              }
   537              steps {
   538                  script {
   539                      try {
   540                          sh """
   541                              ${GO_REPO_PATH}/vz uninstall -y --timeout 45m
   542                          """
   543                      } catch (err) {
   544                          currentBuild.result = "FAILURE"
   545                          echo "Caught: ${err}"
   546                          err 'Verrazzano uninstall failed'
   547                      }
   548                  }
   549              }
   550              post {
   551                  always {
   552                      sh """
   553                          echo "Listing all pods in all namespaces after uninstall"
   554                          kubectl get pods --all-namespaces
   555                          echo "-----------------------------------------------------"
   556                      """
   557                      listNamespacesAndPods('after Verrazzano uninstall')
   558                      listHelmReleases('after Verrazzano uninstall')
   559                  }
   560                  failure {
   561                      dumpK8sCluster('uninstall-failure-cluster-snapshot')
   562                  }
   563                  aborted {
   564                      dumpK8sCluster('uninstall-aborted-cluster-snapshot')
   565                  }
   566              }
   567          }
   568          stage("Verify Uninstall") {
   569              when {
   570                  expression {
   571                      return runPipeline()
   572                  }
   573              }
   574              steps {
   575                  catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   576                      sh """
   577                          ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/post-uninstall-resources false
   578                          ${LOOPING_TEST_SCRIPTS_DIR}/verify_uninstall.sh ${WORKSPACE}/verrazzano/build/resources
   579                      """
   580                  }
   581              }
   582              post {
   583                  success {
   584                      script {
   585                          if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) {
   586                              dumpK8sCluster('verify-uninstall-success-cluster-snapshot')
   587                          }
   588                      }
   589                  }
   590                  failure {
   591                      dumpK8sCluster('verify-uninstall-failed-cluster-snapshot')
   592                  }
   593              }
   594          }
   595      }
   596  
   597      post {
   598          always {
   599              script {
   600                  if (DISTRIBUTION_VARIANT == "Full" && !env.AIR_GAPPED) {
   601                      echo "Airgap check failed, not running post actions."
   602                      sh "exit 0"
   603                  }
   604              }
   605              sh """
   606                  echo "Current execution region: ${ociRegionFull}"
   607                  echo "OCIR Repository: ${ocirRegistry}"
   608                  echo "Image repo subpath: ${imageRepoSubPath}"
   609                  echo "Base image repo: ${baseImageRepo}"
   610                  sh ${TEST_SCRIPTS_DIR}/delete_ocir_repositories.sh -p ${imageRepoSubPath}  -r ${ociRegionFull} -c ${REPOSITORY_COMPARTMENT_OCID} -f -w
   611  
   612                  # Copy the generated test reports to WORKSPACE to archive them
   613                  mkdir -p ${TEST_REPORT_DIR}
   614                  cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   615                  find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR}
   616              """
   617  
   618              script {
   619                  if ( fileExists(env.TESTS_EXECUTED_FILE) ) {
   620                      dumpVerrazzanoSystemPods()
   621                      dumpCattleSystemPods()
   622                      dumpNginxIngressControllerLogs()
   623                      dumpVerrazzanoPlatformOperatorLogs()
   624                      dumpVerrazzanoApplicationOperatorLogs()
   625                      dumpOamKubernetesRuntimeLogs()
   626                      dumpVerrazzanoApiLogs()
   627                  }
   628              }
   629              archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/${TEST_REPORT}", allowEmptyArchive: true
   630              junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true
   631  
   632              sh """
   633                  cd ${GO_REPO_PATH}/verrazzano/platform-operator
   634                  make delete-cluster
   635                  cd ${WORKSPACE}/verrazzano
   636                  if [ -f ${POST_DUMP_FAILED_FILE} ]; then
   637                    echo "Failures seen during dumping of artifacts, treat post as failed"
   638                    exit 1
   639                  fi
   640              """
   641          }
   642          cleanup {
   643              deleteDir()
   644          }
   645      }
   646  }
   647  
   648  def runPipeline() {
   649      return AIR_GAPPED
   650  }
   651  
   652  def runGinkgoRandomize(testSuitePath) {
   653      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   654          sh """
   655              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   656              ginkgo -p --randomize-all -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   657              ../../build/copy-junit-output.sh ${WORKSPACE}
   658          """
   659      }
   660  }
   661  
   662  def runGinkgoKeepGoing(testSuitePath) {
   663      runGinkgo(testSuitePath, '--keep-going')
   664  }
   665  
   666  def runGinkgoFailFast(testSuitePath) {
   667      runGinkgo(testSuitePath, '--fail-fast')
   668  }
   669  
   670  def runGinkgo(testSuitePath, String... extraArgs) {
   671      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   672          // Concatenate the variable args into a single string
   673          String extraGinkgoArgs = ""
   674          extraArgs.each { extraGinkgoArgs += it + " " }
   675          sh """
   676              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   677              ginkgo -v ${extraGinkgoArgs} --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/...
   678              ../../build/copy-junit-output.sh ${WORKSPACE}
   679          """
   680      }
   681  }
   682  
   683  def dumpK8sCluster(dumpDirectory) {
   684      sh """
   685          ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}
   686      """
   687  }
   688  
   689  def dumpVerrazzanoSystemPods() {
   690      sh """
   691          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   692          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log"
   693          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   694          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log"
   695          ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   696          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-opensearchdashboards.log"
   697          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
   698          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log"
   699          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
   700      """
   701  }
   702  
   703  def dumpCattleSystemPods() {
   704      sh """
   705          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   706          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/cattle-system-pods.log"
   707          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
   708          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/rancher.log"
   709          ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   710      """
   711  }
   712  
   713  def dumpNginxIngressControllerLogs() {
   714      sh """
   715          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   716          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log"
   717          ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   718      """
   719  }
   720  
   721  def dumpVerrazzanoPlatformOperatorLogs() {
   722      sh """
   723          ## dump out verrazzano-platform-operator logs
   724          mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs
   725          kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   726          kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   727          echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
   728          echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
   729          echo "------------------------------------------"
   730      """
   731  }
   732  
   733  def dumpVerrazzanoApplicationOperatorLogs() {
   734      sh """
   735          ## dump out verrazzano-application-operator logs
   736          mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs
   737          kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   738          kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   739          echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
   740          echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
   741          echo "------------------------------------------"
   742      """
   743  }
   744  
   745  def dumpOamKubernetesRuntimeLogs() {
   746      sh """
   747          ## dump out oam-kubernetes-runtime logs
   748          mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs
   749          kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   750          kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   751          echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   752          echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   753          echo "------------------------------------------"
   754      """
   755  }
   756  
   757  def dumpVerrazzanoApiLogs() {
   758      sh """
   759          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   760          export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log"
   761          ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   762      """
   763  }
   764  
   765  def setDisplayName() {
   766      echo "Start setDisplayName"
   767      def causes = currentBuild.getBuildCauses()
   768      echo "causes: " + causes.toString()
   769      for (cause in causes) {
   770          def causeString = cause.toString()
   771          echo "current cause: " + causeString
   772          if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) {
   773               echo "This job was caused by " + causeString
   774               if (causeString.contains("verrazzano-periodic-triggered-tests")) {
   775                   currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC"
   776               } else if (causeString.contains("verrazzano-flaky-tests")) {
   777                   currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY"
   778               }
   779           }
   780      }
   781      echo "End setDisplayName"
   782  }
   783  
   784  def getEffectiveDumpOnSuccess() {
   785      def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS
   786      if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) {
   787          effectiveValue = true
   788          echo "Forcing dump on success based on global override setting"
   789      }
   790      return effectiveValue
   791  }
   792  
   793  def listNamespacesAndPods(customMessage) {
   794      sh """
   795          echo "Listing all the namespaces and pods the namespaces ${customMessage}."
   796          kubectl get namespaces
   797          kubectl get pods -A
   798          echo "-----------------------------------------------------"
   799      """
   800  }
   801  
   802  def listHelmReleases(customMessage) {
   803      sh """
   804          echo "Listing the releases across all namespaces ${customMessage}."
   805          helm list -A
   806          echo "-----------------------------------------------------"
   807      """
   808  }
   809  
   810  def getRunnerImage(runnerImageIn) {
   811      if (params.DISTRIBUTION_VARIANT.equals("Full")) {
   812          return runnerImageIn.replaceFirst("phx", "fra")
   813      }
   814      return runnerImageIn
   815  }