github.com/verrazzano/verrazzano@v1.7.1/Jenkinsfile (about)

     1  // Copyright (c) 2020, 2024, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  import groovy.transform.Field
     5  
     6  def DOCKER_IMAGE_TAG
     7  def SKIP_ACCEPTANCE_TESTS = false
     8  def SKIP_TRIGGERED_TESTS = false
     9  @Field
    10  def SUSPECT_LIST = ""
    11  def VERRAZZANO_DEV_VERSION = ""
    12  def VZ_BASE_IMAGE = ""
    13  def tarfilePrefix=""
    14  def storeLocation=""
    15  
    16  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
    17  
    18  pipeline {
    19      options {
    20          skipDefaultCheckout true
    21          copyArtifactPermission('*');
    22          timestamps ()
    23      }
    24  
    25      agent {
    26         docker {
    27              image "${GOLANG20_RUNNER_DOCKER_IMAGE}"
    28              args "${RUNNER_DOCKER_ARGS}"
    29              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    30              registryCredentialsId 'ocir-pull-and-push-account'
    31              label "${agentLabel}"
    32          }
    33      }
    34  
    35      parameters {
    36          booleanParam (description: 'Whether to kick off acceptance test run at the end of this build', name: 'RUN_ACCEPTANCE_TESTS', defaultValue: true)
    37          booleanParam (description: 'Whether to include the slow tests in the acceptance tests', name: 'RUN_SLOW_TESTS', defaultValue: false)
    38          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    39          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    40          booleanParam (description: 'Whether to trigger full testing after a successful run. Off by default. This is always done for successful master and release* builds, this setting only is used to enable the trigger for other branches', name: 'TRIGGER_FULL_TESTS', defaultValue: false)
    41          booleanParam (description: 'Whether to generate a tarball', name: 'GENERATE_TARBALL', defaultValue: false)
    42          booleanParam (description: 'Whether to push images to OCIR', name: 'PUSH_TO_OCIR', defaultValue: false)
    43          booleanParam (description: 'Whether to fail the Integration Tests to test failure handling', name: 'SIMULATE_FAILURE', defaultValue: false)
    44          booleanParam (description: 'Whether to perform a scan of the built images', name: 'PERFORM_SCAN', defaultValue: false)
    45          booleanParam (description: 'Whether to wait for triggered tests or not. This defaults to false, this setting is useful for things like release automation that require everything to complete successfully', name: 'WAIT_FOR_TRIGGERED', defaultValue: false)
    46          booleanParam (description: 'Whether to fail build if UT coverage number decreases lower than its release-* coverage from object storage. This defaults to false, meaning Any non release-*/master branch will WARN only if its coverage is lower. This can be enabled so that jobs will FAIL if coverage drops, for example when testing a PR before merging.', name: 'FAIL_IF_COVERAGE_DECREASED', defaultValue: false)
    47          booleanParam (description: 'Whether to write the UT coverage number to object storage. This always occurs for release-*/master branches. Defaults to true, but it can be disabled to not always upload.', name: 'UPLOAD_UNIT_TEST_COVERAGE', defaultValue: true)
    48          booleanParam (description: 'Whether to run clusterAPI override tests', name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', defaultValue: false)
    49          choice (name: 'WILDCARD_DNS_DOMAIN',
    50                  description: 'Wildcard DNS Domain',
    51                  // 1st choice is the default value
    52                  choices: [ "nip.io", "sslip.io"])
    53          choice (name: 'CRD_API_VERSION',
    54                  description: 'This is the API crd version.',
    55                  // 1st choice is the default value
    56                  choices: [ "v1beta1", "v1alpha1"])
    57          string (name: 'CONSOLE_REPO_BRANCH',
    58                  defaultValue: '',
    59                  description: 'The branch to check out after cloning the console repository.',
    60                  trim: true)
    61      }
    62  
    63      environment {
    64          TEST_ENV = "JENKINS"
    65          CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}"
    66          IS_PERIODIC_PIPELINE = "false"
    67  
    68          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    69          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    70          DOCKER_PLATFORM_IMAGE_NAME = "${env.BRANCH_NAME ==~ /^release-.*/ || env.BRANCH_NAME == 'master' ? env.DOCKER_PLATFORM_PUBLISH_IMAGE_NAME : env.DOCKER_PLATFORM_CI_IMAGE_NAME}"
    71          DOCKER_APP_CI_IMAGE_NAME = 'verrazzano-application-operator-jenkins'
    72          DOCKER_APP_PUBLISH_IMAGE_NAME = 'verrazzano-application-operator'
    73          DOCKER_APP_IMAGE_NAME = "${env.BRANCH_NAME ==~ /^release-.*/ || env.BRANCH_NAME == 'master' ? env.DOCKER_APP_PUBLISH_IMAGE_NAME : env.DOCKER_APP_CI_IMAGE_NAME}"
    74          DOCKER_CLUSTER_CI_IMAGE_NAME = 'verrazzano-cluster-operator-jenkins'
    75          DOCKER_CLUSTER_PUBLISH_IMAGE_NAME = 'verrazzano-cluster-operator'
    76          DOCKER_CLUSTER_IMAGE_NAME = "${env.BRANCH_NAME ==~ /^release-.*/ || env.BRANCH_NAME == 'master' ? env.DOCKER_CLUSTER_PUBLISH_IMAGE_NAME : env.DOCKER_CLUSTER_CI_IMAGE_NAME}"
    77          DOCKER_AUTHPROXY_CI_IMAGE_NAME = 'verrazzano-authproxy-jenkins'
    78          DOCKER_AUTHPROXY_PUBLISH_IMAGE_NAME = 'verrazzano-authproxy'
    79          DOCKER_AUTHPROXY_IMAGE_NAME = "${env.BRANCH_NAME ==~ /^release-.*/ || env.BRANCH_NAME == 'master' ? env.DOCKER_AUTHPROXY_PUBLISH_IMAGE_NAME : env.DOCKER_AUTHPROXY_CI_IMAGE_NAME}"
    80          CREATE_LATEST_TAG = "${env.BRANCH_NAME == 'master' ? '1' : '0'}"
    81          USE_V8O_DOC_STAGE = "${env.BRANCH_NAME == 'master' ? 'true' : 'false'}"
    82          GOPATH = '/home/opc/go'
    83          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    84          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    85          DOCKER_EMAIL = credentials('github-packages-email')
    86          DOCKER_REPO = 'ghcr.io'
    87          DOCKER_NAMESPACE = 'verrazzano'
    88          NETRC_FILE = credentials('netrc')
    89          GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw')
    90          SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY')
    91          RELEASE_OWNERS = credentials('release-version-owners')
    92  
    93          CLUSTER_NAME = 'verrazzano'
    94          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    95          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    96          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    97          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    98          OCR_CREDS = credentials('ocr-pull-and-push-account')
    99          OCR_REPO = 'container-registry.oracle.com'
   100          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
   101          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
   102          INSTALL_PROFILE = "dev"
   103          VZ_ENVIRONMENT_NAME = "default"
   104          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
   105  
   106          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test
   107          DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test
   108  
   109          // used for console artifact capture on failure
   110          JENKINS_READ = credentials('jenkins-auditor')
   111  
   112          OCI_CLI_AUTH="instance_principal"
   113          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   114          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   115          OCI_OS_REGION="us-phoenix-1" // where to download existing artifacts from
   116          OCI_OS_DIST_REGION="eu-frankfurt-1" // where to upload distributions to
   117  
   118          // used to emit metrics
   119          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   120  
   121          OCIR_SCAN_COMPARTMENT = credentials('ocir-scan-compartment')
   122          OCIR_SCAN_TARGET = credentials('ocir-scan-target')
   123          OCIR_SCAN_REGISTRY = credentials('ocir-scan-registry')
   124          OCIR_SCAN_REPOSITORY_PATH = credentials('ocir-scan-repository-path')
   125          DOCKER_SCAN_CREDS = credentials('v8odev-ocir')
   126  
   127          // used to write to object storage, or fail build if UT coverage does not pass
   128          FAIL_BUILD_COVERAGE = "${params.FAIL_IF_COVERAGE_DECREASED}"
   129          UPLOAD_UT_COVERAGE = "${params.UPLOAD_UNIT_TEST_COVERAGE}"
   130  
   131  	// File containing base image information
   132  	BASE_IMAGE_INFO_FILE = "base-image-v1.0.0.txt"
   133      }
   134  
   135      stages {
   136          stage('Clean workspace and checkout') {
   137              steps {
   138                  sh """
   139                      echo "${NODE_LABELS}"
   140                  """
   141  
   142                  script {
   143                      def scmInfo = checkout scm
   144                      env.GIT_COMMIT = scmInfo.GIT_COMMIT
   145                      env.GIT_BRANCH = scmInfo.GIT_BRANCH
   146                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   147                  }
   148                  sh """
   149                      cp -f "${NETRC_FILE}" $HOME/.netrc
   150                      chmod 600 $HOME/.netrc
   151                  """
   152  
   153                  script {
   154                      try {
   155                          sh """
   156                              echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   157                          """
   158                      } catch(error) {
   159                          echo "docker login failed, retrying after sleep"
   160                          retry(4) {
   161                              sleep(30)
   162                              sh """
   163                              echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   164                              """
   165                          }
   166                      }
   167                  }
   168                  moveContentToGoRepoPath()
   169  		downloadBaseImageInfoFile()
   170  
   171                  script {
   172                      def props = readProperties file: '.verrazzano-development-version'
   173                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   174                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   175                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "echo $env.GIT_COMMIT | head -c 8")
   176                      env.VERRAZZANO_VERSION = "${VERRAZZANO_DEV_VERSION}"
   177                      if (!"${env.GIT_BRANCH}".startsWith("release-")) {
   178                          env.VERRAZZANO_VERSION = "${env.VERRAZZANO_VERSION}-${env.BUILD_NUMBER}+${SHORT_COMMIT_HASH}"
   179                      }
   180                      DOCKER_IMAGE_TAG = "v${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   181                      // update the description with some meaningful info
   182                      currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT
   183                      def currentCommitHash = env.GIT_COMMIT
   184                      def commitList = getCommitList()
   185                      withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) {
   186                          def userMappings = readJSON file: JENKINS_TO_SLACK_JSON
   187                          SUSPECT_LIST = getSuspectList(commitList, userMappings)
   188                          echo "Suspect list: ${SUSPECT_LIST}"
   189                      }
   190  
   191                      def imageProps = readProperties file: "${WORKSPACE}/${BASE_IMAGE_INFO_FILE}"
   192                      VZ_BASE_IMAGE = imageProps['base-image']
   193                      env.VZ_BASE_IMAGE = "${VZ_BASE_IMAGE}"
   194                      echo "Verrazzano base image: ${VZ_BASE_IMAGE}"
   195                  }
   196              }
   197          }
   198  
   199          stage('Generate operator.yaml') {
   200              environment {
   201                  MODULE_INTEGRATION = "true"
   202              }
   203              when { not { buildingTag() } }
   204              steps {
   205                  generateOperatorYaml("${DOCKER_IMAGE_TAG}")
   206              }
   207              post {
   208                  failure {
   209                      script {
   210                          SKIP_TRIGGERED_TESTS = true
   211                      }
   212                  }
   213                  success {
   214                      archiveArtifacts artifacts: "generated-operator.yaml", allowEmptyArchive: true
   215                  }
   216              }
   217          }
   218  
   219          stage('Check Repo Clean') {
   220              steps {
   221                  checkRepoClean()
   222              }
   223              post {
   224                  failure {
   225                      script {
   226                          SKIP_TRIGGERED_TESTS = true
   227                      }
   228                  }
   229              }
   230          }
   231         stage('Verrazzano development version check') {
   232                      when { not { buildingTag() } }
   233                      steps {
   234                           script {
   235                              VERRAZZANO_DEV_VERSION_CHECK = sh (script: "git ls-remote --tags origin | grep -F ${VERRAZZANO_DEV_VERSION}", returnStatus: true )
   236                              if (VERRAZZANO_DEV_VERSION_CHECK == 0) {
   237                                  if (env.JOB_NAME ==~ "verrazzano/release-*") {
   238                                      slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nVZ Helper was not run, the Verrazzano Development Version ${VERRAZZANO_DEV_VERSION} matches a prior release\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nRelease Owners:\n ${RELEASE_OWNERS}\n")
   239                                  }
   240                              }
   241                           }
   242                      }
   243                      post {
   244                          failure {
   245                              script {
   246                                  SKIP_TRIGGERED_TESTS = true
   247                              }
   248                          }
   249                      }
   250                  }
   251  
   252          stage('Parallel Build, Test, and Compliance') {
   253              parallel {
   254                  stage('Build Verrazzano CLI and Save Binary') {
   255                      steps {
   256                          buildVerrazzanoCLI("${DOCKER_IMAGE_TAG}")
   257                      }
   258                      post {
   259                          failure {
   260                              script {
   261                                  SKIP_TRIGGERED_TESTS = true
   262                              }
   263                          }
   264                          success {
   265                              script {
   266                                  archiveArtifacts artifacts: '**/*.tar.gz*', allowEmptyArchive: true
   267                                  echo "Saving CLI Binary"
   268                                  saveCLIExecutable()
   269                              }
   270  
   271                          }
   272                      }
   273                  }
   274  
   275                  stage('Build Images and Save Generated Files') {
   276                      when { not { buildingTag() } }
   277                      steps {
   278                          script {
   279                              buildImages("${DOCKER_IMAGE_TAG}")
   280                          }
   281                      }
   282                      post {
   283                          failure {
   284                              script {
   285                                  SKIP_TRIGGERED_TESTS = true
   286                              }
   287                          }
   288                          success {
   289                              echo "Saving generated files"
   290                              saveGeneratedFiles()
   291                              script {
   292                                  archiveArtifacts artifacts: "generated-verrazzano-bom.json,verrazzano_images.txt", allowEmptyArchive: true
   293                              }
   294                          }
   295                      }
   296                  }
   297  
   298                  stage('Quality, Compliance Checks, and Unit Tests') {
   299                      when { not { buildingTag() } }
   300                      steps {
   301                          sh """
   302                      cd ${GO_REPO_PATH}/verrazzano
   303                      make precommit
   304                      make unit-test-coverage-ratcheting
   305                      echo "Checking versions..."
   306                      release/scripts/check_versions.sh ${VERRAZZANO_DEV_VERSION}
   307                  """
   308                      }
   309                      post {
   310                          failure {
   311                              script {
   312                                  SKIP_TRIGGERED_TESTS = true
   313                              }
   314                          }
   315                          always {
   316                              sh """
   317                          cd ${GO_REPO_PATH}/verrazzano
   318                          cp coverage.html ${WORKSPACE}
   319                          cp coverage.xml ${WORKSPACE}
   320                          build/copy-junit-output.sh ${WORKSPACE}
   321                      """
   322                              archiveArtifacts artifacts: '**/coverage.html', allowEmptyArchive: true
   323                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   324                              cobertura(coberturaReportFile: 'coverage.xml',
   325                                      enableNewApi: true,
   326                                      autoUpdateHealth: false,
   327                                      autoUpdateStability: false,
   328                                      failUnstable: true,
   329                                      failUnhealthy: true,
   330                                      failNoReports: true,
   331                                      onlyStable: false,
   332                                      fileCoverageTargets: '100, 0, 0',
   333                                      lineCoverageTargets: '68, 68, 68',
   334                                      packageCoverageTargets: '100, 0, 0',
   335                              )
   336                          }
   337                      }
   338                  }
   339  
   340          }
   341      }
   342  
   343          stage('Scan Image') {
   344              when {
   345                 allOf {
   346                     not { buildingTag() }
   347                     expression {params.PERFORM_SCAN == true}
   348                 }
   349              }
   350              steps {
   351                  script {
   352                      scanContainerImage "${env.DOCKER_REPO}/${env.DOCKER_NAMESPACE}/${DOCKER_PLATFORM_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
   353                      scanContainerImage "${env.DOCKER_REPO}/${env.DOCKER_NAMESPACE}/${DOCKER_APP_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
   354                      scanContainerImage "${env.DOCKER_REPO}/${env.DOCKER_NAMESPACE}/${DOCKER_CLUSTER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
   355                      scanContainerImage "${env.DOCKER_REPO}/${env.DOCKER_NAMESPACE}/${DOCKER_AUTHPROXY_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
   356                  }
   357              }
   358              post {
   359                  failure {
   360                      script {
   361                          SKIP_TRIGGERED_TESTS = true
   362                      }
   363                  }
   364                  always {
   365                      archiveArtifacts artifacts: '**/scanning-report*.json', allowEmptyArchive: true
   366                  }
   367              }
   368          }
   369  
   370          stage('Skip acceptance tests if commit message contains skip-at') {
   371              steps {
   372                  script {
   373                      // note that SKIP_ACCEPTANCE_TESTS will be false at this point (its default value)
   374                      // so we are going to run the AT's unless this logic decides to skip them...
   375  
   376                      // if we are planning to run the AT's (which is the default)
   377                      if (params.RUN_ACCEPTANCE_TESTS == true) {
   378                          SKIP_ACCEPTANCE_TESTS = false
   379                          // check if the user has asked to skip AT using the commit message
   380                          result = sh (script: "git log -1 | grep 'skip-at'", returnStatus: true)
   381                          if (result == 0) {
   382                              // found 'skip-at', so don't run them
   383                              SKIP_ACCEPTANCE_TESTS = true
   384                              echo "Skip acceptance tests based on opt-out in commit message [skip-at]"
   385                              echo "SKIP_ACCEPTANCE_TESTS is ${SKIP_ACCEPTANCE_TESTS}"
   386                          }
   387                      } else {
   388                          SKIP_ACCEPTANCE_TESTS = true
   389                      }
   390                  }
   391              }
   392          }
   393  
   394          stage('Kind Acceptance Tests on 1.27') {
   395              when {
   396                  allOf {
   397                      not { buildingTag() }
   398                      anyOf {
   399                          branch 'master';
   400                          branch 'release-*';
   401                          expression {SKIP_ACCEPTANCE_TESTS == false};
   402                      }
   403                  }
   404              }
   405  
   406              steps {
   407                  retry(count: JOB_PROMOTION_RETRIES) {
   408                      script {
   409                          build job: "verrazzano-new-kind-acceptance-tests/${BRANCH_NAME.replace("/", "%2F")}",
   410                              parameters: [
   411                                  string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'),
   412                                  string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   413                                  string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   414                                  string(name: 'CRD_API_VERSION', value: params.CRD_API_VERSION),
   415                                  booleanParam(name: 'RUN_SLOW_TESTS', value: params.RUN_SLOW_TESTS),
   416                                  booleanParam(name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', value: params.DUMP_K8S_CLUSTER_ON_SUCCESS),
   417                                  booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: params.CREATE_CLUSTER_USE_CALICO),
   418                                  booleanParam(name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', value: params.RUN_CLUSTERAPI_OVERRIDE_TESTS),
   419                                  string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH),
   420                              ], wait: true
   421                      }
   422                  }
   423              }
   424              post {
   425                  failure {
   426                      script {
   427                          SKIP_TRIGGERED_TESTS = true
   428                      }
   429                  }
   430              }
   431          }
   432          stage('Triggered Tests') {
   433              when {
   434                  allOf {
   435                      not { buildingTag() }
   436                      expression {SKIP_TRIGGERED_TESTS == false}
   437                      anyOf {
   438                          branch 'master';
   439                          branch 'release-*';
   440                          expression {params.TRIGGER_FULL_TESTS == true};
   441                      }
   442                  }
   443              }
   444              steps {
   445                  script {
   446                      build job: "verrazzano-push-triggered-acceptance-tests/${BRANCH_NAME.replace("/", "%2F")}",
   447                          parameters: [
   448                              string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   449                              string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   450                              string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH)
   451                          ], wait: params.WAIT_FOR_TRIGGERED
   452                  }
   453              }
   454          }
   455          stage('Zip Build and Test') {
   456              // If the tests are clean and this is a release branch or GENERATE_TARBALL == true,
   457              // generate the Verrazzano full product zip and run the Private Registry tests.
   458              // Optionally push images to OCIR for scanning.
   459              when {
   460                  allOf {
   461                      not { buildingTag() }
   462                      expression {SKIP_TRIGGERED_TESTS == false}
   463                      anyOf {
   464                          expression{params.GENERATE_TARBALL == true};
   465                          expression{params.PUSH_TO_OCIR == true};
   466                      }
   467                  }
   468              }
   469              stages{
   470                  stage("Build Product Zip") {
   471                      steps {
   472                          script {
   473                              try {
   474                                  sh """
   475                                       echo "${OCR_CREDS_PSW}" | docker login -u ${OCR_CREDS_USR} ${OCR_REPO} --password-stdin
   476                                  """
   477                              } catch(error) {
   478                                  echo "OCIR docker login at ${OCIR_REPO} failed, retrying after sleep"
   479                                  retry(4) {
   480                                      sleep(30)
   481                                      sh """
   482                                          echo "${OCR_CREDS_PSW}" | docker login -u ${OCR_CREDS_USR} ${OCR_REPO} --password-stdin
   483                                      """
   484                                  }
   485                              }
   486                          }
   487                          script {
   488                              sh """
   489                                  ci/scripts/generate_vz_distribution.sh ${WORKSPACE} ${VERRAZZANO_DEV_VERSION} ${SHORT_COMMIT_HASH}
   490                              """
   491                          }
   492                      }
   493                  }
   494                  stage("Private Registry Test") {
   495                      steps {
   496                          script {
   497                              tarfilePrefix="verrazzano-${VERRAZZANO_DEV_VERSION}-lite"
   498                              storeLocation="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${tarfilePrefix}.zip"
   499                              echo "Starting private registry test for ${storeLocation}, file prefix ${tarfilePrefix}"
   500                              build job: "verrazzano-private-registry/${BRANCH_NAME.replace("/", "%2F")}",
   501                                  parameters: [
   502                                      string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   503                                      string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   504                                      string(name: 'DISTRIBUTION_VARIANT', value: 'Lite'),
   505                                      string(name: 'ZIPFILE_LOCATION', value: storeLocation)
   506                                  ], wait: true
   507                          }
   508                      }
   509                  }
   510                  stage("Push Images to OCIR") {
   511                      environment {
   512                          OCI_CLI_AUTH="api_key"
   513                          OCI_CLI_TENANCY = credentials('oci-dev-tenancy')
   514                          OCI_CLI_USER = credentials('oci-dev-user-ocid')
   515                          OCI_CLI_FINGERPRINT = credentials('oci-dev-api-key-fingerprint')
   516                          OCI_CLI_KEY_FILE = credentials('oci-dev-api-key-file')
   517                          OCI_CLI_REGION = "us-ashburn-1"
   518                          OCI_REGION = "${env.OCI_CLI_REGION}"
   519                          // Directory containing the Verrazzano image tar files
   520                          VERRAZZANO_IMAGES_DIRECTORY = "${WORKSPACE}/vz-full/verrazzano-${VERRAZZANO_DEV_VERSION}/images"
   521                      }
   522                      when {
   523                          expression{params.PUSH_TO_OCIR == true}
   524                      }
   525                      steps {
   526                          script {
   527                              try {
   528                                  sh """
   529                                      echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin
   530                                  """
   531                              } catch(error) {
   532                                  echo "docker login failed, retrying after sleep"
   533                                  retry(4) {
   534                                      sleep(30)
   535                                      sh """
   536                                      echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin
   537                                      """
   538                                  }
   539                              }
   540  
   541                              sh """
   542                                  echo "Pushing images to OCIR, note that images pushed using this pipeline are NOT treated as the latest scan results, those come from periodic test runs"
   543                                  ci/scripts/push_to_ocir.sh
   544                              """
   545                          }
   546                      }
   547                  }
   548              }
   549          }
   550      }
   551  
   552      post {
   553          always {
   554              archiveArtifacts artifacts: '**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*cluster-snapshot*/**', allowEmptyArchive: true
   555              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   556          }
   557          failure {
   558              script {
   559                  failedOrAborted()
   560              }
   561          }
   562          aborted {
   563              script {
   564                  failedOrAborted()
   565              }
   566          }
   567          cleanup {
   568              deleteDir()
   569          }
   570      }
   571  }
   572  
   573  def failedOrAborted() {
   574      sh """
   575          curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   576      """
   577      archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   578      sh """
   579          curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   580          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   581          rm archive.zip
   582      """
   583      script {
   584          if (isPagerDutyEnabled() && (env.JOB_NAME == "verrazzano/master" || env.JOB_NAME ==~ "verrazzano/release-1.*")) {
   585              pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", incDescription: "Verrazzano: ${env.JOB_NAME} - Failed", incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}")
   586          }
   587          if (env.JOB_NAME == "verrazzano/master" || env.JOB_NAME ==~ "verrazzano/release-1.*" || env.BRANCH_NAME ==~ "mark/*") {
   588              slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}" )
   589          }
   590      }
   591  }
   592  
   593  def isPagerDutyEnabled() {
   594      // this controls whether PD alerts are enabled
   595      if (NOTIFY_PAGERDUTY_MAINJOB_FAILURES.equals("true")) {
   596          echo "Pager-Duty notifications enabled via global override setting"
   597          return true
   598      }
   599      return false
   600  }
   601  
   602  // Called in Stage Clean workspace and checkout steps
   603  def moveContentToGoRepoPath() {
   604      sh """
   605          rm -rf ${GO_REPO_PATH}/verrazzano
   606          mkdir -p ${GO_REPO_PATH}/verrazzano
   607          tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   608      """
   609  }
   610  
   611  // Download the file containing the base image name and digest
   612  def downloadBaseImageInfoFile() {
   613      sh """
   614          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name verrazzano-base-images/${BASE_IMAGE_INFO_FILE} --file ${WORKSPACE}/${BASE_IMAGE_INFO_FILE}
   615      """
   616  }
   617  
   618  // Called in Stage CLI steps
   619  def buildVerrazzanoCLI(dockerImageTag) {
   620      sh """
   621          cd ${GO_REPO_PATH}/verrazzano/tools/vz
   622          make go-build DOCKER_IMAGE_TAG=${dockerImageTag}
   623          ${GO_REPO_PATH}/verrazzano/ci/scripts/save_tooling.sh ${env.BRANCH_NAME} ${SHORT_COMMIT_HASH}
   624          cp out/linux_amd64/vz ${GO_REPO_PATH}/vz
   625      """
   626  }
   627  
   628  def checkRepoClean() {
   629      sh """
   630          cd ${GO_REPO_PATH}/verrazzano
   631          echo 'Check for forgotten manifest/generate actions...'
   632          (cd platform-operator; make check-repo-clean)
   633          (cd application-operator; make check-repo-clean)
   634          (cd cluster-operator; make check-repo-clean)
   635      """
   636  }
   637  
   638  // Called in Stage Build steps
   639  // Makes target docker push for application/platform operator and analysis
   640  def buildImages(dockerImageTag) {
   641      sh """
   642          cd ${GO_REPO_PATH}/verrazzano
   643          echo 'Now build using ${env.VZ_BASE_IMAGE} as base image...'
   644          make docker-push VERRAZZANO_PLATFORM_OPERATOR_IMAGE_NAME=${DOCKER_PLATFORM_IMAGE_NAME} VERRAZZANO_APPLICATION_OPERATOR_IMAGE_NAME=${DOCKER_APP_IMAGE_NAME} VERRAZZANO_CLUSTER_OPERATOR_IMAGE_NAME=${DOCKER_CLUSTER_IMAGE_NAME} VERRAZZANO_AUTHPROXY_IMAGE_NAME=${DOCKER_AUTHPROXY_IMAGE_NAME} DOCKER_REPO=${env.DOCKER_REPO} DOCKER_NAMESPACE=${env.DOCKER_NAMESPACE} DOCKER_IMAGE_TAG=${dockerImageTag} CREATE_LATEST_TAG=${CREATE_LATEST_TAG}
   645          cp ${GO_REPO_PATH}/verrazzano/platform-operator/out/generated-verrazzano-bom.json $WORKSPACE/generated-verrazzano-bom.json
   646          cp $WORKSPACE/generated-verrazzano-bom.json $WORKSPACE/verrazzano-bom.json
   647          ${GO_REPO_PATH}/verrazzano/tools/scripts/generate_image_list.sh $WORKSPACE/generated-verrazzano-bom.json $WORKSPACE/verrazzano_images.txt
   648      """
   649  }
   650  
   651  // Called in Stage Generate operator.yaml steps
   652  def generateOperatorYaml(dockerImageTag) {
   653      sh """
   654          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   655          case "${env.BRANCH_NAME}" in
   656              master|release-*)
   657                  ;;
   658              *)
   659                  echo "Adding image pull secrets to operator.yaml for non master/release branch"
   660                  export IMAGE_PULL_SECRETS=verrazzano-container-registry
   661          esac
   662          DOCKER_IMAGE_NAME=${DOCKER_PLATFORM_IMAGE_NAME} VERRAZZANO_APPLICATION_OPERATOR_IMAGE_NAME=${DOCKER_APP_IMAGE_NAME} VERRAZZANO_CLUSTER_OPERATOR_IMAGE_NAME=${DOCKER_CLUSTER_IMAGE_NAME} VERRAZZANO_AUTHPROXY_IMAGE_NAME=${DOCKER_AUTHPROXY_IMAGE_NAME} DOCKER_REPO=${env.DOCKER_REPO} DOCKER_NAMESPACE=${env.DOCKER_NAMESPACE} DOCKER_IMAGE_TAG=${dockerImageTag} OPERATOR_YAML=$WORKSPACE/generated-operator.yaml make generate-operator-yaml
   663      """
   664  }
   665  
   666  // Called in Stage Save Generated Files steps
   667  def saveGeneratedFiles() {
   668      sh """
   669          cd ${GO_REPO_PATH}/verrazzano
   670          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${env.BRANCH_NAME}/operator.yaml --file $WORKSPACE/generated-operator.yaml
   671          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file $WORKSPACE/generated-operator.yaml
   672          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${env.BRANCH_NAME}/generated-verrazzano-bom.json --file $WORKSPACE/generated-verrazzano-bom.json
   673          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/generated-verrazzano-bom.json --file $WORKSPACE/generated-verrazzano-bom.json
   674      """
   675  }
   676  
   677  def saveCLIExecutable() {
   678      sh """
   679          cd ${GO_REPO_PATH}/verrazzano
   680          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${env.BRANCH_NAME}/vz-linux-amd64.tar.gz --file $WORKSPACE/vz-linux-amd64.tar.gz
   681          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/vz-linux-amd64.tar.gz --file $WORKSPACE/vz-linux-amd64.tar.gz
   682          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${env.BRANCH_NAME}/vz-linux-amd64.tar.gz.sha256 --file $WORKSPACE/vz-linux-amd64.tar.gz.sha256
   683          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/vz-linux-amd64.tar.gz.sha256 --file $WORKSPACE/vz-linux-amd64.tar.gz.sha256
   684      """
   685  }
   686  
   687  // Called in many parallel stages of Stage Run Acceptance Tests
   688  def runGinkgoRandomize(testSuitePath) {
   689      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   690          sh """
   691              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   692              ginkgo -p --randomize-all -v --keep-going --no-color ${testSuitePath}/...
   693              ../../build/copy-junit-output.sh ${WORKSPACE}
   694          """
   695      }
   696  }
   697  
   698  // Called in many parallel stages of Stage Run Acceptance Tests
   699  def runGinkgo(testSuitePath) {
   700      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   701          sh """
   702              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   703              ginkgo -v --keep-going --no-color ${testSuitePath}/...
   704              ../../build/copy-junit-output.sh ${WORKSPACE}
   705          """
   706      }
   707  }
   708  
   709  // Called in Stage Acceptance Tests post
   710  def dumpK8sCluster(dumpDirectory) {
   711      sh """
   712          mkdir -p ${dumpDirectory}/cluster-snapshot
   713          ${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh -d ${dumpDirectory}/cluster-snapshot -r ${dumpDirectory}/cluster-snapshot/analysis.report
   714          ${GO_REPO_PATH}/vz bug-report --report-file ${dumpDirectory}/bug-report.tar.gz
   715          mkdir -p ${dumpDirectory}/bug-report
   716          tar -xvf ${dumpDirectory}/bug-report.tar.gz -C ${dumpDirectory}/bug-report
   717      """
   718  }
   719  
   720  
   721  // Called in Stage Clean workspace and checkout steps
   722  @NonCPS
   723  def getCommitList() {
   724      echo "Checking for change sets"
   725      def commitList = []
   726      def changeSets = currentBuild.changeSets
   727      for (int i = 0; i < changeSets.size(); i++) {
   728          echo "get commits from change set"
   729          def commits = changeSets[i].items
   730          for (int j = 0; j < commits.length; j++) {
   731              def commit = commits[j]
   732              def id = commit.commitId
   733              echo "Add commit id: ${id}"
   734              commitList.add(id)
   735          }
   736      }
   737      return commitList
   738  }
   739  
   740  def trimIfGithubNoreplyUser(userIn) {
   741      if (userIn == null) {
   742          echo "Not a github noreply user, not trimming: ${userIn}"
   743          return userIn
   744      }
   745      if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) {
   746          def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@"))
   747          return userOut;
   748      }
   749      if (userIn.matches(".*<.*@users.noreply.github.com.*")) {
   750          def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@"))
   751          return userOut;
   752      }
   753      if (userIn.matches(".*@users.noreply.github.com")) {
   754          def userOut = userIn.substring(0, userIn.indexOf("@"))
   755          return userOut;
   756      }
   757      echo "Not a github noreply user, not trimming: ${userIn}"
   758      return userIn
   759  }
   760  
   761  def getSuspectList(commitList, userMappings) {
   762      def retValue = ""
   763      def suspectList = []
   764      if (commitList == null || commitList.size() == 0) {
   765          echo "No commits to form suspect list"
   766      } else {
   767          for (int i = 0; i < commitList.size(); i++) {
   768              def id = commitList[i]
   769              try {
   770                  def gitAuthor = sh(
   771                      script: "git log --format='%ae' '$id^!'",
   772                      returnStdout: true
   773                  ).trim()
   774                  if (gitAuthor != null) {
   775                      def author = trimIfGithubNoreplyUser(gitAuthor)
   776                      echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}"
   777                      if (userMappings.containsKey(author)) {
   778                          def slackUser = userMappings.get(author)
   779                          if (!suspectList.contains(slackUser)) {
   780                              echo "Added ${slackUser} as suspect"
   781                              retValue += " ${slackUser}"
   782                              suspectList.add(slackUser)
   783                          }
   784                      } else {
   785                          // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated
   786                          if (!suspectList.contains(author)) {
   787                              echo "Added ${author} as suspect"
   788                              retValue += " ${author}"
   789                              suspectList.add(author)
   790                          }
   791                      }
   792                  } else {
   793                      echo "No author returned from git"
   794                  }
   795              } catch (Exception e) {
   796                  echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString()
   797              }
   798          }
   799      }
   800      def startedByUser = "";
   801      def causes = currentBuild.getBuildCauses()
   802      echo "causes: " + causes.toString()
   803      for (cause in causes) {
   804          def causeString = cause.toString()
   805          echo "current cause: " + causeString
   806          def causeInfo = readJSON text: causeString
   807          if (causeInfo.userId != null) {
   808              startedByUser = causeInfo.userId
   809          }
   810      }
   811  
   812      if (startedByUser.length() > 0) {
   813          echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}"
   814          def author = trimIfGithubNoreplyUser(startedByUser)
   815          echo "DEBUG: author: ${startedByUser}, ${author}"
   816          if (userMappings.containsKey(author)) {
   817              def slackUser = userMappings.get(author)
   818              if (!suspectList.contains(slackUser)) {
   819                  echo "Added ${slackUser} as suspect"
   820                  retValue += " ${slackUser}"
   821                  suspectList.add(slackUser)
   822              }
   823          } else {
   824              // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated
   825              if (!suspectList.contains(author)) {
   826                 echo "Added ${author} as suspect"
   827                 retValue += " ${author}"
   828                 suspectList.add(author)
   829              }
   830          }
   831      } else {
   832          echo "Build not started by a user, not adding to notification list"
   833      }
   834      echo "returning suspect list: ${retValue}"
   835      return retValue
   836  }