github.com/verrazzano/verrazzano@v1.7.0/Jenkinsfile (about)

     1  // Copyright (c) 2020, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  import groovy.transform.Field
     5  
     6  def DOCKER_IMAGE_TAG
     7  def SKIP_ACCEPTANCE_TESTS = false
     8  def SKIP_TRIGGERED_TESTS = false
     9  @Field
    10  def SUSPECT_LIST = ""
    11  def VERRAZZANO_DEV_VERSION = ""
    12  def VZ_BASE_IMAGE = ""
    13  def tarfilePrefix=""
    14  def storeLocation=""
    15  
    16  def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large"
    17  
    18  pipeline {
    19      options {
    20          skipDefaultCheckout true
    21          copyArtifactPermission('*');
    22          timestamps ()
    23      }
    24  
    25      agent {
    26         docker {
    27              image "${RUNNER_DOCKER_IMAGE}"
    28              args "${RUNNER_DOCKER_ARGS}"
    29              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    30              registryCredentialsId 'ocir-pull-and-push-account'
    31              label "${agentLabel}"
    32          }
    33      }
    34  
    35      parameters {
    36          booleanParam (description: 'Whether to kick off acceptance test run at the end of this build', name: 'RUN_ACCEPTANCE_TESTS', defaultValue: true)
    37          booleanParam (description: 'Whether to include the slow tests in the acceptance tests', name: 'RUN_SLOW_TESTS', defaultValue: false)
    38          booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true)
    39          booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false)
    40          booleanParam (description: 'Whether to trigger full testing after a successful run. Off by default. This is always done for successful master and release* builds, this setting only is used to enable the trigger for other branches', name: 'TRIGGER_FULL_TESTS', defaultValue: false)
    41          booleanParam (description: 'Whether to generate a tarball', name: 'GENERATE_TARBALL', defaultValue: false)
    42          booleanParam (description: 'Whether to push images to OCIR', name: 'PUSH_TO_OCIR', defaultValue: false)
    43          booleanParam (description: 'Whether to fail the Integration Tests to test failure handling', name: 'SIMULATE_FAILURE', defaultValue: false)
    44          booleanParam (description: 'Whether to perform a scan of the built images', name: 'PERFORM_SCAN', defaultValue: false)
    45          booleanParam (description: 'Whether to wait for triggered tests or not. This defaults to false, this setting is useful for things like release automation that require everything to complete successfully', name: 'WAIT_FOR_TRIGGERED', defaultValue: false)
    46          booleanParam (description: 'Whether to fail build if UT coverage number decreases lower than its release-* coverage from object storage. This defaults to false, meaning Any non release-*/master branch will WARN only if its coverage is lower. This can be enabled so that jobs will FAIL if coverage drops, for example when testing a PR before merging.', name: 'FAIL_IF_COVERAGE_DECREASED', defaultValue: false)
    47          booleanParam (description: 'Whether to write the UT coverage number to object storage. This always occurs for release-*/master branches. Defaults to true, but it can be disabled to not always upload.', name: 'UPLOAD_UNIT_TEST_COVERAGE', defaultValue: true)
    48          booleanParam (description: 'Whether to run clusterAPI override tests', name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', defaultValue: false)
    49          choice (name: 'WILDCARD_DNS_DOMAIN',
    50                  description: 'Wildcard DNS Domain',
    51                  // 1st choice is the default value
    52                  choices: [ "nip.io", "sslip.io"])
    53          choice (name: 'CRD_API_VERSION',
    54                  description: 'This is the API crd version.',
    55                  // 1st choice is the default value
    56                  choices: [ "v1beta1", "v1alpha1"])
    57          string (name: 'CONSOLE_REPO_BRANCH',
    58                  defaultValue: '',
    59                  description: 'The branch to check out after cloning the console repository.',
    60                  trim: true)
    61      }
    62  
    63      environment {
    64          TEST_ENV = "JENKINS"
    65          CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}"
    66          IS_PERIODIC_PIPELINE = "false"
    67  
    68          DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins'
    69          DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator'
    70          DOCKER_PLATFORM_IMAGE_NAME = "${env.BRANCH_NAME ==~ /^release-.*/ || env.BRANCH_NAME == 'master' ? env.DOCKER_PLATFORM_PUBLISH_IMAGE_NAME : env.DOCKER_PLATFORM_CI_IMAGE_NAME}"
    71          DOCKER_APP_CI_IMAGE_NAME = 'verrazzano-application-operator-jenkins'
    72          DOCKER_APP_PUBLISH_IMAGE_NAME = 'verrazzano-application-operator'
    73          DOCKER_APP_IMAGE_NAME = "${env.BRANCH_NAME ==~ /^release-.*/ || env.BRANCH_NAME == 'master' ? env.DOCKER_APP_PUBLISH_IMAGE_NAME : env.DOCKER_APP_CI_IMAGE_NAME}"
    74          DOCKER_CLUSTER_CI_IMAGE_NAME = 'verrazzano-cluster-operator-jenkins'
    75          DOCKER_CLUSTER_PUBLISH_IMAGE_NAME = 'verrazzano-cluster-operator'
    76          DOCKER_CLUSTER_IMAGE_NAME = "${env.BRANCH_NAME ==~ /^release-.*/ || env.BRANCH_NAME == 'master' ? env.DOCKER_CLUSTER_PUBLISH_IMAGE_NAME : env.DOCKER_CLUSTER_CI_IMAGE_NAME}"
    77          DOCKER_AUTHPROXY_CI_IMAGE_NAME = 'verrazzano-authproxy-jenkins'
    78          DOCKER_AUTHPROXY_PUBLISH_IMAGE_NAME = 'verrazzano-authproxy'
    79          DOCKER_AUTHPROXY_IMAGE_NAME = "${env.BRANCH_NAME ==~ /^release-.*/ || env.BRANCH_NAME == 'master' ? env.DOCKER_AUTHPROXY_PUBLISH_IMAGE_NAME : env.DOCKER_AUTHPROXY_CI_IMAGE_NAME}"
    80          CREATE_LATEST_TAG = "${env.BRANCH_NAME == 'master' ? '1' : '0'}"
    81          USE_V8O_DOC_STAGE = "${env.BRANCH_NAME == 'master' ? 'true' : 'false'}"
    82          GOPATH = '/home/opc/go'
    83          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    84          DOCKER_CREDS = credentials('github-packages-credentials-rw')
    85          DOCKER_EMAIL = credentials('github-packages-email')
    86          DOCKER_REPO = 'ghcr.io'
    87          DOCKER_NAMESPACE = 'verrazzano'
    88          NETRC_FILE = credentials('netrc')
    89          GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw')
    90          SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY')
    91          RELEASE_OWNERS = credentials('release-version-owners')
    92  
    93          CLUSTER_NAME = 'verrazzano'
    94          POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp"
    95          TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp"
    96          KUBECONFIG = "${WORKSPACE}/test_kubeconfig"
    97          VERRAZZANO_KUBECONFIG = "${KUBECONFIG}"
    98          OCR_CREDS = credentials('ocr-pull-and-push-account')
    99          OCR_REPO = 'container-registry.oracle.com'
   100          IMAGE_PULL_SECRET = 'verrazzano-container-registry'
   101          INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml"
   102          INSTALL_PROFILE = "dev"
   103          VZ_ENVIRONMENT_NAME = "default"
   104          TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts"
   105  
   106          WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test
   107          DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test
   108  
   109          // used for console artifact capture on failure
   110          JENKINS_READ = credentials('jenkins-auditor')
   111  
   112          OCI_CLI_AUTH="instance_principal"
   113          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
   114          OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts"
   115          OCI_OS_BUCKET="verrazzano-builds"
   116          OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
   117          OCI_OS_REGION="us-phoenix-1" // where to download existing artifacts from
   118          OCI_OS_DIST_REGION="eu-frankfurt-1" // where to upload distributions to
   119  
   120          // used to emit metrics
   121          PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials')
   122  
   123          OCIR_SCAN_COMPARTMENT = credentials('ocir-scan-compartment')
   124          OCIR_SCAN_TARGET = credentials('ocir-scan-target')
   125          OCIR_SCAN_REGISTRY = credentials('ocir-scan-registry')
   126          OCIR_SCAN_REPOSITORY_PATH = credentials('ocir-scan-repository-path')
   127          DOCKER_SCAN_CREDS = credentials('v8odev-ocir')
   128  
   129          // used to write to object storage, or fail build if UT coverage does not pass
   130          FAIL_BUILD_COVERAGE = "${params.FAIL_IF_COVERAGE_DECREASED}"
   131          UPLOAD_UT_COVERAGE = "${params.UPLOAD_UNIT_TEST_COVERAGE}"
   132  
   133  	// File containing base image information
   134  	BASE_IMAGE_INFO_FILE = "base-image-v1.0.0.txt"
   135      }
   136  
   137      stages {
   138          stage('Clean workspace and checkout') {
   139              steps {
   140                  sh """
   141                      echo "${NODE_LABELS}"
   142                  """
   143  
   144                  script {
   145                      def scmInfo = checkout scm
   146                      env.GIT_COMMIT = scmInfo.GIT_COMMIT
   147                      env.GIT_BRANCH = scmInfo.GIT_BRANCH
   148                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   149                  }
   150                  sh """
   151                      cp -f "${NETRC_FILE}" $HOME/.netrc
   152                      chmod 600 $HOME/.netrc
   153                  """
   154  
   155                  script {
   156                      try {
   157                          sh """
   158                              echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   159                          """
   160                      } catch(error) {
   161                          echo "docker login failed, retrying after sleep"
   162                          retry(4) {
   163                              sleep(30)
   164                              sh """
   165                              echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
   166                              """
   167                          }
   168                      }
   169                  }
   170                  moveContentToGoRepoPath()
   171  		downloadBaseImageInfoFile()
   172  
   173                  script {
   174                      def props = readProperties file: '.verrazzano-development-version'
   175                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   176                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   177                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "echo $env.GIT_COMMIT | head -c 8")
   178                      env.VERRAZZANO_VERSION = "${VERRAZZANO_DEV_VERSION}"
   179                      if (!"${env.GIT_BRANCH}".startsWith("release-")) {
   180                          env.VERRAZZANO_VERSION = "${env.VERRAZZANO_VERSION}-${env.BUILD_NUMBER}+${SHORT_COMMIT_HASH}"
   181                      }
   182                      DOCKER_IMAGE_TAG = "v${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}"
   183                      // update the description with some meaningful info
   184                      currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT
   185                      def currentCommitHash = env.GIT_COMMIT
   186                      def commitList = getCommitList()
   187                      withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) {
   188                          def userMappings = readJSON file: JENKINS_TO_SLACK_JSON
   189                          SUSPECT_LIST = getSuspectList(commitList, userMappings)
   190                          echo "Suspect list: ${SUSPECT_LIST}"
   191                      }
   192  
   193                      def imageProps = readProperties file: "${WORKSPACE}/${BASE_IMAGE_INFO_FILE}"
   194                      VZ_BASE_IMAGE = imageProps['base-image']
   195                      env.VZ_BASE_IMAGE = "${VZ_BASE_IMAGE}"
   196                      echo "Verrazzano base image: ${VZ_BASE_IMAGE}"
   197                  }
   198              }
   199          }
   200  
   201          stage('Generate operator.yaml') {
   202              environment {
   203                  MODULE_INTEGRATION = "true"
   204              }
   205              when { not { buildingTag() } }
   206              steps {
   207                  generateOperatorYaml("${DOCKER_IMAGE_TAG}")
   208              }
   209              post {
   210                  failure {
   211                      script {
   212                          SKIP_TRIGGERED_TESTS = true
   213                      }
   214                  }
   215                  success {
   216                      archiveArtifacts artifacts: "generated-operator.yaml", allowEmptyArchive: true
   217                  }
   218              }
   219          }
   220  
   221          stage('Check Repo Clean') {
   222              steps {
   223                  checkRepoClean()
   224              }
   225              post {
   226                  failure {
   227                      script {
   228                          SKIP_TRIGGERED_TESTS = true
   229                      }
   230                  }
   231              }
   232          }
   233         stage('Verrazzano development version check') {
   234                      when { not { buildingTag() } }
   235                      steps {
   236                           script {
   237                              VERRAZZANO_DEV_VERSION_CHECK = sh (script: "git ls-remote --tags origin | grep -F ${VERRAZZANO_DEV_VERSION}", returnStatus: true )
   238                              if (VERRAZZANO_DEV_VERSION_CHECK == 0) {
   239                                  if (env.JOB_NAME ==~ "verrazzano/release-*") {
   240                                      slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nVZ Helper was not run, the Verrazzano Development Version ${VERRAZZANO_DEV_VERSION} matches a prior release\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nRelease Owners:\n ${RELEASE_OWNERS}\n")
   241                                  }
   242                              }
   243                           }
   244                      }
   245                      post {
   246                          failure {
   247                              script {
   248                                  SKIP_TRIGGERED_TESTS = true
   249                              }
   250                          }
   251                      }
   252                  }
   253  
   254          stage('Parallel Build, Test, and Compliance') {
   255              parallel {
   256                  stage('Build Verrazzano CLI and Save Binary') {
   257                      steps {
   258                          buildVerrazzanoCLI("${DOCKER_IMAGE_TAG}")
   259                      }
   260                      post {
   261                          failure {
   262                              script {
   263                                  SKIP_TRIGGERED_TESTS = true
   264                              }
   265                          }
   266                          success {
   267                              script {
   268                                  archiveArtifacts artifacts: '**/*.tar.gz*', allowEmptyArchive: true
   269                                  echo "Saving CLI Binary"
   270                                  saveCLIExecutable()
   271                              }
   272  
   273                          }
   274                      }
   275                  }
   276  
   277                  stage('Build Images and Save Generated Files') {
   278                      when { not { buildingTag() } }
   279                      steps {
   280                          script {
   281                              buildImages("${DOCKER_IMAGE_TAG}")
   282                          }
   283                      }
   284                      post {
   285                          failure {
   286                              script {
   287                                  SKIP_TRIGGERED_TESTS = true
   288                              }
   289                          }
   290                          success {
   291                              echo "Saving generated files"
   292                              saveGeneratedFiles()
   293                              script {
   294                                  archiveArtifacts artifacts: "generated-verrazzano-bom.json,verrazzano_images.txt", allowEmptyArchive: true
   295                              }
   296                          }
   297                      }
   298                  }
   299  
   300                  stage('Quality, Compliance Checks, and Unit Tests') {
   301                      when { not { buildingTag() } }
   302                      steps {
   303                          sh """
   304                      cd ${GO_REPO_PATH}/verrazzano
   305                      make precommit
   306                      make unit-test-coverage-ratcheting
   307                      echo "Checking versions..."
   308                      release/scripts/check_versions.sh ${VERRAZZANO_DEV_VERSION}
   309                  """
   310                      }
   311                      post {
   312                          failure {
   313                              script {
   314                                  SKIP_TRIGGERED_TESTS = true
   315                              }
   316                          }
   317                          always {
   318                              sh """
   319                          cd ${GO_REPO_PATH}/verrazzano
   320                          cp coverage.html ${WORKSPACE}
   321                          cp coverage.xml ${WORKSPACE}
   322                          build/copy-junit-output.sh ${WORKSPACE}
   323                      """
   324                              archiveArtifacts artifacts: '**/coverage.html', allowEmptyArchive: true
   325                              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   326                              cobertura(coberturaReportFile: 'coverage.xml',
   327                                      enableNewApi: true,
   328                                      autoUpdateHealth: false,
   329                                      autoUpdateStability: false,
   330                                      failUnstable: true,
   331                                      failUnhealthy: true,
   332                                      failNoReports: true,
   333                                      onlyStable: false,
   334                                      fileCoverageTargets: '100, 0, 0',
   335                                      lineCoverageTargets: '68, 68, 68',
   336                                      packageCoverageTargets: '100, 0, 0',
   337                              )
   338                          }
   339                      }
   340                  }
   341  
   342          }
   343      }
   344  
   345          stage('Scan Image') {
   346              when {
   347                 allOf {
   348                     not { buildingTag() }
   349                     expression {params.PERFORM_SCAN == true}
   350                 }
   351              }
   352              steps {
   353                  script {
   354                      scanContainerImage "${env.DOCKER_REPO}/${env.DOCKER_NAMESPACE}/${DOCKER_PLATFORM_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
   355                      scanContainerImage "${env.DOCKER_REPO}/${env.DOCKER_NAMESPACE}/${DOCKER_APP_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
   356                      scanContainerImage "${env.DOCKER_REPO}/${env.DOCKER_NAMESPACE}/${DOCKER_CLUSTER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
   357                      scanContainerImage "${env.DOCKER_REPO}/${env.DOCKER_NAMESPACE}/${DOCKER_AUTHPROXY_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
   358                  }
   359              }
   360              post {
   361                  failure {
   362                      script {
   363                          SKIP_TRIGGERED_TESTS = true
   364                      }
   365                  }
   366                  always {
   367                      archiveArtifacts artifacts: '**/scanning-report*.json', allowEmptyArchive: true
   368                  }
   369              }
   370          }
   371  
   372          stage('Skip acceptance tests if commit message contains skip-at') {
   373              steps {
   374                  script {
   375                      // note that SKIP_ACCEPTANCE_TESTS will be false at this point (its default value)
   376                      // so we are going to run the AT's unless this logic decides to skip them...
   377  
   378                      // if we are planning to run the AT's (which is the default)
   379                      if (params.RUN_ACCEPTANCE_TESTS == true) {
   380                          SKIP_ACCEPTANCE_TESTS = false
   381                          // check if the user has asked to skip AT using the commit message
   382                          result = sh (script: "git log -1 | grep 'skip-at'", returnStatus: true)
   383                          if (result == 0) {
   384                              // found 'skip-at', so don't run them
   385                              SKIP_ACCEPTANCE_TESTS = true
   386                              echo "Skip acceptance tests based on opt-out in commit message [skip-at]"
   387                              echo "SKIP_ACCEPTANCE_TESTS is ${SKIP_ACCEPTANCE_TESTS}"
   388                          }
   389                      } else {
   390                          SKIP_ACCEPTANCE_TESTS = true
   391                      }
   392                  }
   393              }
   394          }
   395  
   396          stage('Kind Acceptance Tests on 1.27') {
   397              when {
   398                  allOf {
   399                      not { buildingTag() }
   400                      anyOf {
   401                          branch 'master';
   402                          branch 'release-*';
   403                          expression {SKIP_ACCEPTANCE_TESTS == false};
   404                      }
   405                  }
   406              }
   407  
   408              environment {
   409                  SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
   410                  SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
   411                  SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
   412              }
   413  
   414              steps {
   415                  retry(count: JOB_PROMOTION_RETRIES) {
   416                      script {
   417                          build job: "verrazzano-new-kind-acceptance-tests/${BRANCH_NAME.replace("/", "%2F")}",
   418                              parameters: [
   419                                  string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'),
   420                                  string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   421                                  string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   422                                  string(name: 'CRD_API_VERSION', value: params.CRD_API_VERSION),
   423                                  booleanParam(name: 'RUN_SLOW_TESTS', value: params.RUN_SLOW_TESTS),
   424                                  booleanParam(name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', value: params.DUMP_K8S_CLUSTER_ON_SUCCESS),
   425                                  booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: params.CREATE_CLUSTER_USE_CALICO),
   426                                  booleanParam(name: 'RUN_CLUSTERAPI_OVERRIDE_TESTS', value: params.RUN_CLUSTERAPI_OVERRIDE_TESTS),
   427                                  string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH),
   428                              ], wait: true
   429                      }
   430                  }
   431              }
   432              post {
   433                  failure {
   434                      script {
   435                          SKIP_TRIGGERED_TESTS = true
   436                      }
   437                  }
   438              }
   439          }
   440          stage('Triggered Tests') {
   441              when {
   442                  allOf {
   443                      not { buildingTag() }
   444                      expression {SKIP_TRIGGERED_TESTS == false}
   445                      anyOf {
   446                          branch 'master';
   447                          branch 'release-*';
   448                          expression {params.TRIGGER_FULL_TESTS == true};
   449                      }
   450                  }
   451              }
   452              steps {
   453                  script {
   454                      build job: "verrazzano-push-triggered-acceptance-tests/${BRANCH_NAME.replace("/", "%2F")}",
   455                          parameters: [
   456                              string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   457                              string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   458                              string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH)
   459                          ], wait: params.WAIT_FOR_TRIGGERED
   460                  }
   461              }
   462          }
   463          stage('Zip Build and Test') {
   464              // If the tests are clean and this is a release branch or GENERATE_TARBALL == true,
   465              // generate the Verrazzano full product zip and run the Private Registry tests.
   466              // Optionally push images to OCIR for scanning.
   467              when {
   468                  allOf {
   469                      not { buildingTag() }
   470                      expression {SKIP_TRIGGERED_TESTS == false}
   471                      anyOf {
   472                          expression{params.GENERATE_TARBALL == true};
   473                          expression{params.PUSH_TO_OCIR == true};
   474                      }
   475                  }
   476              }
   477              stages{
   478                  stage("Build Product Zip") {
   479                      steps {
   480                          script {
   481                              try {
   482                                  sh """
   483                                       echo "${OCR_CREDS_PSW}" | docker login -u ${OCR_CREDS_USR} ${OCR_REPO} --password-stdin
   484                                  """
   485                              } catch(error) {
   486                                  echo "OCIR docker login at ${OCIR_REPO} failed, retrying after sleep"
   487                                  retry(4) {
   488                                      sleep(30)
   489                                      sh """
   490                                          echo "${OCR_CREDS_PSW}" | docker login -u ${OCR_CREDS_USR} ${OCR_REPO} --password-stdin
   491                                      """
   492                                  }
   493                              }
   494                          }
   495                          script {
   496                              sh """
   497                                  ci/scripts/generate_vz_distribution.sh ${WORKSPACE} ${VERRAZZANO_DEV_VERSION} ${SHORT_COMMIT_HASH}
   498                              """
   499                          }
   500                      }
   501                  }
   502                  stage("Private Registry Test") {
   503                      steps {
   504                          script {
   505                              tarfilePrefix="verrazzano-${VERRAZZANO_DEV_VERSION}-lite"
   506                              storeLocation="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${tarfilePrefix}.zip"
   507                              echo "Starting private registry test for ${storeLocation}, file prefix ${tarfilePrefix}"
   508                              build job: "verrazzano-private-registry/${BRANCH_NAME.replace("/", "%2F")}",
   509                                  parameters: [
   510                                      string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   511                                      string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   512                                      string(name: 'DISTRIBUTION_VARIANT', value: 'Lite'),
   513                                      string(name: 'ZIPFILE_LOCATION', value: storeLocation)
   514                                  ], wait: true
   515                          }
   516                      }
   517                  }
   518                  stage("Push Images to OCIR") {
   519                      environment {
   520                          OCI_CLI_AUTH="api_key"
   521                          OCI_CLI_TENANCY = credentials('oci-dev-tenancy')
   522                          OCI_CLI_USER = credentials('oci-dev-user-ocid')
   523                          OCI_CLI_FINGERPRINT = credentials('oci-dev-api-key-fingerprint')
   524                          OCI_CLI_KEY_FILE = credentials('oci-dev-api-key-file')
   525                          OCI_CLI_REGION = "us-ashburn-1"
   526                          OCI_REGION = "${env.OCI_CLI_REGION}"
   527                          // Directory containing the Verrazzano image tar files
   528                          VERRAZZANO_IMAGES_DIRECTORY = "${WORKSPACE}/vz-full/verrazzano-${VERRAZZANO_DEV_VERSION}/images"
   529                      }
   530                      when {
   531                          expression{params.PUSH_TO_OCIR == true}
   532                      }
   533                      steps {
   534                          script {
   535                              try {
   536                                  sh """
   537                                      echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin
   538                                  """
   539                              } catch(error) {
   540                                  echo "docker login failed, retrying after sleep"
   541                                  retry(4) {
   542                                      sleep(30)
   543                                      sh """
   544                                      echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin
   545                                      """
   546                                  }
   547                              }
   548  
   549                              sh """
   550                                  echo "Pushing images to OCIR, note that images pushed using this pipeline are NOT treated as the latest scan results, those come from periodic test runs"
   551                                  ci/scripts/push_to_ocir.sh
   552                              """
   553                          }
   554                      }
   555                  }
   556              }
   557          }
   558      }
   559  
   560      post {
   561          always {
   562              archiveArtifacts artifacts: '**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*cluster-snapshot*/**', allowEmptyArchive: true
   563              junit testResults: '**/*test-result.xml', allowEmptyResults: true
   564          }
   565          failure {
   566              script {
   567                  failedOrAborted()
   568              }
   569          }
   570          aborted {
   571              script {
   572                  failedOrAborted()
   573              }
   574          }
   575          cleanup {
   576              deleteDir()
   577          }
   578      }
   579  }
   580  
   581  def failedOrAborted() {
   582      sh """
   583          curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText
   584      """
   585      archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true
   586      sh """
   587          curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip
   588          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip
   589          rm archive.zip
   590      """
   591      script {
   592          if (isPagerDutyEnabled() && (env.JOB_NAME == "verrazzano/master" || env.JOB_NAME ==~ "verrazzano/release-1.*")) {
   593              pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", incDescription: "Verrazzano: ${env.JOB_NAME} - Failed", incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}")
   594          }
   595          if (env.JOB_NAME == "verrazzano/master" || env.JOB_NAME ==~ "verrazzano/release-1.*" || env.BRANCH_NAME ==~ "mark/*") {
   596              slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}" )
   597          }
   598      }
   599  }
   600  
   601  def isPagerDutyEnabled() {
   602      // this controls whether PD alerts are enabled
   603      if (NOTIFY_PAGERDUTY_MAINJOB_FAILURES.equals("true")) {
   604          echo "Pager-Duty notifications enabled via global override setting"
   605          return true
   606      }
   607      return false
   608  }
   609  
   610  // Called in Stage Clean workspace and checkout steps
   611  def moveContentToGoRepoPath() {
   612      sh """
   613          rm -rf ${GO_REPO_PATH}/verrazzano
   614          mkdir -p ${GO_REPO_PATH}/verrazzano
   615          tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -)
   616      """
   617  }
   618  
   619  // Download the file containing the base image name and digest
   620  def downloadBaseImageInfoFile() {
   621      sh """
   622          oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name verrazzano-base-images/${BASE_IMAGE_INFO_FILE} --file ${WORKSPACE}/${BASE_IMAGE_INFO_FILE}
   623      """
   624  }
   625  
   626  // Called in Stage CLI steps
   627  def buildVerrazzanoCLI(dockerImageTag) {
   628      sh """
   629          cd ${GO_REPO_PATH}/verrazzano/tools/vz
   630          make go-build DOCKER_IMAGE_TAG=${dockerImageTag}
   631          ${GO_REPO_PATH}/verrazzano/ci/scripts/save_tooling.sh ${env.BRANCH_NAME} ${SHORT_COMMIT_HASH}
   632          cp out/linux_amd64/vz ${GO_REPO_PATH}/vz
   633      """
   634  }
   635  
   636  def checkRepoClean() {
   637      sh """
   638          cd ${GO_REPO_PATH}/verrazzano
   639          echo 'Check for forgotten manifest/generate actions...'
   640          (cd platform-operator; make check-repo-clean)
   641          (cd application-operator; make check-repo-clean)
   642          (cd cluster-operator; make check-repo-clean)
   643      """
   644  }
   645  
   646  // Called in Stage Build steps
   647  // Makes target docker push for application/platform operator and analysis
   648  def buildImages(dockerImageTag) {
   649      sh """
   650          cd ${GO_REPO_PATH}/verrazzano
   651          echo 'Now build using ${env.VZ_BASE_IMAGE} as base image...'
   652          make docker-push VERRAZZANO_PLATFORM_OPERATOR_IMAGE_NAME=${DOCKER_PLATFORM_IMAGE_NAME} VERRAZZANO_APPLICATION_OPERATOR_IMAGE_NAME=${DOCKER_APP_IMAGE_NAME} VERRAZZANO_CLUSTER_OPERATOR_IMAGE_NAME=${DOCKER_CLUSTER_IMAGE_NAME} VERRAZZANO_AUTHPROXY_IMAGE_NAME=${DOCKER_AUTHPROXY_IMAGE_NAME} DOCKER_REPO=${env.DOCKER_REPO} DOCKER_NAMESPACE=${env.DOCKER_NAMESPACE} DOCKER_IMAGE_TAG=${dockerImageTag} CREATE_LATEST_TAG=${CREATE_LATEST_TAG}
   653          cp ${GO_REPO_PATH}/verrazzano/platform-operator/out/generated-verrazzano-bom.json $WORKSPACE/generated-verrazzano-bom.json
   654          cp $WORKSPACE/generated-verrazzano-bom.json $WORKSPACE/verrazzano-bom.json
   655          ${GO_REPO_PATH}/verrazzano/tools/scripts/generate_image_list.sh $WORKSPACE/generated-verrazzano-bom.json $WORKSPACE/verrazzano_images.txt
   656      """
   657  }
   658  
   659  // Called in Stage Generate operator.yaml steps
   660  def generateOperatorYaml(dockerImageTag) {
   661      sh """
   662          cd ${GO_REPO_PATH}/verrazzano/platform-operator
   663          case "${env.BRANCH_NAME}" in
   664              master|release-*)
   665                  ;;
   666              *)
   667                  echo "Adding image pull secrets to operator.yaml for non master/release branch"
   668                  export IMAGE_PULL_SECRETS=verrazzano-container-registry
   669          esac
   670          DOCKER_IMAGE_NAME=${DOCKER_PLATFORM_IMAGE_NAME} VERRAZZANO_APPLICATION_OPERATOR_IMAGE_NAME=${DOCKER_APP_IMAGE_NAME} VERRAZZANO_CLUSTER_OPERATOR_IMAGE_NAME=${DOCKER_CLUSTER_IMAGE_NAME} VERRAZZANO_AUTHPROXY_IMAGE_NAME=${DOCKER_AUTHPROXY_IMAGE_NAME} DOCKER_REPO=${env.DOCKER_REPO} DOCKER_NAMESPACE=${env.DOCKER_NAMESPACE} DOCKER_IMAGE_TAG=${dockerImageTag} OPERATOR_YAML=$WORKSPACE/generated-operator.yaml make generate-operator-yaml
   671      """
   672  }
   673  
   674  // Called in Stage Save Generated Files steps
   675  def saveGeneratedFiles() {
   676      sh """
   677          cd ${GO_REPO_PATH}/verrazzano
   678          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${env.BRANCH_NAME}/operator.yaml --file $WORKSPACE/generated-operator.yaml
   679          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file $WORKSPACE/generated-operator.yaml
   680          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${env.BRANCH_NAME}/generated-verrazzano-bom.json --file $WORKSPACE/generated-verrazzano-bom.json
   681          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/generated-verrazzano-bom.json --file $WORKSPACE/generated-verrazzano-bom.json
   682      """
   683  }
   684  
   685  def saveCLIExecutable() {
   686      sh """
   687          cd ${GO_REPO_PATH}/verrazzano
   688          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${env.BRANCH_NAME}/vz-linux-amd64.tar.gz --file $WORKSPACE/vz-linux-amd64.tar.gz
   689          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/vz-linux-amd64.tar.gz --file $WORKSPACE/vz-linux-amd64.tar.gz
   690          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${env.BRANCH_NAME}/vz-linux-amd64.tar.gz.sha256 --file $WORKSPACE/vz-linux-amd64.tar.gz.sha256
   691          oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/vz-linux-amd64.tar.gz.sha256 --file $WORKSPACE/vz-linux-amd64.tar.gz.sha256
   692      """
   693  }
   694  
   695  // Called in many parallel stages of Stage Run Acceptance Tests
   696  def runGinkgoRandomize(testSuitePath) {
   697      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   698          sh """
   699              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   700              ginkgo -p --randomize-all -v --keep-going --no-color ${testSuitePath}/...
   701              ../../build/copy-junit-output.sh ${WORKSPACE}
   702          """
   703      }
   704  }
   705  
   706  // Called in many parallel stages of Stage Run Acceptance Tests
   707  def runGinkgo(testSuitePath) {
   708      catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
   709          sh """
   710              cd ${GO_REPO_PATH}/verrazzano/tests/e2e
   711              ginkgo -v --keep-going --no-color ${testSuitePath}/...
   712              ../../build/copy-junit-output.sh ${WORKSPACE}
   713          """
   714      }
   715  }
   716  
   717  // Called in Stage Acceptance Tests post
   718  def dumpK8sCluster(dumpDirectory) {
   719      sh """
   720          mkdir -p ${dumpDirectory}/cluster-snapshot
   721          ${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh -d ${dumpDirectory}/cluster-snapshot -r ${dumpDirectory}/cluster-snapshot/analysis.report
   722          ${GO_REPO_PATH}/vz bug-report --report-file ${dumpDirectory}/bug-report.tar.gz
   723          mkdir -p ${dumpDirectory}/bug-report
   724          tar -xvf ${dumpDirectory}/bug-report.tar.gz -C ${dumpDirectory}/bug-report
   725      """
   726  }
   727  
   728  
   729  // Called in Stage Clean workspace and checkout steps
   730  @NonCPS
   731  def getCommitList() {
   732      echo "Checking for change sets"
   733      def commitList = []
   734      def changeSets = currentBuild.changeSets
   735      for (int i = 0; i < changeSets.size(); i++) {
   736          echo "get commits from change set"
   737          def commits = changeSets[i].items
   738          for (int j = 0; j < commits.length; j++) {
   739              def commit = commits[j]
   740              def id = commit.commitId
   741              echo "Add commit id: ${id}"
   742              commitList.add(id)
   743          }
   744      }
   745      return commitList
   746  }
   747  
   748  def trimIfGithubNoreplyUser(userIn) {
   749      if (userIn == null) {
   750          echo "Not a github noreply user, not trimming: ${userIn}"
   751          return userIn
   752      }
   753      if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) {
   754          def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@"))
   755          return userOut;
   756      }
   757      if (userIn.matches(".*<.*@users.noreply.github.com.*")) {
   758          def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@"))
   759          return userOut;
   760      }
   761      if (userIn.matches(".*@users.noreply.github.com")) {
   762          def userOut = userIn.substring(0, userIn.indexOf("@"))
   763          return userOut;
   764      }
   765      echo "Not a github noreply user, not trimming: ${userIn}"
   766      return userIn
   767  }
   768  
   769  def getSuspectList(commitList, userMappings) {
   770      def retValue = ""
   771      def suspectList = []
   772      if (commitList == null || commitList.size() == 0) {
   773          echo "No commits to form suspect list"
   774      } else {
   775          for (int i = 0; i < commitList.size(); i++) {
   776              def id = commitList[i]
   777              try {
   778                  def gitAuthor = sh(
   779                      script: "git log --format='%ae' '$id^!'",
   780                      returnStdout: true
   781                  ).trim()
   782                  if (gitAuthor != null) {
   783                      def author = trimIfGithubNoreplyUser(gitAuthor)
   784                      echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}"
   785                      if (userMappings.containsKey(author)) {
   786                          def slackUser = userMappings.get(author)
   787                          if (!suspectList.contains(slackUser)) {
   788                              echo "Added ${slackUser} as suspect"
   789                              retValue += " ${slackUser}"
   790                              suspectList.add(slackUser)
   791                          }
   792                      } else {
   793                          // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated
   794                          if (!suspectList.contains(author)) {
   795                              echo "Added ${author} as suspect"
   796                              retValue += " ${author}"
   797                              suspectList.add(author)
   798                          }
   799                      }
   800                  } else {
   801                      echo "No author returned from git"
   802                  }
   803              } catch (Exception e) {
   804                  echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString()
   805              }
   806          }
   807      }
   808      def startedByUser = "";
   809      def causes = currentBuild.getBuildCauses()
   810      echo "causes: " + causes.toString()
   811      for (cause in causes) {
   812          def causeString = cause.toString()
   813          echo "current cause: " + causeString
   814          def causeInfo = readJSON text: causeString
   815          if (causeInfo.userId != null) {
   816              startedByUser = causeInfo.userId
   817          }
   818      }
   819  
   820      if (startedByUser.length() > 0) {
   821          echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}"
   822          def author = trimIfGithubNoreplyUser(startedByUser)
   823          echo "DEBUG: author: ${startedByUser}, ${author}"
   824          if (userMappings.containsKey(author)) {
   825              def slackUser = userMappings.get(author)
   826              if (!suspectList.contains(slackUser)) {
   827                  echo "Added ${slackUser} as suspect"
   828                  retValue += " ${slackUser}"
   829                  suspectList.add(slackUser)
   830              }
   831          } else {
   832              // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated
   833              if (!suspectList.contains(author)) {
   834                 echo "Added ${author} as suspect"
   835                 retValue += " ${author}"
   836                 suspectList.add(author)
   837              }
   838          }
   839      } else {
   840          echo "Build not started by a user, not adding to notification list"
   841      }
   842      echo "returning suspect list: ${retValue}"
   843      return retValue
   844  }