github.com/verrazzano/verrazzano@v1.7.0/ci/JenkinsfileTestTrigger (about)

     1  // Copyright (c) 2020, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  import groovy.transform.Field
     5  
     6  @Field
     7  def SUSPECT_LIST = ""
     8  
     9  pipeline {
    10      options {
    11          timeout(time: 5, unit: 'HOURS')
    12          skipDefaultCheckout true
    13      }
    14  
    15      agent {
    16         docker {
    17              image "${RUNNER_DOCKER_IMAGE}"
    18              args "${RUNNER_DOCKER_ARGS}"
    19              registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
    20              registryCredentialsId 'ocir-pull-and-push-account'
    21              label "pipeline-job-large"
    22          }
    23      }
    24  
    25      // Use cases:
    26      //        1) (automatic) master, release* will trigger this job by default on successful runs and supply GIT_COMMIT_TO_USE with the exact GIT commit to use for all testing (same as the upstream build)
    27      //        2) (automatic) branch builds with TRIGGER_FULL_TESTS enabled will trigger this job by default on successful runs and supply GIT_COMMIT_TO_USE with the exact GIT commit to use for all testing (same as the upstream build)
    28      //        3) (manual) any branch should be able to call this “trigger” job with a commit that was previously built and has a valid GIT commit hash. In this case the manual job
    29      //           must be started from the desired branch using Build with Parameters, the GIT_COMMIT_TO_USE must supply the GIT commit hash from the previous build, and VERRAZZANO_OPERATOR_IMAGE must NOT BE SPECIFIED or be NONE
    30      //           This allows one to rerun the tests without rerunning the upstream build (ie: if intermittent test issue occurred)
    31      //        4) (manual) any branch should be able to call this “trigger” job with the current head of their branch, pointing to a previously built operator image. This is useful if you are adding/fixing test cases where the
    32      //           operator image was built already (from master, or your branch) and only want to run the tests using it without running the full build. This is not a super likely situation (more likely individual test jobs
    33      //           will be manually kicked off rather than all of them). To accomplish this, specify GIT_COMMIT_TO_USE=NONE, and VERRAZZANO_OPERATOR_IMAGE=image-to-use
    34      parameters {
    35          string (name: 'GIT_COMMIT_TO_USE',
    36                          defaultValue: 'NONE',
    37                          description: 'This is the full git commit hash from the source build to be used for all jobs. A full pipeline specifies a valid commit hash here. NONE can be used for manually triggered jobs, however even for those a commit hash value is preferred to be supplied',
    38                          trim: true)
    39          choice (description: 'Predefined config permutations for Verrazzano installation used by upgrade path pipeline. Prod profile is the default profile for NONE', name: 'VZ_INSTALL_CONFIG',
    40                  choices: ["NONE", "dev-kind-persistence"])
    41          string (name: 'VERRAZZANO_OPERATOR_IMAGE',
    42                          defaultValue: 'NONE',
    43                          description: 'This is for manually testing only where someone needs to use a specific operator image, otherwise the default value of NONE is used',
    44                          trim: true)
    45          string (name: 'WILDCARD_DNS_DOMAIN',
    46                          defaultValue: 'nip.io',
    47                          description: 'This is the wildcard DNS domain',
    48                          trim: true)
    49          string (name: 'TAGGED_TESTS',
    50                  defaultValue: '',
    51                  description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
    52                  trim: true)
    53          string (name: 'INCLUDED_TESTS',
    54                  defaultValue: '.*',
    55                  description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
    56                  trim: true)
    57          string (name: 'EXCLUDED_TESTS',
    58                  defaultValue: '_excluded_test',
    59                  description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
    60                  trim: true)
    61          string (name: 'CONSOLE_REPO_BRANCH',
    62                  defaultValue: '',
    63                  description: 'The branch to check out after cloning the console repository.',
    64                  trim: true)
    65          booleanParam (description: 'This will FORCE a stable commit update without running any tests. This is normally ONLY used for debugging, it can be used to force a manual value into object storage', name: 'FORCE_STABLE_COMMIT_UPDATE', defaultValue: false)
    66      }
    67  
    68      environment {
    69          CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}"
    70          GOPATH = '/home/opc/go'
    71          GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano"
    72          OCI_CLI_AUTH="instance_principal"
    73          OCI_OS_NAMESPACE = credentials('oci-os-namespace')
    74          OCI_OS_BUCKET="verrazzano-builds"
    75          PROMETHEUS_GW_URL = credentials('prometheus-dev-url')
    76          SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY')
    77      }
    78  
    79      stages {
    80          stage('Clean workspace and checkout') {
    81              steps {
    82                  sh """
    83                      echo "${NODE_LABELS}"
    84                  """
    85  
    86                  // REVIEW: I'm not sure that we actually need to fetch the sources here, but I'm doing here as it was easier
    87                  // to test working with the SCM checkout settings starting from this job. We should be able to trigger this job
    88                  // with parameters directly (ie: based on a previous build), in that situation doing this gives us a single point
    89                  // to ensure the commit matches what was intended before triggering a bunch of downstream jobs that will
    90                  // all fail if it wasn't correct. So we may want to keep it here unless there is a compelling reason not to do so.
    91                  // I haven't looked at the executor resource usage yet in all of this, so it may be that could have constraints for
    92                  // using flyweight executors (still need to look at that)
    93                  script {
    94                      if (params.GIT_COMMIT_TO_USE == "NONE") {
    95                          echo "Specific GIT commit was not specified, use current head"
    96                          def scmInfo = checkout([
    97                              $class: 'GitSCM',
    98                              branches: [[name: env.BRANCH_NAME]],
    99                              doGenerateSubmoduleConfigurations: false,
   100                              extensions: [],
   101                              submoduleCfg: [],
   102                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   103                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   104                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   105                      } else {
   106                          echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}"
   107                          def scmInfo = checkout([
   108                              $class: 'GitSCM',
   109                              branches: [[name: params.GIT_COMMIT_TO_USE]],
   110                              doGenerateSubmoduleConfigurations: false,
   111                              extensions: [],
   112                              submoduleCfg: [],
   113                              userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
   114                          env.GIT_COMMIT = scmInfo.GIT_COMMIT
   115                          env.GIT_BRANCH = scmInfo.GIT_BRANCH
   116                          // If the commit we were handed is not what the SCM says we are using, fail
   117                          if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) {
   118                              echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}"
   119                              exit 1
   120                          }
   121                      }
   122                      echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
   123                  }
   124  
   125                  script {
   126                      echo "Generate git tags and save it to tags.txt file in the workspace"
   127                      sh """
   128                          cd ${workspace}
   129                          git tag | awk '/v1[.]/' >  tags.txt
   130                      """
   131                      def props = readProperties file: '.verrazzano-development-version'
   132                      VERRAZZANO_DEV_VERSION = props['verrazzano-development-version']
   133                      LATEST_RELEASE_VERSION = sh(returnStdout: true, script: "go run  ${WORKSPACE}/ci/tools/derive_upgrade_version.go ${workspace} latest-version-for-branch ${VERRAZZANO_DEV_VERSION}").trim()
   134                      TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
   135                      SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
   136                      // update the description with some meaningful info
   137                      currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE
   138                      def currentCommitHash = env.GIT_COMMIT
   139                      def commitList = getCommitList()
   140                      withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) {
   141                          def userMappings = readJSON file: JENKINS_TO_SLACK_JSON
   142                          SUSPECT_LIST = getSuspectList(commitList, userMappings)
   143                          echo "Suspect list: ${SUSPECT_LIST}"
   144                      }
   145                  }
   146              }
   147          }
   148  
   149          stage ('Kick off parallel tests') {
   150              when {
   151                  allOf {
   152                      not { buildingTag() }
   153                      expression {params.FORCE_STABLE_COMMIT_UPDATE == false}
   154                  }
   155              }
   156              parallel {
   157                  stage('Multi Cluster Tests') {
   158                      steps {
   159                          retry(count: JOB_PROMOTION_RETRIES) {
   160                              script {
   161                                  build job: "/verrazzano-multi-cluster-acceptance-tests/${CLEAN_BRANCH_NAME}",
   162                                      parameters: [
   163                                          string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   164                                          string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE),
   165                                          string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   166                                          string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS),
   167                                          string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS),
   168                                          string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS),
   169                                          string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH)
   170                                      ], wait: true
   171                              }
   172                          }
   173                      }
   174                  }
   175                  stage('Upgrade Path Tests with External ES') {
   176                      steps {
   177                          retry(count: JOB_PROMOTION_RETRIES) {
   178                              script {
   179                                  def latestRelease = LATEST_RELEASE_VERSION
   180                                  build job: "/verrazzano-upgrade-path-tests/${CLEAN_BRANCH_NAME}",
   181                                      parameters: [
   182                                          string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   183                                          booleanParam(name: 'EXTERNAL_ELASTICSEARCH', value: true),
   184                                          string(name: 'VERSION_FOR_INSTALL', value: latestRelease),
   185                                          string(name: 'VZ_INSTALL_CONFIG', value: params.VZ_INSTALL_CONFIG),
   186                                          string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE),
   187                                          string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   188                                          string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS),
   189                                          string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS),
   190                                          string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS),
   191                                          string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH),
   192                                      ], wait: true
   193                              }
   194                          }
   195                      }
   196                  }
   197                  stage('Uninstall Tests') {
   198                      steps {
   199                          retry(count: JOB_PROMOTION_RETRIES) {
   200                              script {
   201                                  build job: "/verrazzano-uninstall-test/${CLEAN_BRANCH_NAME}",
   202                                      parameters: [
   203                                          string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   204                                          string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE),
   205                                          string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   206                                          string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS),
   207                                          string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS),
   208                                          string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS),
   209                                          string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH)
   210                                      ], wait: true
   211                              }
   212                          }
   213                      }
   214                  }
   215                  stage('OCI DNS tests') {
   216                      steps {
   217                          retry(count: JOB_PROMOTION_RETRIES) {
   218                              script {
   219                                  build job: "/verrazzano-new-oci-dns-acceptance-tests/${CLEAN_BRANCH_NAME}",
   220                                      parameters: [
   221                                          string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   222                                          string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE),
   223                                          booleanParam(name: 'CREATE_CLUSTER_USE_CALICO', value: true),
   224                                          string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS),
   225                                          string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS),
   226                                          string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS),
   227                                      ], wait: true
   228                              }
   229                          }
   230                      }
   231                  }
   232                  stage('Kind No Istio Injection Tests') {
   233                      steps {
   234                          retry(count: JOB_PROMOTION_RETRIES) {
   235                              script {
   236                                  build job: "/verrazzano-no-injection-tests/${CLEAN_BRANCH_NAME}",
   237                                      parameters: [
   238                                          string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'),
   239                                          string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   240                                          string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE),
   241                                          string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   242                                          string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS),
   243                                          string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS),
   244                                          string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS),
   245                                          booleanParam(name: 'RUN_COHERENCE_TESTS', value: false),
   246                                          string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH)
   247                                      ], wait: true
   248                              }
   249                          }
   250                      }
   251                  }
   252                  stage('Upgrade Path Tests') {
   253                      steps {
   254                          retry(count: JOB_PROMOTION_RETRIES) {
   255                              script {
   256                                  def latestRelease = LATEST_RELEASE_VERSION
   257                                  echo "Printing latest release version: ${latestRelease}"
   258                                  build job: "/verrazzano-upgrade-path-tests/${CLEAN_BRANCH_NAME}",
   259                                      parameters: [
   260                                          string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
   261                                          string(name: 'VERSION_FOR_INSTALL', value: latestRelease),
   262                                          string(name: 'VZ_INSTALL_CONFIG', value: params.VZ_INSTALL_CONFIG),
   263                                          string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE),
   264                                          string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN),
   265                                          string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS),
   266                                          string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS),
   267                                          string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS),
   268                                          string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH),
   269                                      ], wait: true
   270                              }
   271                          }
   272                      }
   273                  }
   274              }
   275          }
   276      }
   277      post {
   278          success {
   279              // WARNING: Only do this object storage update here, do not copy to other jobs
   280              storePipelineArtifacts()
   281          }
   282          failure {
   283              script {
   284                  failedOrAborted()
   285              }
   286          }
   287          aborted {
   288              script {
   289                  failedOrAborted()
   290              }
   291          }
   292      }
   293  }
   294  
   295  def failedOrAborted() {
   296      if (env.JOB_NAME == "verrazzano-push-triggered-acceptance-tests/master" || env.JOB_NAME ==~ "verrazzano-push-triggered-acceptance-tests/release-1.*") {
   297          if (isPagerDutyEnabled()) {
   298              pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", incDescription: "Verrazzano: ${env.JOB_NAME} - Failed", incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}")
   299          }
   300          slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}" )
   301      }
   302  }
   303  
   304  def isPagerDutyEnabled() {
   305      // this controls whether PD alerts are enabled
   306      if (NOTIFY_PAGERDUTY_TRIGGERED_FAILURES.equals("true")) {
   307          echo "Pager-Duty notifications enabled via global override setting"
   308          return true
   309      }
   310      return false
   311  }
   312  
   313  // Called in final post success block of pipeline
   314  // WARNING: Only do this object storage update here, do not copy to other jobs
   315  def storePipelineArtifacts() {
   316      script {
   317          // If this was clean, record the commit in object store so the periodic test jobs can run against that rather than the head of the branch
   318          // NOTE: Normally master and release-* branches are the only ones doing this, but when we need to test out pipeline changes we can make use
   319          // as well
   320          LAST_STABLE_COMMIT=getLastStableCommit()
   321          IS_ANCESTOR=isAncestor()
   322          if (LAST_STABLE_COMMIT == null || IS_ANCESTOR) {
   323              sh """
   324                          echo "git-commit=${env.GIT_COMMIT}" > $WORKSPACE/last-stable-commit.txt
   325                          if [ "true" == "${params.FORCE_STABLE_COMMIT_UPDATE}" ]; then
   326                              echo "forced-by-job=${env.BUILD_URL}" >> $WORKSPACE/last-stable-commit.txt
   327                          fi
   328                          cat $WORKSPACE/last-stable-commit.txt
   329                      """
   330              putLastStableCommit()
   331          }
   332      }
   333  }
   334  
   335  // Sets Job Display Name based on last-stable-commit.txt put status
   336  def putLastStableCommit() {
   337      putLastStableCommitCommandOutput = sh (
   338                                                 label: "Put last stable commit text file",
   339                                                 script: "oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_BRANCH_NAME}/last-stable-commit.txt --file $WORKSPACE/last-stable-commit.txt 2>&1 || true",
   340                                                 returnStdout: true
   341                                                 ).trim()
   342      echo "command out: ${putLastStableCommitCommandOutput}"
   343      if (putLastStableCommitCommandOutput.length() > 0) {
   344          if (putLastStableCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) {
   345              currentBuild.displayName = "${currentBuild.displayName} : Put Last Stable Commit Failed"
   346              slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nFailed to put last-stable-commit.txt in Object Store\n" )
   347          }
   348      }
   349  }
   350  
   351  // Returns true if current commit is ancestor of last stable commit
   352  def isAncestor() {
   353     LAST_STABLE_COMMIT=getLastStableCommit()
   354     ANCESTOR=''
   355     if (LAST_STABLE_COMMIT != null) {
   356         ANCESTOR = sh (
   357                            label: "Check for Ancestry",
   358                            script: "git rev-list ${env.GIT_COMMIT} | grep \$(git rev-parse $LAST_STABLE_COMMIT) 2>&1 || true",
   359                            returnStdout: true
   360                            ).trim()
   361     }
   362     if (ANCESTOR.length() > 0) {
   363         return true
   364     } else {
   365         return false
   366     }
   367  }
   368  
   369  // Returns the last stable commit for the branch, or null if the commit file does not exist yet.
   370  // - fails the pipeline if any error other than 404 is returned by the OCI CLI
   371  def getLastStableCommit() {
   372      lastStableCommitCommandOutput = sh (
   373          label: "Get last stable commit text file",
   374          script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_BRANCH_NAME}/last-stable-commit.txt --file ${WORKSPACE}/stored-stable-commit.txt 2>&1 || true",
   375          returnStdout: true
   376          ).trim()
   377      echo "command out: ${lastStableCommitCommandOutput}"
   378      if (lastStableCommitCommandOutput.length() > 0) {
   379          // We can get warning messages here as well even when the command succeeded, so be more precise on the checking
   380          if (lastStableCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) {
   381              // If we think we had a status: NNN, we ignore 404 and fail for others
   382              assert lastStableCommitCommandOutput =~ /(.*)status(.*)404(.*)/ : "An unexpected error occurred getting last stable commit from ObjectStore: ${lastStableCommitCommandOutput}"
   383          } else {
   384              // If we got here, we have some message that may or may not be an error. If we don't see the file, we assume it was an error
   385              sh """
   386                  if [ ! -f $WORKSPACE/stored-stable-commit.txt ]; then
   387                      echo "An unexpected error occurred getting last stable commit from ObjectStore: ${lastStableCommitCommandOutput}"
   388                      exit 1
   389                  fi
   390              """
   391          }
   392      }
   393      // Get the commit ID for the last stable commit
   394      def lastStableCommitProps = readProperties file: "$WORKSPACE/stored-stable-commit.txt"
   395      return lastStableCommitProps['git-commit']
   396  }
   397  
   398  // Called in Stage Clean workspace and checkout steps
   399  @NonCPS
   400  def getCommitList() {
   401      echo "Checking for change sets"
   402      def commitList = []
   403      def changeSets = currentBuild.changeSets
   404      for (int i = 0; i < changeSets.size(); i++) {
   405          echo "get commits from change set"
   406          def commits = changeSets[i].items
   407          for (int j = 0; j < commits.length; j++) {
   408              def commit = commits[j]
   409              def id = commit.commitId
   410              echo "Add commit id: ${id}"
   411              commitList.add(id)
   412          }
   413      }
   414      return commitList
   415  }
   416  
   417  def trimIfGithubNoreplyUser(userIn) {
   418      if (userIn == null) {
   419          echo "Not a github noreply user, not trimming: ${userIn}"
   420          return userIn
   421      }
   422      if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) {
   423          def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@"))
   424          return userOut;
   425      }
   426      if (userIn.matches(".*<.*@users.noreply.github.com.*")) {
   427          def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@"))
   428          return userOut;
   429      }
   430      if (userIn.matches(".*@users.noreply.github.com")) {
   431          def userOut = userIn.substring(0, userIn.indexOf("@"))
   432          return userOut;
   433      }
   434      echo "Not a github noreply user, not trimming: ${userIn}"
   435      return userIn
   436  }
   437  
   438  def getSuspectList(commitList, userMappings) {
   439      def retValue = ""
   440      def suspectList = []
   441      if (commitList == null || commitList.size() == 0) {
   442          echo "No commits to form suspect list"
   443      } else {
   444          for (int i = 0; i < commitList.size(); i++) {
   445              def id = commitList[i]
   446              try {
   447                  def gitAuthor = sh(
   448                      script: "git log --format='%ae' '$id^!'",
   449                      returnStdout: true
   450                  ).trim()
   451                  if (gitAuthor != null) {
   452                      def author = trimIfGithubNoreplyUser(gitAuthor)
   453                      echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}"
   454                      if (userMappings.containsKey(author)) {
   455                          def slackUser = userMappings.get(author)
   456                          if (!suspectList.contains(slackUser)) {
   457                              echo "Added ${slackUser} as suspect"
   458                              retValue += " ${slackUser}"
   459                              suspectList.add(slackUser)
   460                          }
   461                      } else {
   462                          // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated
   463                          if (!suspectList.contains(author)) {
   464                              echo "Added ${author} as suspect"
   465                              retValue += " ${author}"
   466                              suspectList.add(author)
   467                          }
   468                      }
   469                  } else {
   470                      echo "No author returned from git"
   471                  }
   472              } catch (Exception e) {
   473                  echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString()
   474              }
   475          }
   476      }
   477      def startedByUser = "";
   478      def causes = currentBuild.getBuildCauses()
   479      echo "causes: " + causes.toString()
   480      for (cause in causes) {
   481          def causeString = cause.toString()
   482          echo "current cause: " + causeString
   483          def causeInfo = readJSON text: causeString
   484          if (causeInfo.userId != null) {
   485              startedByUser = causeInfo.userId
   486          }
   487      }
   488  
   489      if (startedByUser.length() > 0) {
   490          echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}"
   491          def author = trimIfGithubNoreplyUser(startedByUser)
   492          echo "DEBUG: author: ${startedByUser}, ${author}"
   493          if (userMappings.containsKey(author)) {
   494              def slackUser = userMappings.get(author)
   495              if (!suspectList.contains(slackUser)) {
   496                  echo "Added ${slackUser} as suspect"
   497                  retValue += " ${slackUser}"
   498                  suspectList.add(slackUser)
   499              }
   500          } else {
   501              // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated
   502              if (!suspectList.contains(author)) {
   503                 echo "Added ${author} as suspect"
   504                 retValue += " ${author}"
   505                 suspectList.add(author)
   506              }
   507          }
   508      } else {
   509          echo "Build not started by a user, not adding to notification list"
   510      }
   511      echo "returning suspect list: ${retValue}"
   512      return retValue
   513  }