github.com/verrazzano/verrazzano@v1.7.0/ci/JenkinsfileBackendTests

// Copyright (c) 2021, 2023, Oracle and/or its affiliates.
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.

import groovy.transform.Field

@Field
def GIT_COMMIT_TO_USE = ""
@Field
def LAST_CLEAN_BACKEND_COMMIT = ""
@Field
def TESTS_FAILED = false
@Field
def storeLocation=""
@Field
def verrazzanoPrefix="verrazzano-"
@Field
def fullBundle=""
@Field
def liteBundle=""
@Field
def SUSPECT_LIST = ""
@Field
def COMPARISON_URL_ON_FAILURE = ""

@Field
def backendTestsUpToDate              = false // If true, indicates that the backend tests already passed at the latest commit
@Field
def backendTestsUpToDateFailed        = false // If true, indicates that the backend tests already ran and failed at the latest commit

// Non Fields
// def branchSpecificSchedule = getCronSchedule()

// File containing the links to download the Verrazzano distributions
@Field
def verrazzanoDistributionsFile = "verrazzano_distributions.html"

def ociOsRegion = "us-phoenix-1"
def ociOsBucket = "verrazzano-builds"
def urlTriggerBranchName = env.BRANCH_NAME.replace('/', '%2F')
def lastStableCommitFile = "last-stable-commit.txt"

pipeline {
    options {
        timeout(time: 12, unit: 'HOURS')
        skipDefaultCheckout true
        disableConcurrentBuilds()
        timestamps ()
    }

    agent {
       docker {
            image "${RUNNER_DOCKER_IMAGE}"
            args "${RUNNER_DOCKER_ARGS}"
            registryUrl "${RUNNER_DOCKER_REGISTRY_URL}"
            registryCredentialsId 'ocir-pull-and-push-account'
            label "pipeline-job-large"
        }
    }

    //  triggers {
    //      cron(branchSpecificSchedule)

    // There is an issue with URLTrigger for backend tests where the polling URL maintains a global state that does not work with specific jobs.
    // Therefore, the below code is disabled until the global state issue is fixed.
    // Ex: - Each job has an URL Trigger poll cron schedule
    //      - When a job’s cron triggers, it checks the state of whether the specific URL X has changed or not.
    //      - If it has changed, then it triggers only the job whose cron schedule fired off (no other jobs that look for that URL X will trigger)
    //      - Any other jobs that wake up and poll on URL X will not see it as changing unless it actually changed again since this time

    //        URLTrigger(
    //            cronTabSpec: branchSpecificSchedule,
    //            entries: [
    //                URLTriggerEntry(
    //                    url: "https://objectstorage.${ociOsRegion}.oraclecloud.com/n/${OS_NAMESPACE_URL_TRIGGER}/b/${ociOsBucket}/o/${urlTriggerBranchName}/${lastStableCommitFile}",
    //                    checkETag: false,
    //                    checkStatus: true,
    //                    statusCode: 403,
    //                    checkLastModificationDate: true,
    //                    timeout: 200,
    //                    requestHeaders: [
    //                        RequestHeader( headerName: "Accept" , headerValue: "application/json" )
    //                    ],
    //                   contentTypes: [
    //                        MD5Sum()
    //                    ]
    //                )
    //            ]
    //        )
    //    }

    parameters {
        string (name: 'TAGGED_TESTS',
                defaultValue: '',
                description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',
                trim: true)
        string (name: 'INCLUDED_TESTS',
                defaultValue: '.*',
                description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',
                trim: true)
        string (name: 'EXCLUDED_TESTS',
                defaultValue: '_excluded_test',
                description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',
                trim: true)
        booleanParam (description: 'Force execution of the tests even if up-to-date', name: 'FORCE', defaultValue: false)
        booleanParam (description: 'Skip test execution (for debugging)', name: 'DRY_RUN', defaultValue: false)
    }

    environment {
        IS_BACKEND_PIPELINE = "true"
        OCIR_SCAN_COMPARTMENT = credentials('ocir-scan-compartment')
        OCIR_SCAN_TARGET = credentials('ocir-scan-target')
        OCIR_SCAN_REGISTRY = credentials('ocir-scan-registry')
        OCIR_SCAN_REPOSITORY_PATH = credentials('ocir-scan-repository-path')
        DOCKER_SCAN_CREDS = credentials('v8odev-ocir')
        DOCKER_CREDS = credentials('github-packages-credentials-rw')
        DOCKER_REPO = 'ghcr.io'

        OCI_CLI_AUTH="instance_principal"
        OCI_OS_NAMESPACE = credentials('oci-os-namespace')
        OCI_OS_BUCKET="verrazzano-builds"
        OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit"
        CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}"
        SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY')

        STABLE_COMMIT_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-stable-commit.txt"
        LAST_BACKEND_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-backend-run-commit.txt"
        CLEAN_BACKEND_OS_LOCATION = "${CLEAN_BRANCH_NAME}-last-clean-backend-test/verrazzano_backend-commit.txt"

        STABLE_COMMIT_LOCATION = "${WORKSPACE}/last-stable-commit.txt"
        LAST_BACKEND_LOCATION = "${WORKSPACE}/last-backend-run-commit.txt"
        CLEAN_BACKEND_LOCATION = "${WORKSPACE}/last-clean-backend-commit.txt"

        OCI_OS_REGION="us-phoenix-1"

        PIPELINE_OWNERS = credentials('backendtests-owners')
    }

    // This job runs against the latest stable master commit. That is defined as the last clean master build and test run whose
    // commit has been stored in object storage. This job will fetch that commit from master and run extended tests using that.
    // This job is NOT currently setup to run extended tests from other branches, if you need to run those extended jobs you will
    // need to run those against your branch individually.

    stages {
        stage('Check last clean backend') {
            steps {
                sh """
                    oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${STABLE_COMMIT_OS_LOCATION} --file ${STABLE_COMMIT_LOCATION}
                """

                script {
                    // Check if there is already a clean backend run at this commit already, and set the display name if
                    // it already is tested, or if doing a special run type (dry run, etc...)
                    preliminaryChecks()
                }
            }
        }
        stage('Clean workspace and checkout') {
            when {
                allOf {
                    expression { return runTests() }
                }
            }
            steps {
                script {
                    cleanWorkspaceAndCheckout()
                }
            }
        }

        stage ('Backend Test Suites') {
            when {
                allOf {
                    expression { return runTests() }
                }
            }
            parallel {
                stage('Quick Create Tests') {
                    steps {
                        script {
                            build job: "/verrazzano-clusterAPI-qc-kind-tests/${CLEAN_BRANCH_NAME}",
                                    parameters: [
                                            string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
                                    ], wait: false, propagate: false
                        }
                    }
                }

                stage('Upgrade Path Minor Release Tests') {
                    steps {
                        script {
                            build job: "/verrazzano-push-triggered-upgrade-minor-release-tests/${CLEAN_BRANCH_NAME}",
                                parameters: [
                                    string(name: 'N_JOBS_FOR_EACH_BATCH', value: '6'),
                                    string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
                                    string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS),
                                    string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS),
                                    string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS),
                                ], wait: true, propagate: true
                        }
                    }
                }

                stage('vz analyze tool') {
                    environment {
                        SEARCH_HTTP_ENDPOINT = credentials('search-gw-url')
                        SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}"
                        SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}"
                    }
                    steps {
                        retry(count: JOB_PROMOTION_RETRIES) {
                            script {
                                build job: "/verrazzano-analyze-tool-test/${CLEAN_BRANCH_NAME}",
                                    parameters: [
                                        string(name: 'KUBERNETES_CLUSTER_VERSION', value: '1.27'),
                                        string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
                                    ], wait: true
                            }
                        }
                    }
                }

                stage('A la carte tests') {
                    steps {
                        script {
                            build job: "/verrazzano-a-la-carte-triggered/${CLEAN_BRANCH_NAME}",
                                parameters: [
                                    string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
                                ], wait: true, propagate: true
                        }
                    }
                }

                stage('Examples on OKE OCI DNS Tests') {
                    steps {
                        retry(count: JOB_PROMOTION_RETRIES) {
                            script {
                                build job: "verrazzano-new-oci-dns-examples-tests/${CLEAN_BRANCH_NAME}",
                                    parameters: [
                                        string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT),
                                        string(name: 'TEST_ENV', value: "ocidns_oke"),
                                        string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS),
                                        string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS),
                                        string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS)
                                    ], wait: true
                            }
                        }
                    }
                    post {
                        failure {
                            script {
                                TESTS_FAILED = true
                            }
                        }
                    }
                }
            }
        }
        stage('Update Last Clean Backend Test') {
            when {
                allOf {
                    expression { return runTests() }
                    expression { TESTS_FAILED == false }
                }
            }
            environment {
                GIT_COMMIT_USED = "${env.GIT_COMMIT}"
            }
            steps {
                script {
                    sh """
                        # Update the clean backend commit
                        echo "git-commit=${GIT_COMMIT_USED}" > commit-that-passed.txt
                        oci --region ${OCI_OS_REGION} os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_BRANCH_NAME}-last-clean-backend-test/verrazzano_backend-commit.txt --file commit-that-passed.txt
                    """
                }
            }
        }
    }
    post {
        always {
            script {
                sh """
                    # Update the last backend commit
                    echo "git-commit=${env.GIT_COMMIT}" > commit-used.txt
                    oci --region ${OCI_OS_REGION} os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${LAST_BACKEND_OS_LOCATION} --file commit-used.txt
                """
            }
        }
        failure {
            script {
                failedOrAborted()
            }
        }
        aborted {
            script {
                failedOrAborted()
            }
        }
        cleanup {
            deleteDir()
        }
    }
}

def failedOrAborted() {
    if (isAlertingEnabled()) {
        if (isPagerDutyEnabled()) {
            pagerduty(resolve: false, serviceKey: "$SERVICE_KEY",
            incDescription: "Verrazzano Backend Tests: ${env.JOB_NAME} - Failed",
            incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}")
        }
        slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST} ${PIPELINE_OWNERS}\n\nChange comparison: ${COMPARISON_URL_ON_FAILURE}" )
        echo "done alerts"
    }
}

// Preliminary job checks and display updates
def preliminaryChecks() {
    // Get the last stable commit ID to pass the triggered tests
    def stableCommitProps = readProperties file: "${STABLE_COMMIT_LOCATION}"
    GIT_COMMIT_TO_USE = stableCommitProps['git-commit']
    echo "Last stable commit: ${GIT_COMMIT_TO_USE}"

    LAST_CLEAN_BACKEND_COMMIT=getLastCleanBackendCommit()
    echo "Last clean backend commit: ${LAST_CLEAN_BACKEND_COMMIT}"

    if (LAST_CLEAN_BACKEND_COMMIT == GIT_COMMIT_TO_USE) {
        backendTestsUpToDate = true
    } else {
        // Check if we are still at the same commit previously run (if so we know it wasn't clean and it failed in some way)
        LAST_BACKEND_RUN_COMMIT=getLastBackendRunCommit()
        if (LAST_BACKEND_RUN_COMMIT != null && LAST_BACKEND_RUN_COMMIT == GIT_COMMIT_TO_USE) {
            backendTestsUpToDateFailed = true
        }
    }

    echo "Up to date: ${backendTestsUpToDate}"
    echo "Dry run: ${params.DRY_RUN}"
    echo "Force run: ${params.FORCE}"
    echo "Execute tests: " + runTests()

    // Indicate in title if run is up-to-date or dry-run
    if (params.DRY_RUN) {
        currentBuild.displayName = "${currentBuild.displayName} : DRY-RUN"
    }
    if (backendTestsUpToDate) {
        currentBuild.displayName = "${currentBuild.displayName} : UP-TO-DATE"
    }
    if (params.FORCE) {
        currentBuild.displayName = "${currentBuild.displayName} : FORCE"
    } else if (backendTestsUpToDateFailed) {
       currentBuild.displayName = "${currentBuild.displayName} : UP-TO-DATE-FAILED"
       currentBuild.result = 'FAILURE'
       error('Failing the build since the current commit matches the commit of previously failing backend build')
    }

    if (runTests()) {
        echo "Executing backend tests for commit ${GIT_COMMIT_TO_USE}"
    }
}

def dockerLogins() {
    try {
        sh """
            echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin
        """
    } catch(error) {
        echo "docker login failed, retrying after sleep"
        retry(4) {
            sleep(30)
            sh """
            echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin
            """
        }
    }
    if (!(env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-"))) {
        try {
            sh """
                echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
            """
        } catch(error) {
            echo "docker login failed, retrying after sleep"
            retry(4) {
                sleep(30)
                sh """
                    echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin
                """
            }
        }
    }
}

def scmCheckout() {
    echo "${NODE_LABELS}"
    echo "SCM checkout of ${GIT_COMMIT_TO_USE}"
    def scmInfo = checkout([
        $class: 'GitSCM',
        branches: [[name: GIT_COMMIT_TO_USE]],
        doGenerateSubmoduleConfigurations: false,
        extensions: [],
        submoduleCfg: [],
        userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]])
    env.GIT_COMMIT = scmInfo.GIT_COMMIT
    env.GIT_BRANCH = scmInfo.GIT_BRANCH
    echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}"
    // If the commit we were handed is not what the SCM says we are using, fail
    if (!env.GIT_COMMIT.equals(GIT_COMMIT_TO_USE)) {
        error( "SCM didn't checkout the commit we expected. Expected: ${GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}")
    }

    if (LAST_CLEAN_BACKEND_COMMIT != null) {
        COMPARISON_URL_ON_FAILURE = "https://github.com/verrazzano/verrazzano/compare/${LAST_CLEAN_BACKEND_COMMIT}...${GIT_COMMIT_TO_USE}"
        def lastClean = "${LAST_CLEAN_BACKEND_COMMIT}"
        def currentStable = "${GIT_COMMIT_TO_USE}"
        def commitList = getCommitListFromGitLog(lastClean, currentStable)
        withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) {
            def userMappings = readJSON file: JENKINS_TO_SLACK_JSON
            SUSPECT_LIST = getSuspectList(commitList, userMappings)
            echo "Suspect list: ${SUSPECT_LIST}"
        }
    }
    echo "URL if fails: ${COMPARISON_URL_ON_FAILURE}"
}

def cleanWorkspaceAndCheckout() {
    scmCheckout()
    dockerLogins()
    TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim()
    SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim()
    // update the description with some meaningful info
    currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + GIT_COMMIT_TO_USE
    storeLocation="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}"
}

// Returns the last clean commit for the backends, or null if the commit file does not exist yet.
// - fails the pipeline if any error other than 404 is returned by the OCI CLI
def getLastCleanBackendCommit() {
    lastBackendCommitCommandOutput = sh (
        label: "Get last clean backend commit ID",
        script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_BACKEND_OS_LOCATION} --file ${CLEAN_BACKEND_LOCATION} 2>&1 || true",
        returnStdout: true
        ).trim()
    echo "command out: ${lastBackendCommitCommandOutput}"
    if (lastBackendCommitCommandOutput.length() > 0) {
        // We can get warning messages here as well even when the command succeeded, so be more precise on the checking
        if (lastBackendCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) {
            // If we think we had a status: NNN, we ignore 404 and fail for others
            assert lastBackendCommitCommandOutput =~ /(.*)status(.*)404(.*)/ : "An unexpected error occurred getting last backend commit from ObjectStore: ${lastBackendCommitCommandOutput}"
        } else {
            // If we got here, we have some message that may or may not be an error. If we don't see the file, we assume it was an error
            sh """
                if [ ! -f ${CLEAN_BACKEND_LOCATION} ]; then
                    echo "An unexpected error occurred getting last backend commit from ObjectStore: ${lastBackendCommitCommandOutput}"
                    exit 1
                fi
            """
        }
    }
    // Get the commit ID for the last known clean pass of the Backend tests
    def cleanBackendsCommitProps = readProperties file: "${CLEAN_BACKEND_LOCATION}"
    return cleanBackendsCommitProps['git-commit']
}

// Returns the last run commit for the Backend, or null if the commit file does not exist yet.
// - fails the pipeline if any error other than 404 is returned by the OCI CLI
def getLastBackendRunCommit() {
    lastBackendCommitCommandOutput = sh (
        label: "Get last clean backend commit ID",
        script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${LAST_BACKEND_OS_LOCATION} --file ${LAST_BACKEND_LOCATION} 2>&1 || true",
        returnStdout: true
        ).trim()
    echo "command out: ${lastBackendCommitCommandOutput}"
    if (lastBackendCommitCommandOutput.length() > 0) {
        // We can get warning messages here as well even when the command succeeded, so be more precise on the checking
        if (lastBackendCommitCommandOutput =~ /(.*)status(.*)\d{1,4}(.*)/) {
            // If we think we had a status: NNN, we ignore 404 and fail for others
            assert lastBackendCommitCommandOutput =~ /(.*)status(.*)404(.*)/ : "An unexpected error occurred getting last backend commit from ObjectStore: ${lastBackendCommitCommandOutput}"
        } else {
            // If we got here, we have some message that may or may not be an error. If we don't see the file, we assume it was an error
            sh """
                if [ ! -f ${LAST_BACKEND_LOCATION} ]; then
                    echo "An unexpected error occurred getting last backend run commit from ObjectStore: ${lastBackendCommitCommandOutput}"
                    exit 1
                fi
            """
        }
    }
    // Get the commit ID for the last known clean pass of the Backend tests
    def lastBackendCommitProps = readProperties file: "${LAST_BACKEND_LOCATION}"
    return lastBackendCommitProps['git-commit']
}

// Checks all the conditions gating test execution and collates the result
def runTests() {
  return params.FORCE || ( ! backendTestsUpToDate && ! backendTestsUpToDateFailed && ! params.DRY_RUN )
}

def isAlertingEnabled() {
    // this controls whether any alerting happens for these tests
    if (NOTIFY_BACKEND_FAILURES.equals("true") && (env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-"))) {
        echo "Alert notifications enabled for ${env.BRANCH_NAME}"
        return true
    }
    return false
}

def isPagerDutyEnabled() {
    // this additionally controls whether PD alerts are enabled (note that you must also enable alerting in general as well if you want these)
    if (NOTIFY_PAGERDUTY_BACKEND_FAILURES.equals("true")) {
        echo "Pager-Duty notifications enabled via global override setting"
        return true
    }
    return false
}

//  def getCronSchedule() {
//      if (env.BRANCH_NAME.equals("master")) {
//          return "@weekly"
//      } else if (env.BRANCH_NAME.startsWith("release-")) {
//          return "@weekly"
//      }
//      return ""
//  }

// Called in Stage Clean workspace and checkout steps
def getCommitListFromGitLog(lastClean, currentStable) {
    echo "Checking for change sets"
    def commitList = sh(returnStdout: true, script: "git log ${lastClean}...${currentStable} --oneline | cut -d \" \" -f 1").trim().split('\n')
    for (int i = 0; i < commitList.size(); i++) {
        echo "Found commit id: ${commitList[i]}"
    }
    return commitList
}

def trimIfGithubNoreplyUser(userIn) {
    if (userIn == null) {
        echo "Not a github noreply user, not trimming: ${userIn}"
        return userIn
    }
    if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) {
        def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@"))
        return userOut;
    }
    if (userIn.matches(".*<.*@users.noreply.github.com.*")) {
        def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@"))
        return userOut;
    }
    if (userIn.matches(".*@users.noreply.github.com")) {
        def userOut = userIn.substring(0, userIn.indexOf("@"))
        return userOut;
    }
    echo "Not a github noreply user, not trimming: ${userIn}"
    return userIn
}

def getSuspectList(commitList, userMappings) {
    def retValue = ""
    def suspectList = []
    if (commitList == null || commitList.size() == 0) {
        echo "No commits to form suspect list"
    } else {
        for (int i = 0; i < commitList.size(); i++) {
            def id = commitList[i]
            try {
                def gitAuthor = sh(
                    script: "git log --format='%ae' '$id^!'",
                    returnStdout: true
                ).trim()
                if (gitAuthor != null) {
                    def author = trimIfGithubNoreplyUser(gitAuthor)
                    echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}"
                    if (userMappings.containsKey(author)) {
                        def slackUser = userMappings.get(author)
                        if (!suspectList.contains(slackUser)) {
                            echo "Added ${slackUser} as suspect"
                            retValue += " ${slackUser}"
                            suspectList.add(slackUser)
                        }
                    } else {
                        // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated
                        if (!suspectList.contains(author)) {
                            echo "Added ${author} as suspect"
                            retValue += " ${author}"
                            suspectList.add(author)
                        }
                    }
                } else {
                    echo "No author returned from git"
                }
            } catch (Exception e) {
                echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString()
            }
        }
    }
    def startedByUser = "";
    def causes = currentBuild.getBuildCauses()
    echo "causes: " + causes.toString()
    for (cause in causes) {
        def causeString = cause.toString()
        echo "current cause: " + causeString
        def causeInfo = readJSON text: causeString
        if (causeInfo.userId != null) {
            startedByUser = causeInfo.userId
        }
    }

    if (startedByUser.length() > 0) {
        echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}"
        def author = trimIfGithubNoreplyUser(startedByUser)
        echo "DEBUG: author: ${startedByUser}, ${author}"
        if (userMappings.containsKey(author)) {
            def slackUser = userMappings.get(author)
            if (!suspectList.contains(slackUser)) {
                echo "Added ${slackUser} as suspect"
                retValue += " ${slackUser}"
                suspectList.add(slackUser)
            }
        } else {
            // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated
            if (!suspectList.contains(author)) {
               echo "Added ${author} as suspect"
               retValue += " ${author}"
               suspectList.add(author)
            }
        }
    } else {
        echo "Build not started by a user, not adding to notification list"
    }
    echo "returning suspect list: ${retValue}"
    return retValue
}

@NonCPS
List extractReleaseTags(final String fileContent) {
    List releases = []
    fileContent.eachLine { tag ->
        releases << tag
    }
    return releases
}

def getLatestReleaseVersion() {
    final String releaseTags = readFile(file: "${workspace}/tags.txt")
    list gitTags = extractReleaseTags(releaseTags)
    echo "gitTags = ${gitTags}"
    return gitTags.pop()
}