github.com/verrazzano/verrazzano@v1.7.1/ci/upgrade-paths/JenkinsfileTestTriggerMinorRelease (about) 1 // Copyright (c) 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 5 def listOfUpgradeJobs 6 def upgradeJobsStageMapping 7 8 pipeline { 9 options { 10 skipDefaultCheckout true 11 } 12 13 agent { 14 docker { 15 image "${RUNNER_DOCKER_IMAGE}" 16 args "${RUNNER_DOCKER_ARGS}" 17 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 18 registryCredentialsId 'ocir-pull-and-push-account' 19 label "${agentLabel}" 20 } 21 } 22 // Use cases: 23 // 1) (automatic) master, release* will trigger this job by default on successful runs and supply GIT_COMMIT_TO_USE with the exact GIT commit to use for all testing (same as the upstream build) 24 // 2) (automatic) branch builds with TRIGGER_FULL_TESTS enabled will trigger this job by default on successful runs and supply GIT_COMMIT_TO_USE with the exact GIT commit to use for all testing (same as the upstream build) 25 // 3) (manual) any branch should be able to call this “trigger” job with a commit that was previously built and has a valid GIT commit hash. In this case the manual job 26 // must be started from the desired branch using Build with Parameters, the GIT_COMMIT_TO_USE must supply the GIT commit hash from the previous build, and VERRAZZANO_OPERATOR_IMAGE must NOT BE SPECIFIED or be NONE 27 // This allows one to rerun the tests without rerunning the upstream build (ie: if intermittent test issue occurred) 28 // 4) (manual) any branch should be able to call this “trigger” job with the current head of their branch, pointing to a previously built operator image. This is useful if you are adding/fixing test cases where the 29 // operator image was built already (from master, or your branch) and only want to run the tests using it without running the full build. This is not a super likely situation (more likely individual test jobs 30 // will be manually kicked off rather than all of them). To accomplish this, specify GIT_COMMIT_TO_USE=NONE, and VERRAZZANO_OPERATOR_IMAGE=image-to-use 31 parameters { 32 string (name: 'GIT_COMMIT_TO_USE', 33 defaultValue: 'NONE', 34 description: 'This is the full git commit hash from the source build to be used for all jobs. A full pipeline specifies a valid commit hash here. NONE can be used for manually triggered jobs, however even for those a commit hash value is preferred to be supplied', 35 trim: true) 36 string (name: 'N_JOBS_FOR_EACH_BATCH', 37 defaultValue: '3', 38 description: 'This is to define total number of jobs executed in parallel per each batch', 39 trim: true) 40 choice (description: 'Predefined config permutations for Verrazzano installation. Prod profile is the default profile for NONE', name: 'VZ_INSTALL_CONFIG', 41 choices: ["NONE", "dev-kind-persistence"]) 42 choice (name: 'KIND_CLUSTER_VERSION', 43 description: 'Kubernetes Version for KIND Cluster', 44 // 1st choice is the default value 45 choices: [ "1.24", "1.25", "1.26", "1.27" ]) 46 string (name: 'EXCLUDE_RELEASES', 47 defaultValue: "v1.0, v1.1, v1.2, v1.3", 48 description: 'This is to exclude the specified releases from upgrade tests.', trim: true) 49 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 50 defaultValue: 'NONE', 51 description: 'This is for manually testing only where someone needs to use a specific operator image, otherwise the default value of NONE is used', 52 trim: true) 53 string (name: 'WILDCARD_DNS_DOMAIN', 54 defaultValue: 'nip.io', 55 description: 'This is the wildcard DNS domain', 56 trim: true) 57 booleanParam (description: 'Whether to use External Elasticsearch', name: 'EXTERNAL_ELASTICSEARCH', defaultValue: false) 58 string (name: 'TAGGED_TESTS', 59 defaultValue: '', 60 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 61 trim: true) 62 string (name: 'INCLUDED_TESTS', 63 defaultValue: '.*', 64 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 65 trim: true) 66 string (name: 'EXCLUDED_TESTS', 67 defaultValue: '_excluded_test', 68 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 69 trim: true) 70 string (name: 'CONSOLE_REPO_BRANCH', 71 defaultValue: '', 72 description: 'The branch to check out after cloning the console repository.', 73 trim: true) 74 } 75 76 environment { 77 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 78 GOPATH = '/home/opc/go' 79 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 80 OCI_CLI_AUTH="instance_principal" 81 PROMETHEUS_GW_URL = credentials('prometheus-dev-url') 82 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 83 TARGET_UPGRADE_VERSION = "current_branch" 84 } 85 86 stages { 87 stage('Clean workspace and checkout') { 88 steps { 89 sh """ 90 echo "${NODE_LABELS}" 91 """ 92 93 script { 94 if (params.GIT_COMMIT_TO_USE == "NONE") { 95 echo "Specific GIT commit was not specified, use current head" 96 def scmInfo = checkout([ 97 $class: 'GitSCM', 98 branches: [[name: env.BRANCH_NAME]], 99 doGenerateSubmoduleConfigurations: false, 100 extensions: [], 101 submoduleCfg: [], 102 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 103 env.GIT_COMMIT = scmInfo.GIT_COMMIT 104 env.GIT_BRANCH = scmInfo.GIT_BRANCH 105 } else { 106 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 107 def scmInfo = checkout([ 108 $class: 'GitSCM', 109 branches: [[name: params.GIT_COMMIT_TO_USE]], 110 doGenerateSubmoduleConfigurations: false, 111 extensions: [], 112 submoduleCfg: [], 113 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 114 env.GIT_COMMIT = scmInfo.GIT_COMMIT 115 env.GIT_BRANCH = scmInfo.GIT_BRANCH 116 // If the commit we were handed is not what the SCM says we are using, fail 117 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 118 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 119 exit 1 120 } 121 } 122 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 123 } 124 125 script { 126 echo "Generate git tags and save it to tags.txt file in the workspace" 127 sh """ 128 cd ${workspace} 129 git tag | awk '/v1[.]/' > tags.txt 130 cat tags.txt 131 git ls-remote --heads https://github.com/verrazzano/verrazzano.git 'refs/heads/release-*' | cut -d '/' -f 3 | cut -d '-' -f 2 | grep -v '0' > releaseBranches.txt 132 cat releaseBranches.txt 133 """ 134 def props = readProperties file: '.verrazzano-development-version' 135 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 136 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 137 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 138 // update the description with some meaningful info 139 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 140 def currentCommitHash = env.GIT_COMMIT 141 def commitList = getCommitList() 142 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 143 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 144 SUSPECT_LIST = getSuspectList(commitList, userMappings) 145 echo "Suspect list: ${SUSPECT_LIST}" 146 } 147 } 148 } 149 } 150 151 stage ('Upgrade every release to next Minor Release') { 152 parallel { 153 stage('Compute required N Upgrade Jobs') { 154 steps { 155 script { 156 // Extract list of releases from git tags 157 final String fileContent = readFile(file: "${workspace}/tags.txt") 158 List gitTags = extractReleaseTags(fileContent) 159 echo "gitTags = ${gitTags}" 160 def finalReleaseList = removeExcludedReleases(gitTags) 161 echo "List of Releases after excluding the user requested releases: ${finalReleaseList}" 162 listOfUpgradeJobs = addTargetUpgradeVersion(finalReleaseList) 163 164 //Extract list of releases from git branches 165 fileContent = readFile(file: "${workspace}/releaseBranches.txt") 166 List releaseBranches = extractReleaseTags(fileContent) 167 echo "Release Branches: ${releaseBranches}" 168 List finalReleaseBranchesList = removeExcludedReleaseBranches(releaseBranches) 169 170 def NJObs = Integer.parseInt(params.N_JOBS_FOR_EACH_BATCH) 171 echo "Number of jobs to be run in parallel : ${params.N_JOBS_FOR_EACH_BATCH}" 172 def k = 0 173 def mapOfJobsInBatches = [] 174 upgradeJobsStageMapping = [:] 175 for(int i=0;i<listOfUpgradeJobs.size();i++){ 176 if(k<NJObs){ 177 upgradeJobsStageMapping.put(listOfUpgradeJobs.get(i),getStageOfUpgradeJob(listOfUpgradeJobs.get(i))) 178 k++; 179 } 180 if(k==NJObs || i == listOfUpgradeJobs.size()-1){ 181 mapOfJobsInBatches.add(upgradeJobsStageMapping) 182 upgradeJobsStageMapping = [:] 183 k = 0; 184 } 185 } 186 print mapOfJobsInBatches 187 for(batch in mapOfJobsInBatches){ 188 parallel batch 189 sleep 5 190 } 191 } 192 } 193 } 194 } 195 } 196 } 197 post { 198 failure { 199 script { 200 if (env.JOB_NAME == "verrazzano-push-triggered-acceptance-tests/master" || env.JOB_NAME ==~ "verrazzano-push-triggered-acceptance-tests/release-1.*") { 201 if (isPagerDutyEnabled()) { 202 pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", incDescription: "Verrazzano: ${env.JOB_NAME} - Failed", incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}") 203 } 204 // slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}" ) 205 } 206 } 207 } 208 } 209 } 210 211 def isPagerDutyEnabled() { 212 // this controls whether PD alerts are enabled 213 if (NOTIFY_PAGERDUTY_TRIGGERED_FAILURES.equals("true")) { 214 echo "Pager-Duty notifications enabled via global override setting" 215 return true 216 } 217 return false 218 } 219 220 // Called in Stage Clean workspace and checkout steps 221 @NonCPS 222 def getCommitList() { 223 echo "Checking for change sets" 224 def commitList = [] 225 def changeSets = currentBuild.changeSets 226 for (int i = 0; i < changeSets.size(); i++) { 227 echo "get commits from change set" 228 def commits = changeSets[i].items 229 for (int j = 0; j < commits.length; j++) { 230 def commit = commits[j] 231 def id = commit.commitId 232 echo "Add commit id: ${id}" 233 commitList.add(id) 234 } 235 } 236 return commitList 237 } 238 239 def trimIfGithubNoreplyUser(userIn) { 240 if (userIn == null) { 241 echo "Not a github noreply user, not trimming: ${userIn}" 242 return userIn 243 } 244 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 245 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 246 return userOut; 247 } 248 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 249 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 250 return userOut; 251 } 252 if (userIn.matches(".*@users.noreply.github.com")) { 253 def userOut = userIn.substring(0, userIn.indexOf("@")) 254 return userOut; 255 } 256 echo "Not a github noreply user, not trimming: ${userIn}" 257 return userIn 258 } 259 260 def getSuspectList(commitList, userMappings) { 261 def retValue = "" 262 def suspectList = [] 263 if (commitList == null || commitList.size() == 0) { 264 echo "No commits to form suspect list" 265 } else { 266 for (int i = 0; i < commitList.size(); i++) { 267 def id = commitList[i] 268 try { 269 def gitAuthor = sh( 270 script: "git log --format='%ae' '$id^!'", 271 returnStdout: true 272 ).trim() 273 if (gitAuthor != null) { 274 def author = trimIfGithubNoreplyUser(gitAuthor) 275 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 276 if (userMappings.containsKey(author)) { 277 def slackUser = userMappings.get(author) 278 if (!suspectList.contains(slackUser)) { 279 echo "Added ${slackUser} as suspect" 280 retValue += " ${slackUser}" 281 suspectList.add(slackUser) 282 } 283 } else { 284 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 285 if (!suspectList.contains(author)) { 286 echo "Added ${author} as suspect" 287 retValue += " ${author}" 288 suspectList.add(author) 289 } 290 } 291 } else { 292 echo "No author returned from git" 293 } 294 } catch (Exception e) { 295 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 296 } 297 } 298 } 299 def startedByUser = ""; 300 def causes = currentBuild.getBuildCauses() 301 echo "causes: " + causes.toString() 302 for (cause in causes) { 303 def causeString = cause.toString() 304 echo "current cause: " + causeString 305 def causeInfo = readJSON text: causeString 306 if (causeInfo.userId != null) { 307 startedByUser = causeInfo.userId 308 } 309 } 310 311 if (startedByUser.length() > 0) { 312 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 313 def author = trimIfGithubNoreplyUser(startedByUser) 314 echo "DEBUG: author: ${startedByUser}, ${author}" 315 if (userMappings.containsKey(author)) { 316 def slackUser = userMappings.get(author) 317 if (!suspectList.contains(slackUser)) { 318 echo "Added ${slackUser} as suspect" 319 retValue += " ${slackUser}" 320 suspectList.add(slackUser) 321 } 322 } else { 323 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 324 if (!suspectList.contains(author)) { 325 echo "Added ${author} as suspect" 326 retValue += " ${author}" 327 suspectList.add(author) 328 } 329 } 330 } else { 331 echo "Build not started by a user, not adding to notification list" 332 } 333 echo "returning suspect list: ${retValue}" 334 return retValue 335 } 336 337 def List<List> addTargetUpgradeVersion(releases){ 338 upgradeJobsList = [] 339 for (release in releases){ 340 upgradeJobsList.add([release, TARGET_UPGRADE_VERSION]) 341 } 342 println("UPGRADE JOB LIST DEBUG" + upgradeJobsList) 343 return upgradeJobsList 344 } 345 346 @NonCPS 347 List extractReleaseTags(final String fileContent) { 348 List releases = [] 349 fileContent.eachLine { tag -> 350 releases << tag 351 } 352 return releases 353 } 354 355 def getTipOfReleaseBranches(releaseNumber){ 356 completeReleaseVersionCommit = "release-" + releaseNumber 357 scmReleaseBranchInfo = checkout([ 358 $class: 'GitSCM', 359 branches: [[name: completeReleaseVersionCommit]], 360 doGenerateSubmoduleConfigurations: false, 361 extensions: [], 362 submoduleCfg: [], 363 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 364 def releaseBranchCommitSHA = scmReleaseBranchInfo.GIT_COMMIT 365 return releaseBranchCommitSHA 366 } 367 368 // Remove the excluded releases from jobs 369 def removeExcludedReleases(List releases){ 370 echo "List of release branches inside function: ${releases}" 371 def excludeReleases = params.EXCLUDE_RELEASES 372 def excludeReleasesList = excludeReleases.trim().split('\\s*,\\s*') 373 println(excludeReleasesList) 374 def finalReleaseString = sh(returnStdout: true, script: "go run ${WORKSPACE}/ci/tools/derive_upgrade_version.go ${workspace} versions-lt ${VERRAZZANO_DEV_VERSION} ${excludeReleasesList}").trim() 375 def finalReleaseList = finalReleaseString.split() 376 echo "Final releases: ${finalReleaseList}" 377 return finalReleaseList 378 } 379 380 // Remove the excluded releases from jobs 381 def removeExcludedReleaseBranches(List releases){ 382 echo "List of release branches inside function: ${releases}" 383 def excludeReleases = params.EXCLUDE_RELEASES 384 def excludeReleasesList = excludeReleases.trim().split('\\s*,\\s*') 385 println(excludeReleasesList) 386 def finalReleaseList = [] 387 388 for(releaseTag in releases){ 389 def excluded = false 390 for(excludedRelease in excludeReleasesList){ 391 String[] splitReleaseTag = releaseTag.replace('v', '').split("\\.") 392 def releaseTagMinorVersion = splitReleaseTag[0] + "." + splitReleaseTag[1] 393 if (releaseTagMinorVersion == excludedRelease || "v"+releaseTagMinorVersion == excludedRelease) { 394 excluded = true 395 } 396 } 397 if(excluded == false){ 398 finalReleaseList.add(releaseTag) 399 } 400 } 401 echo "Final releases: ${finalReleaseList}" 402 return finalReleaseList 403 } 404 405 def getStageOfUpgradeJob(job) { 406 return { 407 stage("Upgrade Verrazzano from version ${job[0]} to ${job[1].replace('_', " ")}"){ 408 script { 409 try { 410 echo "Running upgrade job from version ${job[0]} to ${job[1].replace('_', " ")}" 411 retry(count: JOB_PROMOTION_RETRIES) { 412 def jobStatus = build job: "/verrazzano-upgrade-path-tests/${CLEAN_BRANCH_NAME}", 413 parameters: [ 414 string(name: 'GIT_COMMIT_TO_USE', value: params.GIT_COMMIT_TO_USE), 415 string(name: 'VERSION_FOR_INSTALL', value: job[0]), 416 string(name: 'VERSION_FOR_UPGRADE', value: job[1]), 417 string(name: 'VZ_INSTALL_CONFIG', value: params.VZ_INSTALL_CONFIG), 418 string(name: 'IS_TRIGGERED_MANUALLY', value: "NO"), 419 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 420 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 421 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 422 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 423 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 424 string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH) 425 ], wait: true, propagate: true 426 } 427 }catch(err){ 428 catchError(message: "${STAGE_NAME} Failed with ${err}", buildResult: 'FAILURE', stageResult: 'FAILURE'){ 429 echo "Caught: ${err}" 430 sh "exit 1" 431 } 432 } 433 } 434 } 435 } 436 }