github.com/verrazzano/verrazzano@v1.7.1/ci/periodic/JenkinsfileDistributions (about) 1 // Copyright (c) 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 import groovy.transform.Field 5 6 @Field def verrazzanoPrefix="verrazzano-" 7 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 8 9 pipeline { 10 options { 11 timeout(time: 4, unit: 'HOURS') 12 skipDefaultCheckout true 13 timestamps () 14 } 15 16 agent { 17 docker { 18 image "${RUNNER_DOCKER_IMAGE}" 19 args "${RUNNER_DOCKER_ARGS}" 20 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 21 registryCredentialsId 'ocir-pull-and-push-account' 22 label "${agentLabel}" 23 } 24 } 25 26 parameters { 27 string (name: 'GIT_COMMIT_TO_USE', 28 defaultValue: 'NONE', 29 description: 'This is the full git commit hash from the source build to be used for all jobs', 30 trim: true) 31 string (name: 'TAGGED_TESTS', 32 defaultValue: '', 33 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 34 trim: true) 35 string (name: 'INCLUDED_TESTS', 36 defaultValue: '.*', 37 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 38 trim: true) 39 string (name: 'EXCLUDED_TESTS', 40 defaultValue: '_excluded_test', 41 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 42 trim: true) 43 booleanParam (name: 'BASELINE_IMAGE_SIZES_FILE_UPDATE', 44 defaultValue: false, 45 description: 'If true, image-sizes.txt file will be updated to reflect latest image sizes. GIT_COMMIT_TO_USE is required when BASELINE_IMAGE_SIZES_FILE_UPDATE is set to true.') 46 booleanParam (name: 'RUN_TESTS', 47 defaultValue: false, 48 description: 'If true, the private registry pipeline will run on a development branch. It will automatically run on master and release-* branches.') 49 } 50 51 environment { 52 IS_PERIODIC_PIPELINE = "true" 53 NETRC_FILE = credentials('netrc') 54 GOPATH = '/home/opc/go' 55 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 56 57 OCIR_SCAN_COMPARTMENT = credentials('ocir-scan-compartment') 58 OCIR_SCAN_TARGET = credentials('ocir-scan-target') 59 OCIR_SCAN_REGISTRY = credentials('ocir-scan-registry') 60 OCIR_SCAN_REPOSITORY_PATH = credentials('ocir-scan-repository-path') 61 DOCKER_SCAN_CREDS = credentials('v8odev-ocir') 62 DOCKER_CREDS = credentials('github-packages-credentials-rw') 63 DOCKER_EMAIL = credentials('github-packages-email') 64 DOCKER_REPO = 'ghcr.io' 65 OCR_CREDS = credentials('ocr-pull-and-push-account') 66 OCR_REPO = 'container-registry.oracle.com' 67 68 OCI_CLI_AUTH="instance_principal" 69 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 70 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 71 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 72 73 STABLE_COMMIT_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-stable-commit.txt" 74 LAST_PERIODIC_OS_LOCATION = "${CLEAN_BRANCH_NAME}/last-periodic-run-commit.txt" 75 CLEAN_PERIODIC_OS_LOCATION = "${CLEAN_BRANCH_NAME}-last-clean-periodic-test/verrazzano_periodic-commit.txt" 76 77 STABLE_COMMIT_LOCATION = "${WORKSPACE}/last-stable-commit.txt" 78 LAST_PERIODIC_LOCATION = "${WORKSPACE}/last-periodic-run-commit.txt" 79 CLEAN_PERIODIC_LOCATION = "${WORKSPACE}/last-clean-periodic-commit.txt" 80 81 OCI_OS_REGION="us-phoenix-1" // where to download existing artifacts from 82 OCI_OS_DIST_REGION="eu-frankfurt-1" // where to upload distributions to 83 84 BASELINE_COMMIT_SHORT_HASH="" 85 COMPARISON_URL_ON_FAILURE="" 86 } 87 88 stages { 89 stage('Clean workspace and checkout') { 90 steps { 91 cleanWorkspaceAndCheckout() 92 } 93 } 94 95 stage('Build Release Distributions') { 96 when { 97 anyOf { 98 expression { params.BASELINE_IMAGE_SIZES_FILE_UPDATE == false }; 99 } 100 } 101 steps { 102 script { 103 try { 104 sh """ 105 echo "${OCR_CREDS_PSW}" | docker login -u ${OCR_CREDS_USR} ${OCR_REPO} --password-stdin 106 """ 107 } 108 catch(error) { 109 echo "OCIR docker login at ${OCIR_REPO} failed, retrying after sleep" 110 retry(4) { 111 sleep(30) 112 sh """ 113 echo "${OCR_CREDS_PSW}" | docker login -u ${OCR_CREDS_USR} ${OCR_REPO} --password-stdin 114 """ 115 } 116 } 117 } 118 sh """ 119 ci/scripts/build_distribution.sh ${env.GIT_COMMIT} ${SHORT_COMMIT_HASH} ${VERRAZZANO_DEV_VERSION} 120 121 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ${CLEAN_BRANCH_NAME}/${SHORT_COMMIT_HASH}/image-list --file ${WORKSPACE}/image-sizes.txt 122 """ 123 } 124 post { 125 always { 126 archiveArtifacts artifacts: 'image-sizes.txt', allowEmptyArchive: true 127 } 128 } 129 } 130 131 stage('Update Baseline Image-Sizes File') { 132 when { 133 anyOf { 134 expression { params.BASELINE_IMAGE_SIZES_FILE_UPDATE == true }; 135 } 136 } 137 steps { 138 script { 139 currentBuild.displayName = "${currentBuild.displayName} : Update Baseline image-sizes.txt" 140 if (params.GIT_COMMIT_TO_USE == "NONE") { 141 error("Specific GIT commit was not specified. GIT_COMMIT_TO_USE is required when BASELINE_IMAGE_SIZES_FILE_UPDATE is set to true.") 142 } 143 sh "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ${CLEAN_BRANCH_NAME}/${SHORT_COMMIT_HASH}/image-list --file ${WORKSPACE}/image-sizes.txt" 144 sh "oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_BUCKET} --name ${CLEAN_BRANCH_NAME}/image-list --file ${WORKSPACE}/image-sizes.txt" 145 } 146 } 147 } 148 149 stage('Check Image Sizes') { 150 when { 151 anyOf { 152 expression { params.BASELINE_IMAGE_SIZES_FILE_UPDATE == false }; 153 } 154 } 155 steps { 156 script { 157 sh """ 158 ci/scripts/check_image_sizes.sh 159 """ 160 if (NOTIFY_IMAGE_SIZE_INCREASE_FAILURES.equals("true") && (env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-"))) { 161 BASELINE_COMMIT_SHORT_HASH = sh(returnStdout: true, script: "cat ${WORKSPACE}/commitID.txt") 162 if ( fileExists("${WORKSPACE}/image-increases.txt") ){ 163 def commitList = getCommitList() 164 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 165 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 166 SUSPECT_LIST = getSuspectList(commitList, userMappings) 167 echo "Suspect list: ${SUSPECT_LIST}" 168 } 169 COMPARISON_URL_ON_FAILURE = "https://github.com/verrazzano/verrazzano/compare/${BASELINE_COMMIT_SHORT_HASH}...${env.GIT_COMMIT}" 170 slackSend ( channel: "$SLACK_PERF_ALERT_CHANNEL", message: "Image Size Has Increased - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}\n\nChange comparison: ${COMPARISON_URL_ON_FAILURE}" ) 171 } 172 } 173 } 174 } 175 post { 176 always { 177 archiveArtifacts artifacts: 'image-sizes-objectstore.txt', allowEmptyArchive: true 178 archiveArtifacts artifacts: 'image-increases.txt', allowEmptyArchive: true 179 } 180 } 181 } 182 183 stage('Private Registry Tests') { 184 when { 185 allOf { 186 expression { params.BASELINE_IMAGE_SIZES_FILE_UPDATE == false }; 187 anyOf { 188 branch 'master'; 189 branch 'release-*'; 190 expression { params.RUN_TESTS == true }; 191 } 192 } 193 } 194 parallel { 195 stage('Private Registry - Lite Distribution') { 196 steps { 197 retry(count: JOB_PROMOTION_RETRIES) { 198 script { 199 build job: "verrazzano-private-registry/${CLEAN_BRANCH_NAME}", 200 parameters: [ 201 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 202 string(name: 'DISTRIBUTION_VARIANT', value: 'Lite'), 203 string(name: 'ZIPFILE_LOCATION', value: liteBundle) 204 ], wait: true 205 } 206 } 207 } 208 } 209 210 stage('Private Registry - Full Distribution') { 211 steps { 212 retry(count: JOB_PROMOTION_RETRIES) { 213 script { 214 build job: "verrazzano-private-registry/${CLEAN_BRANCH_NAME}", 215 parameters: [ 216 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 217 string(name: 'DISTRIBUTION_VARIANT', value: 'Full'), 218 string(name: 'ZIPFILE_LOCATION', value: fullBundle) 219 ], wait: true 220 } 221 } 222 } 223 } 224 225 stage('Start Asynchronous Scan Distributions') { 226 // Only release-* branches scan by default, all other branches including master must enable RUN_TESTS explicitly to get scanning to be done. 227 when { 228 anyOf { 229 branch 'release-*'; 230 expression { params.RUN_TESTS == true }; 231 } 232 } 233 steps { 234 script { 235 build job: "verrazzano-scan-distribution/${CLEAN_BRANCH_NAME}", 236 parameters: [ 237 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 238 string(name: 'VERRAZZANO_DEV_VERSION', value: "${VERRAZZANO_DEV_VERSION}" ), 239 booleanParam(name: 'TEST_RUN', value: false) 240 ], wait: false 241 } 242 } 243 } 244 245 stage('Push to OCIR for scanning') { 246 environment { 247 OCI_CLI_AUTH="api_key" 248 OCI_CLI_TENANCY = credentials('oci-dev-tenancy') 249 OCI_CLI_USER = credentials('oci-dev-user-ocid') 250 OCI_CLI_FINGERPRINT = credentials('oci-dev-api-key-fingerprint') 251 OCI_CLI_KEY_FILE = credentials('oci-dev-api-key-file') 252 OCI_CLI_REGION = "us-ashburn-1" 253 OCI_REGION = "${env.OCI_CLI_REGION}" 254 // Directory containing the Verrazzano image tar files 255 VERRAZZANO_IMAGES_DIRECTORY = "${WORKSPACE}/vz-full/verrazzano-${VERRAZZANO_DEV_VERSION}/images" 256 } 257 steps { 258 script { 259 sh """ 260 echo "Pushing images to OCIR, these are treated as the latest scan results for ${CLEAN_BRANCH_NAME}" 261 ci/scripts/push_to_ocir.sh 262 """ 263 } 264 } 265 } 266 } 267 } 268 } 269 } 270 271 // Called in Stage Clean workspace and checkout steps 272 @NonCPS 273 def getCommitList() { 274 echo "Checking for change sets" 275 def commitList = [] 276 def changeSets = currentBuild.changeSets 277 for (int i = 0; i < changeSets.size(); i++) { 278 echo "get commits from change set" 279 def commits = changeSets[i].items 280 for (int j = 0; j < commits.length; j++) { 281 def commit = commits[j] 282 def id = commit.commitId 283 echo "Add commit id: ${id}" 284 commitList.add(id) 285 } 286 } 287 return commitList 288 } 289 290 def trimIfGithubNoreplyUser(userIn) { 291 if (userIn == null) { 292 echo "Not a github noreply user, not trimming: ${userIn}" 293 return userIn 294 } 295 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 296 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 297 return userOut; 298 } 299 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 300 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 301 return userOut; 302 } 303 if (userIn.matches(".*@users.noreply.github.com")) { 304 def userOut = userIn.substring(0, userIn.indexOf("@")) 305 return userOut; 306 } 307 echo "Not a github noreply user, not trimming: ${userIn}" 308 return userIn 309 } 310 311 def getSuspectList(commitList, userMappings) { 312 def retValue = "" 313 def suspectList = [] 314 if (commitList == null || commitList.size() == 0) { 315 echo "No commits to form suspect list" 316 } else { 317 for (int i = 0; i < commitList.size(); i++) { 318 def id = commitList[i] 319 try { 320 def gitAuthor = sh( 321 script: "git log --format='%ae' '$id^!'", 322 returnStdout: true 323 ).trim() 324 if (gitAuthor != null) { 325 def author = trimIfGithubNoreplyUser(gitAuthor) 326 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 327 if (userMappings.containsKey(author)) { 328 def slackUser = userMappings.get(author) 329 if (!suspectList.contains(slackUser)) { 330 echo "Added ${slackUser} as suspect" 331 retValue += " ${slackUser}" 332 suspectList.add(slackUser) 333 } 334 } else { 335 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 336 if (!suspectList.contains(author)) { 337 echo "Added ${author} as suspect" 338 retValue += " ${author}" 339 suspectList.add(author) 340 } 341 } 342 } else { 343 echo "No author returned from git" 344 } 345 } catch (Exception e) { 346 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 347 } 348 } 349 } 350 def startedByUser = ""; 351 def causes = currentBuild.getBuildCauses() 352 echo "causes: " + causes.toString() 353 for (cause in causes) { 354 def causeString = cause.toString() 355 echo "current cause: " + causeString 356 def causeInfo = readJSON text: causeString 357 if (causeInfo.userId != null) { 358 startedByUser = causeInfo.userId 359 } 360 } 361 362 if (startedByUser.length() > 0) { 363 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 364 def author = trimIfGithubNoreplyUser(startedByUser) 365 echo "DEBUG: author: ${startedByUser}, ${author}" 366 if (userMappings.containsKey(author)) { 367 def slackUser = userMappings.get(author) 368 if (!suspectList.contains(slackUser)) { 369 echo "Added ${slackUser} as suspect" 370 retValue += " ${slackUser}" 371 suspectList.add(slackUser) 372 } 373 } else { 374 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 375 if (!suspectList.contains(author)) { 376 echo "Added ${author} as suspect" 377 retValue += " ${author}" 378 suspectList.add(author) 379 } 380 } 381 } else { 382 echo "Build not started by a user, not adding to notification list" 383 } 384 echo "returning suspect list: ${retValue}" 385 return retValue 386 } 387 388 def setDisplayName() { 389 echo "Start setDisplayName" 390 def causes = currentBuild.getBuildCauses() 391 echo "causes: " + causes.toString() 392 for (cause in causes) { 393 def causeString = cause.toString() 394 echo "current cause: " + causeString 395 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 396 echo "This job was caused by " + causeString 397 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 398 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 399 } else if (causeString.contains("verrazzano-flaky-tests")) { 400 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 401 } 402 } 403 } 404 echo "End setDisplayName" 405 } 406 407 def dockerLogins() { 408 try { 409 sh """ 410 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 411 """ 412 } catch(error) { 413 echo "docker login failed, retrying after sleep" 414 retry(4) { 415 sleep(30) 416 sh """ 417 echo "${DOCKER_SCAN_CREDS_PSW}" | docker login ${env.OCIR_SCAN_REGISTRY} -u ${DOCKER_SCAN_CREDS_USR} --password-stdin 418 """ 419 } 420 } 421 if (!(env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-1."))) { 422 try { 423 sh """ 424 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 425 """ 426 } catch(error) { 427 echo "docker login failed, retrying after sleep" 428 retry(4) { 429 sleep(30) 430 sh """ 431 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 432 """ 433 } 434 } 435 } 436 } 437 438 def scmCheckout() { 439 if (params.GIT_COMMIT_TO_USE == "NONE") { 440 echo "Specific GIT commit was not specified, use current head" 441 def scmInfo = checkout scm 442 env.GIT_COMMIT = scmInfo.GIT_COMMIT 443 env.GIT_BRANCH = scmInfo.GIT_BRANCH 444 } else { 445 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 446 def scmInfo = checkout([ 447 $class: 'GitSCM', 448 branches: [[name: params.GIT_COMMIT_TO_USE]], 449 doGenerateSubmoduleConfigurations: false, 450 extensions: [], 451 submoduleCfg: [], 452 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 453 env.GIT_COMMIT = scmInfo.GIT_COMMIT 454 env.GIT_BRANCH = scmInfo.GIT_BRANCH 455 // If the commit we were handed is not what the SCM says we are using, fail 456 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 457 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 458 exit 1 459 } 460 } 461 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 462 } 463 464 def cleanWorkspaceAndCheckout() { 465 scmCheckout() 466 467 sh """ 468 cp -f "${NETRC_FILE}" $HOME/.netrc 469 chmod 600 $HOME/.netrc 470 """ 471 472 dockerLogins() 473 474 sh """ 475 rm -rf ${GO_REPO_PATH}/verrazzano 476 mkdir -p ${GO_REPO_PATH}/verrazzano 477 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 478 cd ${GO_REPO_PATH}/verrazzano 479 git config --global credential.helper "!f() { echo username=\\$DOCKER_CREDS_USR; echo password=\\$DOCKER_CREDS_PSW; }; f" 480 git config --global user.name $DOCKER_CREDS_USR 481 git config --global user.email "${DOCKER_EMAIL}" 482 git checkout -b ${env.BRANCH_NAME} 483 """ 484 485 def props = readProperties file: '.verrazzano-development-version' 486 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 487 488 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 489 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 490 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 491 492 // update the description with some meaningful info 493 setDisplayName() 494 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 495 496 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 497 storeLocation="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 498 fullBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}.zip" 499 liteBundle="${storeLocation}/${verrazzanoPrefix}${VERRAZZANO_DEV_VERSION}-lite.zip" 500 }