github.com/verrazzano/verrazzano@v1.7.0/ci/dynamic-updates/JenkinsfileTrigger (about) 1 // Copyright (c) 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 pipeline { 5 options { 6 skipDefaultCheckout true 7 timestamps () 8 } 9 10 agent { 11 docker { 12 image "${RUNNER_DOCKER_IMAGE}" 13 args "${RUNNER_DOCKER_ARGS}" 14 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 15 registryCredentialsId 'ocir-pull-and-push-account' 16 label "pipeline-job-large" 17 } 18 } 19 20 parameters { 21 string (name: 'GIT_COMMIT_TO_USE', 22 defaultValue: 'NONE', 23 description: 'This is the full git commit hash from the source build to be used for all jobs. A full pipeline specifies a valid commit hash here. NONE can be used for manually triggered jobs, however even for those a commit hash value is preferred to be supplied', 24 trim: true) 25 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 26 defaultValue: 'NONE', 27 description: 'This is for manually testing only where someone needs to use a specific operator image, otherwise the default value of NONE is used', 28 trim: true) 29 string (name: 'WILDCARD_DNS_DOMAIN', 30 defaultValue: 'nip.io', 31 description: 'This is the wildcard DNS domain', 32 trim: true) 33 string (name: 'TAGGED_TESTS', 34 defaultValue: '', 35 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 36 trim: true) 37 string (name: 'INCLUDED_TESTS', 38 defaultValue: '.*', 39 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 40 trim: true) 41 string (name: 'EXCLUDED_TESTS', 42 defaultValue: '_excluded_test', 43 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 44 trim: true) 45 string (name: 'CONSOLE_REPO_BRANCH', 46 defaultValue: '', 47 description: 'The branch to check out after cloning the console repository.', 48 trim: true) 49 } 50 51 environment { 52 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 53 GOPATH = '/home/opc/go' 54 PROMETHEUS_GW_URL = credentials('prometheus-dev-url') 55 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 56 } 57 58 stages { 59 stage('Clean workspace and checkout') { 60 steps { 61 script { 62 if (params.GIT_COMMIT_TO_USE == "NONE") { 63 echo "Specific GIT commit was not specified, use current head" 64 def scmInfo = checkout([ 65 $class: 'GitSCM', 66 branches: [[name: env.BRANCH_NAME]], 67 doGenerateSubmoduleConfigurations: false, 68 extensions: [], 69 submoduleCfg: [], 70 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 71 env.GIT_COMMIT = scmInfo.GIT_COMMIT 72 env.GIT_BRANCH = scmInfo.GIT_BRANCH 73 } else { 74 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 75 def scmInfo = checkout([ 76 $class: 'GitSCM', 77 branches: [[name: params.GIT_COMMIT_TO_USE]], 78 doGenerateSubmoduleConfigurations: false, 79 extensions: [], 80 submoduleCfg: [], 81 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 82 env.GIT_COMMIT = scmInfo.GIT_COMMIT 83 env.GIT_BRANCH = scmInfo.GIT_BRANCH 84 // If the commit we were handed is not what the SCM says we are using, fail 85 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 86 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 87 exit 1 88 } 89 } 90 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 91 } 92 93 script { 94 def props = readProperties file: '.verrazzano-development-version' 95 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 96 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 97 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 98 // update the description with some meaningful info 99 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 100 def currentCommitHash = env.GIT_COMMIT 101 def commitList = getCommitList() 102 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 103 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 104 SUSPECT_LIST = getSuspectList(commitList, userMappings) 105 echo "Suspect list: ${SUSPECT_LIST}" 106 } 107 } 108 } 109 } 110 111 stage ('Kick off dynamic update tests') { 112 parallel { 113 stage ('Post-Update1, Post-Update2, and Nginx/Istio Update Tests') { 114 steps { 115 retry (count: JOB_PROMOTION_RETRIES) { 116 script { 117 startDCTestJob([booleanParam(name: 'RUN_POST_UPDATE1', value: true), 118 booleanParam(name: 'RUN_POST_UPDATE2', value: true), 119 booleanParam(name: 'RUN_INFRA_TESTS', value: true), 120 booleanParam(name: 'RUN_NGINX_ISTIO', value: true)]) 121 } 122 } 123 } 124 } 125 stage ('Availability Status, AuthProxy, CertManager, and Fluentd Update Tests') { 126 steps { 127 retry (count: JOB_PROMOTION_RETRIES) { 128 script { 129 startDCTestJob([booleanParam(name: 'RUN_INFRA_TESTS', value: true), 130 booleanParam(name: 'RUN_AVAILABILITY_STATUS', value: true), 131 booleanParam(name: 'RUN_AUTHPROXY', value: true), 132 booleanParam(name: 'RUN_CERT_MANAGER', value: true), 133 booleanParam(name: 'RUN_FLUENTD', value: true), 134 booleanParam(name: 'RUN_FLUENT_OPERATOR', value: true)]) 135 } 136 } 137 } 138 } 139 stage ('Availability Status, AuthProxy, CertManager, and Fluentd Update Tests With External cert-manager') { 140 steps { 141 retry (count: JOB_PROMOTION_RETRIES) { 142 script { 143 startDCTestJob([booleanParam(name: 'RUN_INFRA_TESTS', value: true), 144 booleanParam(name: 'RUN_AVAILABILITY_STATUS', value: true), 145 booleanParam(name: 'RUN_AUTHPROXY', value: true), 146 booleanParam(name: 'RUN_CERT_MANAGER', value: true), 147 booleanParam(name: 'RUN_FLUENTD', value: true), 148 booleanParam(name: 'EXTERNAL_CERT_MANAGER', value: true), 149 string(name: 'CLUSTER_RESOURCE_NAMESPACE', value: "test-namespace")]) 150 } 151 } 152 } 153 } 154 stage ('API Conversion Update, Opensearch Update, Jaeger Update, and Post-install Overrides Tests') { 155 steps { 156 retry (count: JOB_PROMOTION_RETRIES) { 157 script { 158 startDCTestJob([booleanParam(name: 'RUN_INFRA_TESTS', value: true), 159 booleanParam(name: 'RUN_API_CONVERSION', value: true), 160 booleanParam(name: 'RUN_OPENSEARCH', value: true), 161 booleanParam(name: 'RUN_JAEGER', value: true), 162 booleanParam(name: 'RUN_POST_INSTALL_OVERRIDES', value: true)]) 163 } 164 } 165 } 166 } 167 168 stage ('Update during Keycloak Install') { 169 steps { 170 retry (count: JOB_PROMOTION_RETRIES) { 171 script { 172 startDCInstallTestJob([string(name: 'WAIT_FOR_RESOURCE_BEFORE_UPDATE', value: 'mysql statefulset')]) 173 } 174 } 175 } 176 } 177 stage ('Update during Rancher Install') { 178 steps { 179 retry (count: JOB_PROMOTION_RETRIES) { 180 script { 181 startDCInstallTestJob([string(name: 'WAIT_FOR_RESOURCE_BEFORE_UPDATE', value: 'rancher bootstrap secret')]) 182 } 183 } 184 } 185 } 186 } 187 } 188 } 189 190 post { 191 failure { 192 script { 193 if (env.JOB_NAME == "verrazzano-dynamic-config-suite/master" || env.JOB_NAME ==~ "verrazzano-dynamic-config-suite/release-1.*") { 194 if (isPagerDutyEnabled()) { 195 pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", incDescription: "Verrazzano: ${env.JOB_NAME} - Failed", incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}") 196 } 197 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}" ) 198 } 199 } 200 } 201 } 202 } 203 204 def isPagerDutyEnabled() { 205 // this controls whether PD alerts are enabled 206 if (NOTIFY_PAGERDUTY_TRIGGERED_FAILURES.equals("true")) { 207 echo "Pager-Duty notifications enabled via global override setting" 208 return true 209 } 210 return false 211 } 212 213 // Called in Stage Clean workspace and checkout steps 214 @NonCPS 215 def getCommitList() { 216 echo "Checking for change sets" 217 def commitList = [] 218 def changeSets = currentBuild.changeSets 219 for (int i = 0; i < changeSets.size(); i++) { 220 echo "get commits from change set" 221 def commits = changeSets[i].items 222 for (int j = 0; j < commits.length; j++) { 223 def commit = commits[j] 224 def id = commit.commitId 225 echo "Add commit id: ${id}" 226 commitList.add(id) 227 } 228 } 229 return commitList 230 } 231 232 def trimIfGithubNoreplyUser(userIn) { 233 if (userIn == null) { 234 echo "Not a github noreply user, not trimming: ${userIn}" 235 return userIn 236 } 237 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 238 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 239 return userOut; 240 } 241 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 242 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 243 return userOut; 244 } 245 if (userIn.matches(".*@users.noreply.github.com")) { 246 def userOut = userIn.substring(0, userIn.indexOf("@")) 247 return userOut; 248 } 249 echo "Not a github noreply user, not trimming: ${userIn}" 250 return userIn 251 } 252 253 def getSuspectList(commitList, userMappings) { 254 def retValue = "" 255 def suspectList = [] 256 if (commitList == null || commitList.size() == 0) { 257 echo "No commits to form suspect list" 258 } else { 259 for (int i = 0; i < commitList.size(); i++) { 260 def id = commitList[i] 261 try { 262 def gitAuthor = sh( 263 script: "git log --format='%ae' '$id^!'", 264 returnStdout: true 265 ).trim() 266 if (gitAuthor != null) { 267 def author = trimIfGithubNoreplyUser(gitAuthor) 268 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 269 if (userMappings.containsKey(author)) { 270 def slackUser = userMappings.get(author) 271 if (!suspectList.contains(slackUser)) { 272 echo "Added ${slackUser} as suspect" 273 retValue += " ${slackUser}" 274 suspectList.add(slackUser) 275 } 276 } else { 277 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 278 if (!suspectList.contains(author)) { 279 echo "Added ${author} as suspect" 280 retValue += " ${author}" 281 suspectList.add(author) 282 } 283 } 284 } else { 285 echo "No author returned from git" 286 } 287 } catch (Exception e) { 288 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 289 } 290 } 291 } 292 def startedByUser = ""; 293 def causes = currentBuild.getBuildCauses() 294 echo "causes: " + causes.toString() 295 for (cause in causes) { 296 def causeString = cause.toString() 297 echo "current cause: " + causeString 298 def causeInfo = readJSON text: causeString 299 if (causeInfo.userId != null) { 300 startedByUser = causeInfo.userId 301 } 302 } 303 304 if (startedByUser.length() > 0) { 305 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 306 def author = trimIfGithubNoreplyUser(startedByUser) 307 echo "DEBUG: author: ${startedByUser}, ${author}" 308 if (userMappings.containsKey(author)) { 309 def slackUser = userMappings.get(author) 310 if (!suspectList.contains(slackUser)) { 311 echo "Added ${slackUser} as suspect" 312 retValue += " ${slackUser}" 313 suspectList.add(slackUser) 314 } 315 } else { 316 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 317 if (!suspectList.contains(author)) { 318 echo "Added ${author} as suspect" 319 retValue += " ${author}" 320 suspectList.add(author) 321 } 322 } 323 } else { 324 echo "Build not started by a user, not adding to notification list" 325 } 326 echo "returning suspect list: ${retValue}" 327 return retValue 328 } 329 330 def startDCTestJob(additionalParams) { 331 def jobParameters = [ 332 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 333 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 334 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 335 string(name: 'CONSOLE_REPO_BRANCH', value: params.CONSOLE_REPO_BRANCH), 336 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 337 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 338 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 339 ] 340 jobParameters = jobParameters + additionalParams 341 342 build job: "/verrazzano-dynamic-config-tests/${CLEAN_BRANCH_NAME}", 343 parameters: jobParameters, wait: true 344 } 345 346 def startDCInstallTestJob(additionalParams) { 347 def jobParameters = [ 348 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 349 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 350 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 351 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 352 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 353 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 354 ] 355 jobParameters = jobParameters + additionalParams 356 357 build job: "/verrazzano-dynamic-config-install-tests/${CLEAN_BRANCH_NAME}", 358 parameters: jobParameters, wait: true 359 }