github.com/verrazzano/verrazzano@v1.7.1/ci/chaos/JenkinsfileResiliencyTrigger (about) 1 // Copyright (c) 2022, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 pipeline { 5 options { 6 skipDefaultCheckout true 7 timestamps () 8 } 9 10 agent { 11 docker { 12 image "${RUNNER_DOCKER_IMAGE}" 13 args "${RUNNER_DOCKER_ARGS}" 14 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 15 registryCredentialsId 'ocir-pull-and-push-account' 16 label "pipeline-job-large" 17 } 18 } 19 20 parameters { 21 string (name: 'GIT_COMMIT_TO_USE', 22 defaultValue: 'NONE', 23 description: 'This is the full git commit hash from the source build to be used for all jobs. A full pipeline specifies a valid commit hash here. NONE can be used for manually triggered jobs, however even for those a commit hash value is preferred to be supplied', 24 trim: true) 25 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 26 defaultValue: 'NONE', 27 description: 'This is for manually testing only where someone needs to use a specific operator image, otherwise the default value of NONE is used', 28 trim: true) 29 string (name: 'WILDCARD_DNS_DOMAIN', 30 defaultValue: 'nip.io', 31 description: 'This is the wildcard DNS domain', 32 trim: true) 33 string (name: 'EXCLUDE_RELEASES', 34 defaultValue: "v1.0.0, v1.0.1, v1.0.2, v1.0.3, v1.0.4, v1.1.0, v1.1.1, v1.1.2, v1.2.0, v1.2.1, v1.2.2," + 35 " v1.3.0, v1.3.1, v1.3.2, v1.3.3, v1.3.4, v1.3.5, v1.3.6, v1.3.7, v1.3.8", 36 description: 'This is to exclude the specified releases from upgrade tests.', trim: true) 37 string (name: 'TAGGED_TESTS', 38 defaultValue: '', 39 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 40 trim: true) 41 string (name: 'INCLUDED_TESTS', 42 defaultValue: '.*', 43 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 44 trim: true) 45 string (name: 'EXCLUDED_TESTS', 46 defaultValue: '_excluded_test', 47 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 48 trim: true) 49 string (name: 'CONSOLE_REPO_BRANCH', 50 defaultValue: '', 51 description: 'The branch to check out after cloning the console repository.', 52 trim: true) 53 } 54 55 environment { 56 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 57 GOPATH = '/home/opc/go' 58 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 59 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 60 61 OCI_CLI_AUTH="instance_principal" 62 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 63 RELEASE_OWNERS = credentials('release-version-owners') 64 } 65 66 stages { 67 stage('Clean workspace and checkout') { 68 steps { 69 script { 70 if (params.GIT_COMMIT_TO_USE == "NONE") { 71 echo "Specific GIT commit was not specified, use current head" 72 def scmInfo = checkout([ 73 $class: 'GitSCM', 74 branches: [[name: env.BRANCH_NAME]], 75 doGenerateSubmoduleConfigurations: false, 76 extensions: [], 77 submoduleCfg: [], 78 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 79 env.GIT_COMMIT = scmInfo.GIT_COMMIT 80 env.GIT_BRANCH = scmInfo.GIT_BRANCH 81 } else { 82 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 83 def scmInfo = checkout([ 84 $class: 'GitSCM', 85 branches: [[name: params.GIT_COMMIT_TO_USE]], 86 doGenerateSubmoduleConfigurations: false, 87 extensions: [], 88 submoduleCfg: [], 89 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 90 env.GIT_COMMIT = scmInfo.GIT_COMMIT 91 env.GIT_BRANCH = scmInfo.GIT_BRANCH 92 // If the commit we were handed is not what the SCM says we are using, fail 93 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 94 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 95 exit 1 96 } 97 } 98 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 99 } 100 101 script { 102 def props = readProperties file: '.verrazzano-development-version' 103 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 104 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 105 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 106 def excludeReleases = params.EXCLUDE_RELEASES 107 def excludeReleasesList = excludeReleases.trim().split('\\s*,\\s*') 108 VERSION_FOR_INSTALL = sh(returnStdout: true, script: "go run ${WORKSPACE}/ci/tools/derive_upgrade_version.go ${workspace} install-version ${excludeReleasesList}").trim() 109 INTERIM_UPGRADE_VERSION = sh(returnStdout: true, script: "go run ${WORKSPACE}/ci/tools/derive_upgrade_version.go ${workspace} interim-version ${excludeReleasesList}").trim() 110 // update the description with some meaningful info 111 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 112 def currentCommitHash = env.GIT_COMMIT 113 def commitList = getCommitList() 114 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 115 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 116 SUSPECT_LIST = getSuspectList(commitList, userMappings) 117 echo "Suspect list: ${SUSPECT_LIST}" 118 } 119 } 120 } 121 } 122 123 stage('Verrazzano development version check') { 124 steps { 125 sh """ 126 [[ -z \$(git ls-remote --tags origin | grep -F ${VERRAZZANO_DEV_VERSION}) ]] || exit 1 127 128 129 """ 130 } 131 post { 132 failure { 133 script { 134 if (env.JOB_NAME ==~ "verrazzano/release-*") { 135 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nVZ Helper was not run, the Verrazzano Development Version ${VERRAZZANO_DEV_VERSION} matches a prior release\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nRelease Owners:\n ${RELEASE_OWNERS}\n") 136 } 137 } 138 } 139 } 140 } 141 142 stage ('Kick off resiliency tests') { 143 parallel { 144 stage('VPO killed during upgrade') { 145 steps { 146 retry(count: JOB_PROMOTION_RETRIES) { 147 script { 148 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 149 parameters: [ 150 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 151 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 152 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 153 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 154 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 155 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 156 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 157 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 158 string(name: 'CHAOS_TEST_TYPE', value: 'vpo.killed') 159 ], wait: true 160 } 161 } 162 } 163 } 164 stage('Upgrade using ephemeral storage') { 165 steps { 166 retry(count: JOB_PROMOTION_RETRIES) { 167 script { 168 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 169 parameters: [ 170 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 171 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 172 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 173 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 174 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 175 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 176 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 177 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 178 string(name: 'CHAOS_TEST_TYPE', value: 'ephemeral.storage.upgrade') 179 ], wait: true 180 } 181 } 182 } 183 } 184 stage('Component helm install failure') { 185 steps { 186 retry(count: JOB_PROMOTION_RETRIES) { 187 script { 188 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 189 parameters: [ 190 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 191 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 192 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 193 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 194 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 195 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 196 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 197 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 198 string(name: 'CHAOS_TEST_TYPE', value: 'helm.chart.corrupted') 199 ], wait: true 200 } 201 } 202 } 203 } 204 stage('Uninstall failed upgrade') { 205 steps { 206 retry(count: JOB_PROMOTION_RETRIES) { 207 script { 208 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 209 parameters: [ 210 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 211 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 212 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 213 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 214 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 215 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 216 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 217 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 218 string(name: 'CHAOS_TEST_TYPE', value: 'uninstall.failed.upgrade') 219 ], wait: true 220 } 221 } 222 } 223 } 224 stage('Upgrade a failed upgrade') { 225 steps { 226 retry(count: JOB_PROMOTION_RETRIES) { 227 script { 228 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 229 parameters: [ 230 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 231 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 232 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 233 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 234 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 235 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 236 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 237 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 238 string(name: 'CHAOS_TEST_TYPE', value: 'upgrade.failed.upgrade') 239 ], wait: true 240 } 241 } 242 } 243 } 244 stage('Upgrade an in-process failing upgrade') { 245 steps { 246 retry(count: JOB_PROMOTION_RETRIES) { 247 script { 248 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 249 parameters: [ 250 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 251 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 252 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 253 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 254 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 255 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 256 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 257 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 258 string(name: 'CHAOS_TEST_TYPE', value: 'upgrade.failing.upgrade') 259 ], wait: true 260 } 261 } 262 } 263 } 264 } 265 } 266 } 267 post { 268 failure { 269 script { 270 if (env.JOB_NAME == "verrazzano-upgrade-resiliency-tests/master" || env.JOB_NAME ==~ "verrazzano-upgrade-resiliency-tests/release-1.*") { 271 if (isPagerDutyEnabled()) { 272 pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", incDescription: "Verrazzano: ${env.JOB_NAME} - Failed", incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}") 273 } 274 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}" ) 275 } 276 } 277 } 278 } 279 } 280 281 def isPagerDutyEnabled() { 282 // this controls whether PD alerts are enabled 283 if (NOTIFY_PAGERDUTY_TRIGGERED_FAILURES.equals("true")) { 284 echo "Pager-Duty notifications enabled via global override setting" 285 return true 286 } 287 return false 288 } 289 290 291 // Called in Stage Clean workspace and checkout steps 292 @NonCPS 293 def getCommitList() { 294 echo "Checking for change sets" 295 def commitList = [] 296 def changeSets = currentBuild.changeSets 297 for (int i = 0; i < changeSets.size(); i++) { 298 echo "get commits from change set" 299 def commits = changeSets[i].items 300 for (int j = 0; j < commits.length; j++) { 301 def commit = commits[j] 302 def id = commit.commitId 303 echo "Add commit id: ${id}" 304 commitList.add(id) 305 } 306 } 307 return commitList 308 } 309 310 def trimIfGithubNoreplyUser(userIn) { 311 if (userIn == null) { 312 echo "Not a github noreply user, not trimming: ${userIn}" 313 return userIn 314 } 315 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 316 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 317 return userOut; 318 } 319 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 320 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 321 return userOut; 322 } 323 if (userIn.matches(".*@users.noreply.github.com")) { 324 def userOut = userIn.substring(0, userIn.indexOf("@")) 325 return userOut; 326 } 327 echo "Not a github noreply user, not trimming: ${userIn}" 328 return userIn 329 } 330 331 def getSuspectList(commitList, userMappings) { 332 def retValue = "" 333 def suspectList = [] 334 if (commitList == null || commitList.size() == 0) { 335 echo "No commits to form suspect list" 336 } else { 337 for (int i = 0; i < commitList.size(); i++) { 338 def id = commitList[i] 339 try { 340 def gitAuthor = sh( 341 script: "git log --format='%ae' '$id^!'", 342 returnStdout: true 343 ).trim() 344 if (gitAuthor != null) { 345 def author = trimIfGithubNoreplyUser(gitAuthor) 346 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 347 if (userMappings.containsKey(author)) { 348 def slackUser = userMappings.get(author) 349 if (!suspectList.contains(slackUser)) { 350 echo "Added ${slackUser} as suspect" 351 retValue += " ${slackUser}" 352 suspectList.add(slackUser) 353 } 354 } else { 355 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 356 if (!suspectList.contains(author)) { 357 echo "Added ${author} as suspect" 358 retValue += " ${author}" 359 suspectList.add(author) 360 } 361 } 362 } else { 363 echo "No author returned from git" 364 } 365 } catch (Exception e) { 366 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 367 } 368 } 369 } 370 def startedByUser = ""; 371 def causes = currentBuild.getBuildCauses() 372 echo "causes: " + causes.toString() 373 for (cause in causes) { 374 def causeString = cause.toString() 375 echo "current cause: " + causeString 376 def causeInfo = readJSON text: causeString 377 if (causeInfo.userId != null) { 378 startedByUser = causeInfo.userId 379 } 380 } 381 382 if (startedByUser.length() > 0) { 383 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 384 def author = trimIfGithubNoreplyUser(startedByUser) 385 echo "DEBUG: author: ${startedByUser}, ${author}" 386 if (userMappings.containsKey(author)) { 387 def slackUser = userMappings.get(author) 388 if (!suspectList.contains(slackUser)) { 389 echo "Added ${slackUser} as suspect" 390 retValue += " ${slackUser}" 391 suspectList.add(slackUser) 392 } 393 } else { 394 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 395 if (!suspectList.contains(author)) { 396 echo "Added ${author} as suspect" 397 retValue += " ${author}" 398 suspectList.add(author) 399 } 400 } 401 } else { 402 echo "Build not started by a user, not adding to notification list" 403 } 404 echo "returning suspect list: ${retValue}" 405 return retValue 406 } 407 408 def getCronSchedule() { 409 if (env.BRANCH_NAME.equals("master")) { 410 return "H */2 * * *" 411 } else if (env.BRANCH_NAME.startsWith("release-1")) { 412 return "@daily" 413 } 414 return "" 415 }