github.com/verrazzano/verrazzano@v1.7.0/ci/chaos/JenkinsfileResiliencyTrigger (about) 1 // Copyright (c) 2022, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 pipeline { 5 options { 6 skipDefaultCheckout true 7 timestamps () 8 } 9 10 agent { 11 docker { 12 image "${RUNNER_DOCKER_IMAGE}" 13 args "${RUNNER_DOCKER_ARGS}" 14 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 15 registryCredentialsId 'ocir-pull-and-push-account' 16 label "pipeline-job-large" 17 } 18 } 19 20 parameters { 21 string (name: 'GIT_COMMIT_TO_USE', 22 defaultValue: 'NONE', 23 description: 'This is the full git commit hash from the source build to be used for all jobs. A full pipeline specifies a valid commit hash here. NONE can be used for manually triggered jobs, however even for those a commit hash value is preferred to be supplied', 24 trim: true) 25 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 26 defaultValue: 'NONE', 27 description: 'This is for manually testing only where someone needs to use a specific operator image, otherwise the default value of NONE is used', 28 trim: true) 29 string (name: 'WILDCARD_DNS_DOMAIN', 30 defaultValue: 'nip.io', 31 description: 'This is the wildcard DNS domain', 32 trim: true) 33 string (name: 'EXCLUDE_RELEASES', 34 defaultValue: "v1.0.0, v1.0.1, v1.0.2, v1.0.3, v1.0.4, v1.1.0, v1.1.1, v1.1.2, v1.2.0, v1.2.1, v1.2.2," + 35 " v1.3.0, v1.3.1, v1.3.2, v1.3.3, v1.3.4, v1.3.5, v1.3.6, v1.3.7, v1.3.8", 36 description: 'This is to exclude the specified releases from upgrade tests.', trim: true) 37 string (name: 'TAGGED_TESTS', 38 defaultValue: '', 39 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 40 trim: true) 41 string (name: 'INCLUDED_TESTS', 42 defaultValue: '.*', 43 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 44 trim: true) 45 string (name: 'EXCLUDED_TESTS', 46 defaultValue: '_excluded_test', 47 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 48 trim: true) 49 string (name: 'CONSOLE_REPO_BRANCH', 50 defaultValue: '', 51 description: 'The branch to check out after cloning the console repository.', 52 trim: true) 53 } 54 55 environment { 56 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 57 GOPATH = '/home/opc/go' 58 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 59 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 60 61 OCI_CLI_AUTH="instance_principal" 62 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 63 OCI_OS_BUCKET="verrazzano-builds" 64 RELEASE_OWNERS = credentials('release-version-owners') 65 } 66 67 stages { 68 stage('Clean workspace and checkout') { 69 steps { 70 script { 71 if (params.GIT_COMMIT_TO_USE == "NONE") { 72 echo "Specific GIT commit was not specified, use current head" 73 def scmInfo = checkout([ 74 $class: 'GitSCM', 75 branches: [[name: env.BRANCH_NAME]], 76 doGenerateSubmoduleConfigurations: false, 77 extensions: [], 78 submoduleCfg: [], 79 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 80 env.GIT_COMMIT = scmInfo.GIT_COMMIT 81 env.GIT_BRANCH = scmInfo.GIT_BRANCH 82 } else { 83 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 84 def scmInfo = checkout([ 85 $class: 'GitSCM', 86 branches: [[name: params.GIT_COMMIT_TO_USE]], 87 doGenerateSubmoduleConfigurations: false, 88 extensions: [], 89 submoduleCfg: [], 90 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 91 env.GIT_COMMIT = scmInfo.GIT_COMMIT 92 env.GIT_BRANCH = scmInfo.GIT_BRANCH 93 // If the commit we were handed is not what the SCM says we are using, fail 94 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 95 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 96 exit 1 97 } 98 } 99 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 100 } 101 102 script { 103 def props = readProperties file: '.verrazzano-development-version' 104 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 105 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 106 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 107 def excludeReleases = params.EXCLUDE_RELEASES 108 def excludeReleasesList = excludeReleases.trim().split('\\s*,\\s*') 109 VERSION_FOR_INSTALL = sh(returnStdout: true, script: "go run ${WORKSPACE}/ci/tools/derive_upgrade_version.go ${workspace} install-version ${excludeReleasesList}").trim() 110 INTERIM_UPGRADE_VERSION = sh(returnStdout: true, script: "go run ${WORKSPACE}/ci/tools/derive_upgrade_version.go ${workspace} interim-version ${excludeReleasesList}").trim() 111 // update the description with some meaningful info 112 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 113 def currentCommitHash = env.GIT_COMMIT 114 def commitList = getCommitList() 115 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 116 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 117 SUSPECT_LIST = getSuspectList(commitList, userMappings) 118 echo "Suspect list: ${SUSPECT_LIST}" 119 } 120 } 121 } 122 } 123 124 stage('Verrazzano development version check') { 125 steps { 126 sh """ 127 [[ -z \$(git ls-remote --tags origin | grep -F ${VERRAZZANO_DEV_VERSION}) ]] || exit 1 128 129 130 """ 131 } 132 post { 133 failure { 134 script { 135 if (env.JOB_NAME ==~ "verrazzano/release-*") { 136 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nVZ Helper was not run, the Verrazzano Development Version ${VERRAZZANO_DEV_VERSION} matches a prior release\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nRelease Owners:\n ${RELEASE_OWNERS}\n") 137 } 138 } 139 } 140 } 141 } 142 143 stage ('Kick off resiliency tests') { 144 parallel { 145 stage('VPO killed during upgrade') { 146 steps { 147 retry(count: JOB_PROMOTION_RETRIES) { 148 script { 149 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 150 parameters: [ 151 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 152 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 153 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 154 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 155 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 156 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 157 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 158 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 159 string(name: 'CHAOS_TEST_TYPE', value: 'vpo.killed') 160 ], wait: true 161 } 162 } 163 } 164 } 165 stage('Upgrade using ephemeral storage') { 166 steps { 167 retry(count: JOB_PROMOTION_RETRIES) { 168 script { 169 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 170 parameters: [ 171 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 172 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 173 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 174 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 175 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 176 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 177 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 178 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 179 string(name: 'CHAOS_TEST_TYPE', value: 'ephemeral.storage.upgrade') 180 ], wait: true 181 } 182 } 183 } 184 } 185 stage('Component helm install failure') { 186 steps { 187 retry(count: JOB_PROMOTION_RETRIES) { 188 script { 189 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 190 parameters: [ 191 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 192 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 193 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 194 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 195 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 196 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 197 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 198 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 199 string(name: 'CHAOS_TEST_TYPE', value: 'helm.chart.corrupted') 200 ], wait: true 201 } 202 } 203 } 204 } 205 stage('Uninstall failed upgrade') { 206 steps { 207 retry(count: JOB_PROMOTION_RETRIES) { 208 script { 209 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 210 parameters: [ 211 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 212 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 213 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 214 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 215 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 216 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 217 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 218 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 219 string(name: 'CHAOS_TEST_TYPE', value: 'uninstall.failed.upgrade') 220 ], wait: true 221 } 222 } 223 } 224 } 225 stage('Upgrade a failed upgrade') { 226 steps { 227 retry(count: JOB_PROMOTION_RETRIES) { 228 script { 229 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 230 parameters: [ 231 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 232 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 233 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 234 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 235 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 236 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 237 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 238 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 239 string(name: 'CHAOS_TEST_TYPE', value: 'upgrade.failed.upgrade') 240 ], wait: true 241 } 242 } 243 } 244 } 245 stage('Upgrade an in-process failing upgrade') { 246 steps { 247 retry(count: JOB_PROMOTION_RETRIES) { 248 script { 249 build job: "/verrazzano-chaos-tests/${CLEAN_BRANCH_NAME}", 250 parameters: [ 251 string(name: 'GIT_COMMIT_FOR_UPGRADE', value: env.GIT_COMMIT), 252 string(name: 'VERSION_FOR_INSTALL', value: VERSION_FOR_INSTALL), 253 string(name: 'INTERIM_UPGRADE_VERSION', value: INTERIM_UPGRADE_VERSION), 254 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 255 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 256 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 257 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 258 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 259 string(name: 'CHAOS_TEST_TYPE', value: 'upgrade.failing.upgrade') 260 ], wait: true 261 } 262 } 263 } 264 } 265 } 266 } 267 } 268 post { 269 failure { 270 script { 271 if (env.JOB_NAME == "verrazzano-upgrade-resiliency-tests/master" || env.JOB_NAME ==~ "verrazzano-upgrade-resiliency-tests/release-1.*") { 272 if (isPagerDutyEnabled()) { 273 pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", incDescription: "Verrazzano: ${env.JOB_NAME} - Failed", incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}") 274 } 275 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}" ) 276 } 277 } 278 } 279 } 280 } 281 282 def isPagerDutyEnabled() { 283 // this controls whether PD alerts are enabled 284 if (NOTIFY_PAGERDUTY_TRIGGERED_FAILURES.equals("true")) { 285 echo "Pager-Duty notifications enabled via global override setting" 286 return true 287 } 288 return false 289 } 290 291 292 // Called in Stage Clean workspace and checkout steps 293 @NonCPS 294 def getCommitList() { 295 echo "Checking for change sets" 296 def commitList = [] 297 def changeSets = currentBuild.changeSets 298 for (int i = 0; i < changeSets.size(); i++) { 299 echo "get commits from change set" 300 def commits = changeSets[i].items 301 for (int j = 0; j < commits.length; j++) { 302 def commit = commits[j] 303 def id = commit.commitId 304 echo "Add commit id: ${id}" 305 commitList.add(id) 306 } 307 } 308 return commitList 309 } 310 311 def trimIfGithubNoreplyUser(userIn) { 312 if (userIn == null) { 313 echo "Not a github noreply user, not trimming: ${userIn}" 314 return userIn 315 } 316 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 317 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 318 return userOut; 319 } 320 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 321 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 322 return userOut; 323 } 324 if (userIn.matches(".*@users.noreply.github.com")) { 325 def userOut = userIn.substring(0, userIn.indexOf("@")) 326 return userOut; 327 } 328 echo "Not a github noreply user, not trimming: ${userIn}" 329 return userIn 330 } 331 332 def getSuspectList(commitList, userMappings) { 333 def retValue = "" 334 def suspectList = [] 335 if (commitList == null || commitList.size() == 0) { 336 echo "No commits to form suspect list" 337 } else { 338 for (int i = 0; i < commitList.size(); i++) { 339 def id = commitList[i] 340 try { 341 def gitAuthor = sh( 342 script: "git log --format='%ae' '$id^!'", 343 returnStdout: true 344 ).trim() 345 if (gitAuthor != null) { 346 def author = trimIfGithubNoreplyUser(gitAuthor) 347 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 348 if (userMappings.containsKey(author)) { 349 def slackUser = userMappings.get(author) 350 if (!suspectList.contains(slackUser)) { 351 echo "Added ${slackUser} as suspect" 352 retValue += " ${slackUser}" 353 suspectList.add(slackUser) 354 } 355 } else { 356 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 357 if (!suspectList.contains(author)) { 358 echo "Added ${author} as suspect" 359 retValue += " ${author}" 360 suspectList.add(author) 361 } 362 } 363 } else { 364 echo "No author returned from git" 365 } 366 } catch (Exception e) { 367 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 368 } 369 } 370 } 371 def startedByUser = ""; 372 def causes = currentBuild.getBuildCauses() 373 echo "causes: " + causes.toString() 374 for (cause in causes) { 375 def causeString = cause.toString() 376 echo "current cause: " + causeString 377 def causeInfo = readJSON text: causeString 378 if (causeInfo.userId != null) { 379 startedByUser = causeInfo.userId 380 } 381 } 382 383 if (startedByUser.length() > 0) { 384 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 385 def author = trimIfGithubNoreplyUser(startedByUser) 386 echo "DEBUG: author: ${startedByUser}, ${author}" 387 if (userMappings.containsKey(author)) { 388 def slackUser = userMappings.get(author) 389 if (!suspectList.contains(slackUser)) { 390 echo "Added ${slackUser} as suspect" 391 retValue += " ${slackUser}" 392 suspectList.add(slackUser) 393 } 394 } else { 395 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 396 if (!suspectList.contains(author)) { 397 echo "Added ${author} as suspect" 398 retValue += " ${author}" 399 suspectList.add(author) 400 } 401 } 402 } else { 403 echo "Build not started by a user, not adding to notification list" 404 } 405 echo "returning suspect list: ${retValue}" 406 return retValue 407 } 408 409 def getCronSchedule() { 410 if (env.BRANCH_NAME.equals("master")) { 411 return "H */2 * * *" 412 } else if (env.BRANCH_NAME.startsWith("release-1")) { 413 return "@daily" 414 } 415 return "" 416 }