github.com/verrazzano/verrazzano@v1.7.0/ci/uninstall/JenkinsfileUninstallSuite (about) 1 // Copyright (c) 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 pipeline { 5 options { 6 skipDefaultCheckout true 7 timestamps () 8 } 9 10 agent { 11 docker { 12 image "${RUNNER_DOCKER_IMAGE}" 13 args "${RUNNER_DOCKER_ARGS}" 14 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 15 registryCredentialsId 'ocir-pull-and-push-account' 16 label "pipeline-job-large" 17 } 18 } 19 20 parameters { 21 string (name: 'GIT_COMMIT_TO_USE', 22 defaultValue: 'NONE', 23 description: 'This is the full git commit hash from the source build to be used for all jobs. A full pipeline specifies a valid commit hash here. NONE can be used for manually triggered jobs, however even for those a commit hash value is preferred to be supplied', 24 trim: true) 25 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 26 defaultValue: 'NONE', 27 description: 'This is for manually testing only where someone needs to use a specific operator image, otherwise the default value of NONE is used', 28 trim: true) 29 string (name: 'WILDCARD_DNS_DOMAIN', 30 defaultValue: 'nip.io', 31 description: 'This is the wildcard DNS domain', 32 trim: true) 33 string (name: 'INSTALL_LOOP_COUNT', 34 description: 'Install loop count, valid for Uninstall loop tests', 35 defaultValue: "3") 36 booleanParam (name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', 37 description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', 38 defaultValue: false) 39 booleanParam (name: 'CAPTURE_FULL_CLUSTER', 40 description: 'Whether to capture full cluster snapshot on test failure', 41 defaultValue: false) 42 string (name: 'TAGGED_TESTS', 43 defaultValue: '', 44 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 45 trim: true) 46 string (name: 'INCLUDED_TESTS', 47 defaultValue: '.*', 48 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 49 trim: true) 50 string (name: 'EXCLUDED_TESTS', 51 defaultValue: '_excluded_test', 52 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 53 trim: true) 54 string (name: 'CONSOLE_REPO_BRANCH', 55 defaultValue: '', 56 description: 'The branch to check out after cloning the console repository.', 57 trim: true) 58 } 59 60 environment { 61 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 62 GOPATH = '/home/opc/go' 63 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 64 OCI_CLI_AUTH="instance_principal" 65 PROMETHEUS_GW_URL = credentials('prometheus-dev-url') 66 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 67 } 68 69 stages { 70 stage('Clean workspace and checkout') { 71 steps { 72 sh """ 73 echo "${NODE_LABELS}" 74 """ 75 76 // REVIEW: I'm not sure that we actually need to fetch the sources here, but I'm doing here as it was easier 77 // to test working with the SCM checkout settings starting from this job. We should be able to trigger this job 78 // with parameters directly (ie: based on a previous build), in that situation doing this gives us a single point 79 // to ensure the commit matches what was intended before triggering a bunch of downstream jobs that will 80 // all fail if it wasn't correct. So we may want to keep it here unless there is a compelling reason not to do so. 81 // I haven't looked at the executor resource usage yet in all of this, so it may be that could have constraints for 82 // using flyweight executors (still need to look at that) 83 script { 84 if (params.GIT_COMMIT_TO_USE == "NONE") { 85 echo "Specific GIT commit was not specified, use current head" 86 def scmInfo = checkout([ 87 $class: 'GitSCM', 88 branches: [[name: env.BRANCH_NAME]], 89 doGenerateSubmoduleConfigurations: false, 90 extensions: [], 91 submoduleCfg: [], 92 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 93 env.GIT_COMMIT = scmInfo.GIT_COMMIT 94 env.GIT_BRANCH = scmInfo.GIT_BRANCH 95 } else { 96 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 97 def scmInfo = checkout([ 98 $class: 'GitSCM', 99 branches: [[name: params.GIT_COMMIT_TO_USE]], 100 doGenerateSubmoduleConfigurations: false, 101 extensions: [], 102 submoduleCfg: [], 103 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 104 env.GIT_COMMIT = scmInfo.GIT_COMMIT 105 env.GIT_BRANCH = scmInfo.GIT_BRANCH 106 // If the commit we were handed is not what the SCM says we are using, fail 107 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 108 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 109 exit 1 110 } 111 } 112 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 113 } 114 115 script { 116 def props = readProperties file: '.verrazzano-development-version' 117 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 118 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 119 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 120 // update the description with some meaningful info 121 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 122 def currentCommitHash = env.GIT_COMMIT 123 def commitList = getCommitList() 124 withCredentials([file(credentialsId: 'jenkins-to-slack-users', variable: 'JENKINS_TO_SLACK_JSON')]) { 125 def userMappings = readJSON file: JENKINS_TO_SLACK_JSON 126 SUSPECT_LIST = getSuspectList(commitList, userMappings) 127 echo "Suspect list: ${SUSPECT_LIST}" 128 } 129 } 130 } 131 } 132 133 stage ('Uninstall resiliency tests') { 134 parallel { 135 stage('VPO killed during uninstall') { 136 steps { 137 retry(count: JOB_PROMOTION_RETRIES) { 138 script { 139 build job: "/verrazzano-uninstall-resiliency-test/${CLEAN_BRANCH_NAME}", 140 parameters: [ 141 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 142 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 143 string(name: 'INSTALL_PROFILE', value: "prod"), 144 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 145 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 146 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 147 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 148 string(name: 'CHAOS_TEST_TYPE', value: 'uninstall.interrupt.uninstall'), 149 booleanParam(name: 'CAPTURE_FULL_CLUSTER', value: params.CAPTURE_FULL_CLUSTER), 150 booleanParam(name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', value: params.DUMP_K8S_CLUSTER_ON_SUCCESS) 151 ], wait: true 152 } 153 } 154 } 155 } 156 stage('Install Loop - Prod') { 157 steps { 158 retry(count: JOB_PROMOTION_RETRIES) { 159 script { 160 build job: "/verrazzano-uninstall-resiliency-test/${CLEAN_BRANCH_NAME}", 161 parameters: [ 162 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 163 string(name: 'INSTALL_LOOP_COUNT', value: params.INSTALL_LOOP_COUNT), 164 string(name: 'INSTALL_PROFILE', value: "prod"), 165 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 166 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 167 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 168 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 169 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 170 string(name: 'CHAOS_TEST_TYPE', value: 'uninstall.reinstall.loop'), 171 booleanParam(name: 'CAPTURE_FULL_CLUSTER', value: params.CAPTURE_FULL_CLUSTER), 172 booleanParam(name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', value: params.DUMP_K8S_CLUSTER_ON_SUCCESS) 173 ], wait: true 174 } 175 } 176 } 177 } 178 stage('Install Loop - Dev') { 179 steps { 180 retry(count: JOB_PROMOTION_RETRIES) { 181 script { 182 build job: "/verrazzano-uninstall-resiliency-test/${CLEAN_BRANCH_NAME}", 183 parameters: [ 184 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 185 string(name: 'INSTALL_LOOP_COUNT', value: params.INSTALL_LOOP_COUNT), 186 string(name: 'INSTALL_PROFILE', value: "dev"), 187 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 188 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 189 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 190 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 191 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 192 string(name: 'CHAOS_TEST_TYPE', value: 'uninstall.reinstall.loop'), 193 booleanParam(name: 'CAPTURE_FULL_CLUSTER', value: params.CAPTURE_FULL_CLUSTER), 194 booleanParam(name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', value: params.DUMP_K8S_CLUSTER_ON_SUCCESS) 195 ], wait: true 196 } 197 } 198 } 199 } 200 stage('Install Loop - Managed Cluster') { 201 steps { 202 retry(count: JOB_PROMOTION_RETRIES) { 203 script { 204 build job: "/verrazzano-uninstall-resiliency-test/${CLEAN_BRANCH_NAME}", 205 parameters: [ 206 string(name: 'GIT_COMMIT_TO_USE', value: env.GIT_COMMIT), 207 string(name: 'INSTALL_LOOP_COUNT', value: params.INSTALL_LOOP_COUNT), 208 string(name: 'INSTALL_PROFILE', value: "managed-cluster"), 209 string(name: 'VERRAZZANO_OPERATOR_IMAGE', value: params.VERRAZZANO_OPERATOR_IMAGE), 210 string(name: 'WILDCARD_DNS_DOMAIN', value: params.WILDCARD_DNS_DOMAIN), 211 string(name: 'TAGGED_TESTS', value: params.TAGGED_TESTS), 212 string(name: 'INCLUDED_TESTS', value: params.INCLUDED_TESTS), 213 string(name: 'EXCLUDED_TESTS', value: params.EXCLUDED_TESTS), 214 string(name: 'CHAOS_TEST_TYPE', value: 'uninstall.reinstall.loop'), 215 booleanParam(name: 'CAPTURE_FULL_CLUSTER', value: params.CAPTURE_FULL_CLUSTER), 216 booleanParam(name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', value: params.DUMP_K8S_CLUSTER_ON_SUCCESS) 217 ], wait: true 218 } 219 } 220 } 221 } 222 } 223 } 224 } 225 post { 226 failure { 227 script { 228 if (isAlertingEnabled()) { 229 if (isPagerDutyEnabled()) { 230 pagerduty(resolve: false, serviceKey: "$SERVICE_KEY", incDescription: "Verrazzano: ${env.JOB_NAME} - Failed", incDetails: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}") 231 } 232 slackSend ( channel: "$SLACK_ALERT_CHANNEL", message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}\n\nSuspects:\n${SUSPECT_LIST}" ) 233 } 234 } 235 } 236 } 237 } 238 239 def isPagerDutyEnabled() { 240 // this controls whether PD alerts are enabled 241 if (NOTIFY_PAGERDUTY_TRIGGERED_FAILURES.equals("true")) { 242 echo "Pager-Duty notifications enabled via global override setting" 243 return true 244 } 245 return false 246 } 247 248 def isAlertingEnabled() { 249 // this controls whether any alerting happens for these tests 250 if (env.BRANCH_NAME.equals("master") || env.BRANCH_NAME.startsWith("release-1.")) { 251 echo "Alert notifications enabled for ${env.BRANCH_NAME}" 252 return true 253 } 254 return false 255 } 256 257 // Called in Stage Clean workspace and checkout steps 258 @NonCPS 259 def getCommitList() { 260 echo "Checking for change sets" 261 def commitList = [] 262 def changeSets = currentBuild.changeSets 263 for (int i = 0; i < changeSets.size(); i++) { 264 echo "get commits from change set" 265 def commits = changeSets[i].items 266 for (int j = 0; j < commits.length; j++) { 267 def commit = commits[j] 268 def id = commit.commitId 269 echo "Add commit id: ${id}" 270 commitList.add(id) 271 } 272 } 273 return commitList 274 } 275 276 def trimIfGithubNoreplyUser(userIn) { 277 if (userIn == null) { 278 echo "Not a github noreply user, not trimming: ${userIn}" 279 return userIn 280 } 281 if (userIn.matches(".*\\+.*@users.noreply.github.com.*")) { 282 def userOut = userIn.substring(userIn.indexOf("+") + 1, userIn.indexOf("@")) 283 return userOut; 284 } 285 if (userIn.matches(".*<.*@users.noreply.github.com.*")) { 286 def userOut = userIn.substring(userIn.indexOf("<") + 1, userIn.indexOf("@")) 287 return userOut; 288 } 289 if (userIn.matches(".*@users.noreply.github.com")) { 290 def userOut = userIn.substring(0, userIn.indexOf("@")) 291 return userOut; 292 } 293 echo "Not a github noreply user, not trimming: ${userIn}" 294 return userIn 295 } 296 297 def getSuspectList(commitList, userMappings) { 298 def retValue = "" 299 def suspectList = [] 300 if (commitList == null || commitList.size() == 0) { 301 echo "No commits to form suspect list" 302 } else { 303 for (int i = 0; i < commitList.size(); i++) { 304 def id = commitList[i] 305 try { 306 def gitAuthor = sh( 307 script: "git log --format='%ae' '$id^!'", 308 returnStdout: true 309 ).trim() 310 if (gitAuthor != null) { 311 def author = trimIfGithubNoreplyUser(gitAuthor) 312 echo "DEBUG: author: ${gitAuthor}, ${author}, id: ${id}" 313 if (userMappings.containsKey(author)) { 314 def slackUser = userMappings.get(author) 315 if (!suspectList.contains(slackUser)) { 316 echo "Added ${slackUser} as suspect" 317 retValue += " ${slackUser}" 318 suspectList.add(slackUser) 319 } 320 } else { 321 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 322 if (!suspectList.contains(author)) { 323 echo "Added ${author} as suspect" 324 retValue += " ${author}" 325 suspectList.add(author) 326 } 327 } 328 } else { 329 echo "No author returned from git" 330 } 331 } catch (Exception e) { 332 echo "INFO: Problem processing commit ${id}, skipping commit: " + e.toString() 333 } 334 } 335 } 336 def startedByUser = ""; 337 def causes = currentBuild.getBuildCauses() 338 echo "causes: " + causes.toString() 339 for (cause in causes) { 340 def causeString = cause.toString() 341 echo "current cause: " + causeString 342 def causeInfo = readJSON text: causeString 343 if (causeInfo.userId != null) { 344 startedByUser = causeInfo.userId 345 } 346 } 347 348 if (startedByUser.length() > 0) { 349 echo "Build was started by a user, adding them to the suspect notification list: ${startedByUser}" 350 def author = trimIfGithubNoreplyUser(startedByUser) 351 echo "DEBUG: author: ${startedByUser}, ${author}" 352 if (userMappings.containsKey(author)) { 353 def slackUser = userMappings.get(author) 354 if (!suspectList.contains(slackUser)) { 355 echo "Added ${slackUser} as suspect" 356 retValue += " ${slackUser}" 357 suspectList.add(slackUser) 358 } 359 } else { 360 // If we don't have a name mapping use the commit.author, at least we can easily tell if the mapping gets dated 361 if (!suspectList.contains(author)) { 362 echo "Added ${author} as suspect" 363 retValue += " ${author}" 364 suspectList.add(author) 365 } 366 } 367 } else { 368 echo "Build not started by a user, not adding to notification list" 369 } 370 echo "returning suspect list: ${retValue}" 371 return retValue 372 }