github.com/verrazzano/verrazzano@v1.7.1/ci/chaos/Jenkinsfile (about) 1 // Copyright (c) 2022, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 7 8 pipeline { 9 options { 10 timeout(time: 2, unit: 'HOURS') 11 skipDefaultCheckout true 12 timestamps () 13 } 14 15 agent { 16 docker { 17 image "${RUNNER_DOCKER_IMAGE}" 18 args "${RUNNER_DOCKER_ARGS}" 19 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 20 registryCredentialsId 'ocir-pull-and-push-account' 21 label "${agentLabel}" 22 } 23 } 24 25 parameters { 26 choice (name: 'KUBERNETES_CLUSTER_VERSION', 27 description: 'Kubernetes Version for KinD Cluster', 28 // 1st choice is the default value 29 choices: ["1.24", "1.25", "1.26", "1.27" ]) 30 string (name: 'VERSION_FOR_INSTALL', 31 defaultValue: 'v1.4.3', 32 description: 'This is the Verrazzano version for install. By default, the latest Master release will be installed', 33 trim: true) 34 string (name: 'INTERIM_UPGRADE_VERSION', 35 defaultValue: 'v1.5.0', 36 description: 'This is the Verrazzano version for interim upgrade.', 37 trim: true) 38 string (name: 'GIT_COMMIT_FOR_UPGRADE', 39 defaultValue: 'NONE', 40 description: 'This is the full git commit hash for the target upgrade. By default, the Verrazzano last stable commit will be used.', 41 trim: true) 42 choice (name: 'WILDCARD_DNS_DOMAIN', 43 description: 'This is the wildcard DNS domain', 44 // 1st choice is the default value 45 choices: [ "nip.io", "sslip.io" ]) 46 choice (name: 'CRD_API_VERSION', 47 description: 'This is the API crd version.', 48 // 1st choice is the default value 49 choices: [ "v1alpha1", "v1beta1"]) 50 booleanParam (description: 'Whether to create kind cluster with Calico for AT testing (defaults to true)', name: 'CREATE_KIND_USE_CALICO', defaultValue: true) 51 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 52 booleanParam (description: 'Whether to run BOM Validator after Upgrade', name: 'RUN_BOM_VALIDATOR', defaultValue: true) 53 string (name: 'TAGGED_TESTS', 54 defaultValue: '', 55 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 56 trim: true) 57 string (name: 'INCLUDED_TESTS', 58 defaultValue: '.*', 59 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 60 trim: true) 61 string (name: 'EXCLUDED_TESTS', 62 defaultValue: '_excluded_test', 63 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 64 trim: true) 65 choice (name: 'CHAOS_TEST_TYPE', 66 description: 'Type of chaos to inflict', 67 // 1st choice is the default value 68 choices: [ "uninstall.failed.upgrade", "helm.chart.corrupted", "vpo.killed", "ephemeral.storage.upgrade", "upgrade.failed.upgrade", "upgrade.failing.upgrade" ]) 69 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 70 } 71 72 environment { 73 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 74 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 75 GOPATH = '/home/opc/go' 76 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 77 DOCKER_CREDS = credentials('github-packages-credentials-rw') 78 DOCKER_EMAIL = credentials('github-packages-email') 79 DOCKER_REPO = 'ghcr.io' 80 DOCKER_NAMESPACE = 'verrazzano' 81 NETRC_FILE = credentials('netrc') 82 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 83 84 // Environment variables required to capture cluster snapshot and bug report on test failure 85 DUMP_KUBECONFIG="${KUBECONFIG}" 86 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 87 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 88 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 89 90 // Environment variable for Verrazzano CLI executable 91 VZ_COMMAND="${GO_REPO_PATH}/vz" 92 93 CLUSTER_NAME = 'verrazzano' 94 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 95 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 96 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 97 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 98 OCR_CREDS = credentials('ocr-pull-and-push-account') 99 OCR_REPO = 'container-registry.oracle.com' 100 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 101 INSTALL_CONFIG_FILE_KIND_PERSISTENCE = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-with-persistence.yaml" 102 INSTALL_CONFIG_FILE_KIND_EPHEMERAL = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-no-persistence.yaml" 103 INSTALL_CONFIG_FILE_KIND = "${params.CHAOS_TEST_TYPE.equals("ephemeral.storage.upgrade") ? env.INSTALL_CONFIG_FILE_KIND_EPHEMERAL : env.INSTALL_CONFIG_FILE_KIND_PERSISTENCE}" 104 INSTALL_PROFILE = "${params.CHAOS_TEST_TYPE.equals("ephemeral.storage.upgrade") ? "dev" : "prod"}" 105 VZ_ENVIRONMENT_NAME = "default" 106 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 107 LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test" 108 // needed for prepare at shell script 109 VERRAZZANO_OPERATOR_IMAGE="NONE" 110 111 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test 112 DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test 113 114 OLD_PODS_FILE="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-old-vpo-app-pods.out" 115 NEW_PODS_FILE="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-new-vpo-app-pods.out" 116 NEW_PODS_FILE2="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-new-vpo-app-pods-2.out" 117 APP_NAMEPACES="'todo-list bobs-books hello-helidon springboot sockshop'" 118 119 // used to emit metrics 120 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 121 TEST_ENV_LABEL = "kind" 122 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 123 TEST_ENV = "KIND" 124 125 // used to generate Ginkgo test reports 126 TEST_REPORT = "test-report.xml" 127 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 128 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 129 } 130 131 stages { 132 stage('Clean Workspace and Checkout') { 133 environment { 134 OCI_CLI_AUTH="instance_principal" 135 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 136 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 137 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 138 } 139 steps { 140 script { 141 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 142 if (params.GIT_COMMIT_FOR_UPGRADE == "NONE") { 143 echo "Specific GIT commit was not specified, use current head" 144 def scmInfo = checkout([ 145 $class: 'GitSCM', 146 branches: [[name: env.BRANCH_NAME]], 147 doGenerateSubmoduleConfigurations: false, 148 extensions: [], 149 submoduleCfg: [], 150 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 151 env.GIT_COMMIT = scmInfo.GIT_COMMIT 152 env.GIT_BRANCH = scmInfo.GIT_BRANCH 153 } else { 154 COMMIT_FOR_UPGRADE = params.GIT_COMMIT_FOR_UPGRADE 155 echo "SCM checkout of ${COMMIT_FOR_UPGRADE}" 156 def scmInfo = checkout([ 157 $class: 'GitSCM', 158 branches: [[name: COMMIT_FOR_UPGRADE]], 159 doGenerateSubmoduleConfigurations: false, 160 extensions: [], 161 submoduleCfg: [], 162 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 163 env.GIT_COMMIT = scmInfo.GIT_COMMIT 164 env.GIT_BRANCH = scmInfo.GIT_BRANCH 165 // If the commit we were handed is not what the SCM says we are using, fail 166 if (!env.GIT_COMMIT.equals(COMMIT_FOR_UPGRADE)) { 167 echo "SCM didn't checkout the commit we expected. Expected: ${COMMIT_FOR_UPGRADE}, Found: ${scmInfo.GIT_COMMIT}" 168 exit 1 169 } 170 } 171 172 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 173 174 } 175 176 sh """ 177 cp -f "${NETRC_FILE}" $HOME/.netrc 178 chmod 600 $HOME/.netrc 179 """ 180 181 script { 182 try { 183 sh """ 184 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 185 """ 186 } catch(error) { 187 echo "docker login failed, retrying after sleep" 188 retry(4) { 189 sleep(30) 190 sh """ 191 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 192 """ 193 } 194 } 195 } 196 197 sh """ 198 rm -rf ${GO_REPO_PATH}/verrazzano 199 mkdir -p ${GO_REPO_PATH}/verrazzano 200 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 201 """ 202 203 script { 204 def props = readProperties file: '.verrazzano-development-version' 205 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 206 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 207 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 208 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 209 // update the description with some meaningful info 210 setDisplayName() 211 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_FOR_UPGRADE 212 OPERATOR_YAML_FILE = sh(returnStdout: true, script: "ci/scripts/derive_operator_yaml.sh ${params.VERSION_FOR_INSTALL}").trim() 213 echo "Downloading ${OPERATOR_YAML_FILE} for release ${params.VERSION_FOR_INSTALL}" 214 } 215 216 script { 217 sh """ 218 echo "Downloading VZ CLI from object storage" 219 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 220 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 221 ${GO_REPO_PATH}/vz version 222 """ 223 } 224 } 225 } 226 stage('Install Release Version') { 227 environment { 228 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 229 OCI_CLI_AUTH="instance_principal" 230 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 231 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 232 OPERATOR_YAML="${OPERATOR_YAML_FILE}" 233 CLUSTER_SNAPSHOT_DIR="${WORKSPACE}/verrazzano/build/resources/pre-install-resources" 234 } 235 steps { 236 sh """ 237 cd ${GO_REPO_PATH}/verrazzano 238 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_KIND_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} 239 """ 240 } 241 post { 242 always { 243 sh """ 244 ## dump out install logs 245 mkdir -p ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs 246 kubectl -n verrazzano-install logs --selector=job-name=verrazzano-install-my-verrazzano --tail -1 > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-install.log 247 kubectl -n verrazzano-install describe pod --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-install-job-pod.out 248 cp ${INSTALL_CONFIG_FILE_KIND} ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs 249 echo "Verrazzano Installation logs dumped to verrazzano-install.log" 250 echo "Verrazzano Install pod description dumped to verrazzano-install-job-pod.out" 251 echo "------------------------------------------" 252 """ 253 script { 254 dumpVerrazzanoPlatformOperatorLogs("before-upgrade") 255 } 256 archiveArtifacts artifacts: "**/logs/**,${env.INSTALL_CONFIG_FILE_KIND}", allowEmptyArchive: true 257 } 258 } 259 } 260 261 stage('examples helidon') { 262 steps { 263 runGinkgoNamespace('examples/helidon', "false", "true", "hello-helidon") 264 runGinkgoRandomize('upgrade/pre-upgrade/verify') 265 } 266 post { 267 always { 268 archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true 269 junit testResults: '**/*test-result.xml', allowEmptyResults: true 270 } 271 failure { 272 script { 273 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 274 dumpK8sCluster('pre-upgrade-tests-cluster-snapshot') 275 } 276 } 277 } 278 success { 279 script { 280 if (params.DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 281 dumpK8sCluster('install-success-cluster-snapshot') 282 } 283 } 284 } 285 } 286 } 287 stage("Upgrade Platform Operator") { 288 environment { 289 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 290 OCI_CLI_AUTH="instance_principal" 291 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 292 } 293 steps { 294 sh """ 295 git fetch 296 """ 297 script { 298 VERRAZZANO_DEV_VERSION = sh(returnStdout: true, script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/generated-verrazzano-bom.json --file - | jq -r '.version'").trim() 299 } 300 sh """ 301 # Download the operator.yaml for the target version that we are upgrading to 302 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 303 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/upgrade-test-operator.yaml 304 305 echo "Upgrading the Verrazzano platform operator" 306 if [[ "upgrade.failed.upgrade" == ${params.CHAOS_TEST_TYPE} || "upgrade.failing.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 307 then 308 echo "Upgrading VPO to interim version ${params.INTERIM_UPGRADE_VERSION}" 309 if [[ ${params.INTERIM_UPGRADE_VERSION} == "v1.3."* ]] 310 then 311 kubectl apply -f https://github.com/verrazzano/verrazzano/releases/download/${params.INTERIM_UPGRADE_VERSION}/operator.yaml 312 else 313 kubectl apply -f https://github.com/verrazzano/verrazzano/releases/download/${params.INTERIM_UPGRADE_VERSION}/verrazzano-platform-operator.yaml 314 fi 315 else 316 echo "Upgrading VPO to this commit version" 317 kubectl apply -f ${WORKSPACE}/upgrade-test-operator.yaml 318 fi 319 320 321 # ensure operator pod is up 322 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 323 324 # need to sleep since the old operator needs to transition to terminating state 325 sleep 45 326 """ 327 328 } 329 post { 330 always { 331 archiveArtifacts artifacts: "upgrade-test-operator.yaml", allowEmptyArchive: true 332 } 333 failure { 334 script { 335 if (fileExists(env.TESTS_EXECUTED_FILE)) { 336 dumpK8sCluster('post-upgrade-vpo-failure-cluster-snapshot') 337 } 338 } 339 } 340 } 341 } 342 343 stage("Upgrade Verrazzano and execute selected test") { 344 environment { 345 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 346 OCI_CLI_AUTH="instance_principal" 347 } 348 steps { 349 script { 350 // Download the bom for the target version that we are upgrading to, then extract the version 351 // Note, this version will have a semver suffix which is generated for each build, e.g. 1.0.1-33+d592fed6 352 VERRAZZANO_INTERIM_VERSION = "NONE" 353 if ("upgrade.failed.upgrade".equals(params.CHAOS_TEST_TYPE)) { 354 VERRAZZANO_INTERIM_VERSION = params.INTERIM_UPGRADE_VERSION 355 } else if ("upgrade.failing.upgrade".equals(params.CHAOS_TEST_TYPE)) { 356 VERRAZZANO_INTERIM_VERSION = params.INTERIM_UPGRADE_VERSION 357 } 358 VERRAZZANO_DEV_VERSION = sh(returnStdout: true, script: "curl https://objectstorage.us-phoenix-1.oraclecloud.com/n/stevengreenberginc/b/${OCI_OS_COMMIT_BUCKET}/o/ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/generated-verrazzano-bom.json | jq -r '.version'").trim() 359 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 360 } 361 sh """ 362 # Modify the version field in the Verrazzano CR file to be this new Verrazzano version 363 cd ${GO_REPO_PATH}/verrazzano 364 cp ${INSTALL_CONFIG_FILE_KIND} ${WORKSPACE}/verrazzano-upgrade-cr.yaml 365 ${TEST_SCRIPTS_DIR}/process_upgrade_yaml.sh ${VERRAZZANO_DEV_VERSION} ${WORKSPACE}/verrazzano-upgrade-cr.yaml 366 if [[ "NONE" != ${VERRAZZANO_INTERIM_VERSION} ]] 367 then 368 # create the interim VZ CR 369 cp ${INSTALL_CONFIG_FILE_KIND} ${WORKSPACE}/verrazzano-interim-cr.yaml 370 ${TEST_SCRIPTS_DIR}/process_upgrade_yaml.sh ${VERRAZZANO_INTERIM_VERSION} ${WORKSPACE}/verrazzano-interim-cr.yaml 371 cp ${WORKSPACE}/verrazzano-interim-cr.yaml ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs 372 fi 373 # Copy upgrade CR where it will be archived 374 cp ${WORKSPACE}/verrazzano-upgrade-cr.yaml ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs 375 376 # Get the install job in verrazzano-install namespace 377 kubectl -n verrazzano-install get job -o yaml --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-pre-upgrade-job.out 378 379 echo "Following is the verrazzano CR file with the new version:" 380 cat ${WORKSPACE}/verrazzano-upgrade-cr.yaml 381 382 echo "CHAOS TEST TYPE = ${params.CHAOS_TEST_TYPE}" 383 384 if [[ "ephemeral.storage.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 385 then 386 # Initiate upgrade 387 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 388 389 # A successful completion of the upgrade means the Keycloak configuration was successfully rebuilt 390 kubectl wait --timeout=45m --for=condition=UpgradeComplete verrazzano/my-verrazzano 391 392 # Kill MySQL and wait for Keycloak to be healthy again 393 ${GO_REPO_PATH}/verrazzano/ci/scripts/chaos_test_ephemeral_storage.sh 394 fi 395 396 if [[ "vpo.killed" == ${params.CHAOS_TEST_TYPE} ]] 397 then 398 # Initiate upgrade 399 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 400 # Wait 15 seconds then kill vpo 401 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 402 echo "Wait 15 seconds and kill VPO \$POD" 403 sleep 15 404 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 405 kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-before-vpo-killed-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 406 echo "dumping out the VZ CR:" 407 kubectl get verrazzano my-verrazzano -o yaml 408 echo "dumping out the VMI" 409 kubectl get vmi system -n verrazzano-system -o yaml 410 kubectl -n verrazzano-install delete po \$POD 411 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 412 kubectl -n verrazzano-install wait --for=condition=ready --timeout=600s pod/\$POD 413 echo "VPO \$POD successfully restarted" 414 fi 415 416 417 if [[ "helm.chart.corrupted" == ${params.CHAOS_TEST_TYPE} ]] 418 then 419 # Move values.yaml so helm install of Verrazzano will fail 420 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 421 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak" 422 # Initiate upgrade 423 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 424 # wait for the expected error message 425 until [ `kubectl logs -l app=verrazzano-platform-operator -n verrazzano-install --tail -1 | grep "Failed running Helm command for release verrazzano" | wc -l` -gt 0 ] 426 do 427 echo "Waiting for the error message ..." 428 sleep 30 429 done 430 # Fix it 431 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml" 432 fi 433 434 if [[ "upgrade.failed.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 435 then 436 # Move values.yaml so helm install of Verrazzano will fail 437 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 438 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak" 439 # Initiate upgrade 440 kubectl apply -f ${WORKSPACE}/verrazzano-interim-cr.yaml 441 # The upgrade will indefinitely remain in the upgrading state. Wait for indication that the Verrazzano component is failing to upgrade 442 until [ `kubectl logs -l app=verrazzano-platform-operator -n verrazzano-install --tail -1 | grep "Failed upgrading component verrazzano" | wc -l` -gt 0 ] 443 do 444 echo "Waiting for Verrazzano to fail upgrade ..." 445 sleep 30 446 done 447 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 448 kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-before-final-upgrade-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 449 # Upgrade to the latest VPO 450 kubectl apply -f ${WORKSPACE}/upgrade-test-operator.yaml 451 # ensure operator pod is up 452 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 453 # need to sleep since the old operator needs to transition to terminating state 454 sleep 45 455 # apply the CR 456 echo "Following is the verrazzano CR file with the updated version:" 457 cat ${WORKSPACE}/verrazzano-upgrade-cr.yaml 458 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 459 fi 460 461 if [[ "upgrade.failing.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 462 then 463 # Move values.yaml so helm install of Verrazzano will fail 464 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 465 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak" 466 # Initiate upgrade 467 kubectl apply -f ${WORKSPACE}/verrazzano-interim-cr.yaml 468 # wait for the expected error 469 until [ `kubectl logs -l app=verrazzano-platform-operator -n verrazzano-install --tail -1 | grep "Failed running Helm command for release verrazzano" | wc -l` -gt 0 ] 470 do 471 echo "Waiting for the error message ..." 472 sleep 30 473 done 474 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 475 kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-before-final-upgrade-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 476 # Make sure the upgrade is still in progress 477 vz_status=\$(kubectl get vz -o jsonpath='{.items[0].status.conditions[?(@.type=="UpgradeStarted")].type}') 478 if [[ \${vz_status} == "UpgradeStarted" ]] 479 then 480 # Upgrade to the latest VPO 481 kubectl apply -f ${WORKSPACE}/upgrade-test-operator.yaml 482 # ensure operator pod is up 483 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 484 # need to sleep since the old operator needs to transition to terminating state 485 sleep 45 486 # apply the CR 487 echo "Following is the verrazzano CR file with the updated version:" 488 cat ${WORKSPACE}/verrazzano-upgrade-cr.yaml 489 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 490 fi 491 fi 492 493 if [[ "uninstall.failed.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 494 then 495 # Move values.yaml so helm install of Verrazzano will fail 496 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 497 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak" 498 # Initiate upgrade 499 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 500 # wait for the expected error message 501 until [ `kubectl logs -l app=verrazzano-platform-operator -n verrazzano-install --tail -1 | grep "Failed running Helm command for release verrazzano" | wc -l` -gt 0 ] 502 do 503 echo "Waiting for the error message ..." 504 sleep 30 505 done 506 # Make sure the upgrade is still in progress 507 vz_status=\$(kubectl get vz -o jsonpath='{.items[0].status.conditions[?(@.type=="UpgradeStarted")].type}') 508 if [[ \${vz_status} == "UpgradeStarted" ]]; then 509 ${GO_REPO_PATH}/vz uninstall -y --timeout 45m 510 511 ## dump out uninstall logs 512 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/uninstall/build/logs 513 kubectl logs --selector=job-name=verrazzano-uninstall-my-verrazzano -n verrazzano-install > ${WORKSPACE}/verrazzano-platform-operator/scripts/uninstall/build/logs/verrazzano-uninstall.log --tail -1 514 kubectl describe pod --selector=job-name=verrazzano-uninstall-my-verrazzano -n verrazzano-install > ${WORKSPACE}/verrazzano-platform-operator/scripts/uninstall/build/logs/verrazzano-uninstall-job-pod.out 515 echo "Listing all pods in all namespaces after uninstall" 516 kubectl get pods --all-namespaces 517 echo "-----------------------------------------------------" 518 519 520 # verify Uninstall 521 ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/post-uninstall-resources false 522 ${LOOPING_TEST_SCRIPTS_DIR}/verify_uninstall.sh ${WORKSPACE}/verrazzano/build/resources 523 else 524 echo "Upgrade should still be in progress. Exiting with error" 525 exit 1 526 fi 527 fi 528 529 if [[ "uninstall.failed.upgrade" != ${params.CHAOS_TEST_TYPE} ]] 530 then 531 # wait for the upgrade to complete 532 kubectl wait --timeout=45m --for=condition=UpgradeComplete verrazzano/my-verrazzano 533 534 # Dump the resource to debug 535 kubectl get -o yaml verrazzano/my-verrazzano || true 536 537 # ideally we don't need to wait here 538 sleep 15 539 echo "helm list : releases across all namespaces, after upgrading Verrazzano installation ..." 540 helm list -A 541 542 # Get the install job(s) and mke sure the it matches pre-install. If there is more than 1 job or the job changed, then it won't match 543 kubectl -n verrazzano-install get job -o yaml --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-post-upgrade-job.out 544 545 echo "Ensuring that the install job(s) in verrazzzano-system are identical pre and post install" 546 cmp -s ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-pre-upgrade-job.out ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-post-upgrade-job.out 547 fi 548 549 """ 550 } 551 post { 552 always { 553 dumpVerrazzanoPlatformOperatorLogs("after-upgrade") 554 archiveArtifacts artifacts: "verrazzano-upgrade-cr.yaml", allowEmptyArchive: true 555 } 556 failure { 557 script { 558 if (fileExists(env.TESTS_EXECUTED_FILE)) { 559 dumpK8sCluster('upgrade-failure-cluster-snapshot') 560 } 561 } 562 } 563 } 564 } 565 566 stage('Post-upgrade Acceptance Tests') { 567 when { 568 expression{params.CHAOS_TEST_TYPE != "uninstall.failed.upgrade"} 569 } 570 parallel { 571 stage('verify-upgrade post-upgrade') { 572 steps { 573 runGinkgoRandomize('upgrade/post-upgrade/verify') 574 } 575 } 576 stage('examples helidon') { 577 steps { 578 runGinkgoNamespace('examples/helidon', "true", "false", "hello-helidon") 579 } 580 } 581 // run todo example test to verify upgraded app/weblogic/coherence operators are functioning correctly 582 stage('examples todo') { 583 environment { 584 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/todo" 585 } 586 steps { 587 runGinkgoNamespace('examples/todo', "false", "false", "todo-list") 588 } 589 } 590 } 591 post { 592 always { 593 archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true 594 junit testResults: '**/*test-result.xml', allowEmptyResults: true 595 } 596 failure { 597 script { 598 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 599 dumpK8sCluster('post-upgrade-failure-cluster-snapshot') 600 } 601 } 602 } 603 success { 604 script { 605 if (params.DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 606 dumpK8sCluster('post-upgrade-success-cluster-snapshot') 607 } 608 } 609 } 610 } 611 } 612 } 613 614 post { 615 always { 616 script { 617 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 618 dumpVerrazzanoSystemPods() 619 dumpCattleSystemPods() 620 dumpNginxIngressControllerLogs() 621 dumpVerrazzanoApplicationOperatorLogs() 622 dumpOamKubernetesRuntimeLogs() 623 dumpVerrazzanoApiLogs() 624 } 625 } 626 627 sh """ 628 # Copy the generated test reports to WORKSPACE to archive them 629 mkdir -p ${TEST_REPORT_DIR} 630 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 631 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 632 """ 633 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/build/resources/**,**/*full-cluster*/**,**/bug-report/**,**/${TEST_REPORT}", allowEmptyArchive: true 634 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 635 636 sh """ 637 cd ${GO_REPO_PATH}/verrazzano/platform-operator 638 make delete-cluster 639 cd ${WORKSPACE}/verrazzano 640 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 641 echo "Failures seen during dumping of artifacts, treat post as failed" 642 exit 1 643 fi 644 """ 645 } 646 cleanup { 647 deleteDir() 648 } 649 } 650 } 651 652 def runGinkgoRandomize(testSuitePath) { 653 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 654 sh """ 655 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 656 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 657 """ 658 } 659 } 660 661 def runGinkgo(testSuitePath, skipDeploy, skipUndeploy) { 662 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 663 sh """ 664 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 665 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} 666 """ 667 } 668 } 669 670 def runGinkgoNamespace(testSuitePath, skipDeploy, skipUndeploy, namespace) { 671 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 672 sh """ 673 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 674 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace} 675 """ 676 } 677 } 678 679 def dumpK8sCluster(dumpDirectory) { 680 sh """ 681 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 682 """ 683 } 684 685 def dumpVerrazzanoSystemPods() { 686 sh """ 687 cd ${GO_REPO_PATH}/verrazzano/platform-operator 688 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log" 689 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 690 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log" 691 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 692 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log" 693 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 694 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log" 695 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 696 """ 697 } 698 699 def dumpCattleSystemPods() { 700 sh """ 701 cd ${GO_REPO_PATH}/verrazzano/platform-operator 702 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/cattle-system-pods.log" 703 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 704 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/rancher.log" 705 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 706 """ 707 } 708 709 def dumpNginxIngressControllerLogs() { 710 sh """ 711 cd ${GO_REPO_PATH}/verrazzano/platform-operator 712 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log" 713 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 714 """ 715 } 716 717 def dumpVerrazzanoPlatformOperatorLogs(stage) { 718 sh """ 719 ## dump out verrazzano-platform-operator logs 720 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 721 kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 722 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 723 echo "------------------------------------------" 724 """ 725 } 726 727 def dumpVerrazzanoApplicationOperatorLogs() { 728 sh """ 729 ## dump out verrazzano-application-operator logs 730 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 731 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator --tail -1 > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 732 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 733 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 734 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 735 echo "------------------------------------------" 736 """ 737 } 738 739 def dumpOamKubernetesRuntimeLogs() { 740 sh """ 741 ## dump out oam-kubernetes-runtime logs 742 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 743 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime --tail -1 > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 744 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 745 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 746 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 747 echo "------------------------------------------" 748 """ 749 } 750 751 def dumpVerrazzanoApiLogs() { 752 sh """ 753 cd ${GO_REPO_PATH}/verrazzano/platform-operator 754 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log" 755 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 756 """ 757 } 758 759 def setDisplayName() { 760 echo "Start setDisplayName" 761 def causes = currentBuild.getBuildCauses() 762 echo "causes: " + causes.toString() 763 for (cause in causes) { 764 def causeString = cause.toString() 765 echo "current cause: " + causeString 766 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 767 echo "This job was caused by " + causeString 768 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 769 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 770 } else if (causeString.contains("verrazzano-flaky-tests")) { 771 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 772 } 773 } 774 } 775 echo "End setDisplayName" 776 } 777 778 def getEffectiveDumpOnSuccess() { 779 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 780 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 781 effectiveValue = true 782 echo "Forcing dump on success based on global override setting" 783 } 784 return effectiveValue 785 } 786 787 def listNamespacesAndPods(customMessage) { 788 sh """ 789 echo "Listing all the namespaces and pods the namespaces ${customMessage}." 790 kubectl get namespaces 791 kubectl get pods -A 792 echo "-----------------------------------------------------" 793 """ 794 } 795 796 def listHelmReleases(customMessage) { 797 sh """ 798 echo "Listing the releases across all namespaces ${customMessage}." 799 helm list -A 800 echo "-----------------------------------------------------" 801 """ 802 } 803 804 def restartExampleApps() { 805 sh """ 806 kubectl rollout restart deployment -n springboot 807 808 kubectl rollout restart deployment -n hello-helidon 809 """ 810 }