github.com/verrazzano/verrazzano@v1.7.0/ci/chaos/Jenkinsfile (about) 1 // Copyright (c) 2022, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 7 8 pipeline { 9 options { 10 timeout(time: 2, unit: 'HOURS') 11 skipDefaultCheckout true 12 timestamps () 13 } 14 15 agent { 16 docker { 17 image "${RUNNER_DOCKER_IMAGE}" 18 args "${RUNNER_DOCKER_ARGS}" 19 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 20 registryCredentialsId 'ocir-pull-and-push-account' 21 label "${agentLabel}" 22 } 23 } 24 25 parameters { 26 choice (name: 'KUBERNETES_CLUSTER_VERSION', 27 description: 'Kubernetes Version for KinD Cluster', 28 // 1st choice is the default value 29 choices: ["1.24", "1.25", "1.26", "1.27" ]) 30 string (name: 'VERSION_FOR_INSTALL', 31 defaultValue: 'v1.4.3', 32 description: 'This is the Verrazzano version for install. By default, the latest Master release will be installed', 33 trim: true) 34 string (name: 'INTERIM_UPGRADE_VERSION', 35 defaultValue: 'v1.5.0', 36 description: 'This is the Verrazzano version for interim upgrade.', 37 trim: true) 38 string (name: 'GIT_COMMIT_FOR_UPGRADE', 39 defaultValue: 'NONE', 40 description: 'This is the full git commit hash for the target upgrade. By default, the Verrazzano last stable commit will be used.', 41 trim: true) 42 choice (name: 'WILDCARD_DNS_DOMAIN', 43 description: 'This is the wildcard DNS domain', 44 // 1st choice is the default value 45 choices: [ "nip.io", "sslip.io" ]) 46 choice (name: 'CRD_API_VERSION', 47 description: 'This is the API crd version.', 48 // 1st choice is the default value 49 choices: [ "v1alpha1", "v1beta1"]) 50 booleanParam (description: 'Whether to create kind cluster with Calico for AT testing (defaults to true)', name: 'CREATE_KIND_USE_CALICO', defaultValue: true) 51 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 52 booleanParam (description: 'Whether to run BOM Validator after Upgrade', name: 'RUN_BOM_VALIDATOR', defaultValue: true) 53 string (name: 'TAGGED_TESTS', 54 defaultValue: '', 55 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 56 trim: true) 57 string (name: 'INCLUDED_TESTS', 58 defaultValue: '.*', 59 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 60 trim: true) 61 string (name: 'EXCLUDED_TESTS', 62 defaultValue: '_excluded_test', 63 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 64 trim: true) 65 choice (name: 'CHAOS_TEST_TYPE', 66 description: 'Type of chaos to inflict', 67 // 1st choice is the default value 68 choices: [ "uninstall.failed.upgrade", "helm.chart.corrupted", "vpo.killed", "ephemeral.storage.upgrade", "upgrade.failed.upgrade", "upgrade.failing.upgrade" ]) 69 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 70 } 71 72 environment { 73 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 74 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 75 GOPATH = '/home/opc/go' 76 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 77 DOCKER_CREDS = credentials('github-packages-credentials-rw') 78 DOCKER_EMAIL = credentials('github-packages-email') 79 DOCKER_REPO = 'ghcr.io' 80 DOCKER_NAMESPACE = 'verrazzano' 81 NETRC_FILE = credentials('netrc') 82 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 83 84 // Environment variables required to capture cluster snapshot and bug report on test failure 85 DUMP_KUBECONFIG="${KUBECONFIG}" 86 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 87 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 88 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 89 90 // Environment variable for Verrazzano CLI executable 91 VZ_COMMAND="${GO_REPO_PATH}/vz" 92 93 CLUSTER_NAME = 'verrazzano' 94 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 95 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 96 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 97 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 98 OCR_CREDS = credentials('ocr-pull-and-push-account') 99 OCR_REPO = 'container-registry.oracle.com' 100 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 101 INSTALL_CONFIG_FILE_KIND_PERSISTENCE = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-with-persistence.yaml" 102 INSTALL_CONFIG_FILE_KIND_EPHEMERAL = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-no-persistence.yaml" 103 INSTALL_CONFIG_FILE_KIND = "${params.CHAOS_TEST_TYPE.equals("ephemeral.storage.upgrade") ? env.INSTALL_CONFIG_FILE_KIND_EPHEMERAL : env.INSTALL_CONFIG_FILE_KIND_PERSISTENCE}" 104 INSTALL_PROFILE = "${params.CHAOS_TEST_TYPE.equals("ephemeral.storage.upgrade") ? "dev" : "prod"}" 105 VZ_ENVIRONMENT_NAME = "default" 106 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 107 LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test" 108 // needed for prepare at shell script 109 VERRAZZANO_OPERATOR_IMAGE="NONE" 110 111 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test 112 DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test 113 114 OLD_PODS_FILE="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-old-vpo-app-pods.out" 115 NEW_PODS_FILE="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-new-vpo-app-pods.out" 116 NEW_PODS_FILE2="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-new-vpo-app-pods-2.out" 117 APP_NAMEPACES="'todo-list bobs-books hello-helidon springboot sockshop'" 118 119 // used to emit metrics 120 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 121 TEST_ENV_LABEL = "kind" 122 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 123 TEST_ENV = "KIND" 124 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 125 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 126 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 127 128 // used to generate Ginkgo test reports 129 TEST_REPORT = "test-report.xml" 130 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 131 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 132 } 133 134 stages { 135 stage('Clean Workspace and Checkout') { 136 environment { 137 OCI_CLI_AUTH="instance_principal" 138 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 139 OCI_OS_BUCKET="verrazzano-builds" 140 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 141 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 142 CLEAN_BRANCH_NAME = "${env.BRANCH_NAME.replace("/", "%2F")}" 143 } 144 steps { 145 script { 146 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 147 if (params.GIT_COMMIT_FOR_UPGRADE == "NONE") { 148 echo "Specific GIT commit was not specified, use current head" 149 def scmInfo = checkout([ 150 $class: 'GitSCM', 151 branches: [[name: env.BRANCH_NAME]], 152 doGenerateSubmoduleConfigurations: false, 153 extensions: [], 154 submoduleCfg: [], 155 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 156 env.GIT_COMMIT = scmInfo.GIT_COMMIT 157 env.GIT_BRANCH = scmInfo.GIT_BRANCH 158 } else { 159 COMMIT_FOR_UPGRADE = params.GIT_COMMIT_FOR_UPGRADE 160 echo "SCM checkout of ${COMMIT_FOR_UPGRADE}" 161 def scmInfo = checkout([ 162 $class: 'GitSCM', 163 branches: [[name: COMMIT_FOR_UPGRADE]], 164 doGenerateSubmoduleConfigurations: false, 165 extensions: [], 166 submoduleCfg: [], 167 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 168 env.GIT_COMMIT = scmInfo.GIT_COMMIT 169 env.GIT_BRANCH = scmInfo.GIT_BRANCH 170 // If the commit we were handed is not what the SCM says we are using, fail 171 if (!env.GIT_COMMIT.equals(COMMIT_FOR_UPGRADE)) { 172 echo "SCM didn't checkout the commit we expected. Expected: ${COMMIT_FOR_UPGRADE}, Found: ${scmInfo.GIT_COMMIT}" 173 exit 1 174 } 175 } 176 177 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 178 179 } 180 181 sh """ 182 cp -f "${NETRC_FILE}" $HOME/.netrc 183 chmod 600 $HOME/.netrc 184 """ 185 186 script { 187 try { 188 sh """ 189 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 190 """ 191 } catch(error) { 192 echo "docker login failed, retrying after sleep" 193 retry(4) { 194 sleep(30) 195 sh """ 196 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 197 """ 198 } 199 } 200 } 201 202 sh """ 203 rm -rf ${GO_REPO_PATH}/verrazzano 204 mkdir -p ${GO_REPO_PATH}/verrazzano 205 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 206 """ 207 208 script { 209 def props = readProperties file: '.verrazzano-development-version' 210 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 211 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 212 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 213 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 214 // update the description with some meaningful info 215 setDisplayName() 216 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_FOR_UPGRADE 217 OPERATOR_YAML_FILE = sh(returnStdout: true, script: "ci/scripts/derive_operator_yaml.sh ${params.VERSION_FOR_INSTALL}").trim() 218 echo "Downloading ${OPERATOR_YAML_FILE} for release ${params.VERSION_FOR_INSTALL}" 219 } 220 221 script { 222 sh """ 223 echo "Downloading VZ CLI from object storage" 224 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 225 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 226 ${GO_REPO_PATH}/vz version 227 """ 228 } 229 } 230 } 231 stage('Install Release Version') { 232 environment { 233 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 234 OCI_CLI_AUTH="instance_principal" 235 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 236 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 237 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 238 OPERATOR_YAML="${OPERATOR_YAML_FILE}" 239 CLUSTER_SNAPSHOT_DIR="${WORKSPACE}/verrazzano/build/resources/pre-install-resources" 240 } 241 steps { 242 sh """ 243 cd ${GO_REPO_PATH}/verrazzano 244 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_KIND_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} 245 """ 246 } 247 post { 248 always { 249 sh """ 250 ## dump out install logs 251 mkdir -p ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs 252 kubectl -n verrazzano-install logs --selector=job-name=verrazzano-install-my-verrazzano --tail -1 > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-install.log 253 kubectl -n verrazzano-install describe pod --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-install-job-pod.out 254 cp ${INSTALL_CONFIG_FILE_KIND} ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs 255 echo "Verrazzano Installation logs dumped to verrazzano-install.log" 256 echo "Verrazzano Install pod description dumped to verrazzano-install-job-pod.out" 257 echo "------------------------------------------" 258 """ 259 script { 260 dumpVerrazzanoPlatformOperatorLogs("before-upgrade") 261 } 262 archiveArtifacts artifacts: "**/logs/**,${env.INSTALL_CONFIG_FILE_KIND}", allowEmptyArchive: true 263 } 264 } 265 } 266 267 stage('examples helidon') { 268 steps { 269 runGinkgoNamespace('examples/helidon', "false", "true", "hello-helidon") 270 runGinkgoRandomize('upgrade/pre-upgrade/verify') 271 } 272 post { 273 always { 274 archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true 275 junit testResults: '**/*test-result.xml', allowEmptyResults: true 276 } 277 failure { 278 script { 279 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 280 dumpK8sCluster('pre-upgrade-tests-cluster-snapshot') 281 } 282 } 283 } 284 success { 285 script { 286 if (params.DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 287 dumpK8sCluster('install-success-cluster-snapshot') 288 } 289 } 290 } 291 } 292 } 293 stage("Upgrade Platform Operator") { 294 environment { 295 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 296 OCI_CLI_AUTH="instance_principal" 297 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 298 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 299 } 300 steps { 301 sh """ 302 git fetch 303 """ 304 script { 305 VERRAZZANO_DEV_VERSION = sh(returnStdout: true, script: "oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/generated-verrazzano-bom.json --file - | jq -r '.version'").trim() 306 } 307 sh """ 308 # Download the operator.yaml for the target version that we are upgrading to 309 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 310 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/upgrade-test-operator.yaml 311 312 echo "Upgrading the Verrazzano platform operator" 313 if [[ "upgrade.failed.upgrade" == ${params.CHAOS_TEST_TYPE} || "upgrade.failing.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 314 then 315 echo "Upgrading VPO to interim version ${params.INTERIM_UPGRADE_VERSION}" 316 if [[ ${params.INTERIM_UPGRADE_VERSION} == "v1.3."* ]] 317 then 318 kubectl apply -f https://github.com/verrazzano/verrazzano/releases/download/${params.INTERIM_UPGRADE_VERSION}/operator.yaml 319 else 320 kubectl apply -f https://github.com/verrazzano/verrazzano/releases/download/${params.INTERIM_UPGRADE_VERSION}/verrazzano-platform-operator.yaml 321 fi 322 else 323 echo "Upgrading VPO to this commit version" 324 kubectl apply -f ${WORKSPACE}/upgrade-test-operator.yaml 325 fi 326 327 328 # ensure operator pod is up 329 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 330 331 # need to sleep since the old operator needs to transition to terminating state 332 sleep 45 333 """ 334 335 } 336 post { 337 always { 338 archiveArtifacts artifacts: "upgrade-test-operator.yaml", allowEmptyArchive: true 339 } 340 failure { 341 script { 342 if (fileExists(env.TESTS_EXECUTED_FILE)) { 343 dumpK8sCluster('post-upgrade-vpo-failure-cluster-snapshot') 344 } 345 } 346 } 347 } 348 } 349 350 stage("Upgrade Verrazzano and execute selected test") { 351 environment { 352 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 353 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 354 OCI_CLI_AUTH="instance_principal" 355 } 356 steps { 357 script { 358 // Download the bom for the target version that we are upgrading to, then extract the version 359 // Note, this version will have a semver suffix which is generated for each build, e.g. 1.0.1-33+d592fed6 360 VERRAZZANO_INTERIM_VERSION = "NONE" 361 if ("upgrade.failed.upgrade".equals(params.CHAOS_TEST_TYPE)) { 362 VERRAZZANO_INTERIM_VERSION = params.INTERIM_UPGRADE_VERSION 363 } else if ("upgrade.failing.upgrade".equals(params.CHAOS_TEST_TYPE)) { 364 VERRAZZANO_INTERIM_VERSION = params.INTERIM_UPGRADE_VERSION 365 } 366 VERRAZZANO_DEV_VERSION = sh(returnStdout: true, script: "curl https://objectstorage.us-phoenix-1.oraclecloud.com/n/stevengreenberginc/b/${OCI_OS_COMMIT_BUCKET}/o/ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/generated-verrazzano-bom.json | jq -r '.version'").trim() 367 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 368 } 369 sh """ 370 # Modify the version field in the Verrazzano CR file to be this new Verrazzano version 371 cd ${GO_REPO_PATH}/verrazzano 372 cp ${INSTALL_CONFIG_FILE_KIND} ${WORKSPACE}/verrazzano-upgrade-cr.yaml 373 ${TEST_SCRIPTS_DIR}/process_upgrade_yaml.sh ${VERRAZZANO_DEV_VERSION} ${WORKSPACE}/verrazzano-upgrade-cr.yaml 374 if [[ "NONE" != ${VERRAZZANO_INTERIM_VERSION} ]] 375 then 376 # create the interim VZ CR 377 cp ${INSTALL_CONFIG_FILE_KIND} ${WORKSPACE}/verrazzano-interim-cr.yaml 378 ${TEST_SCRIPTS_DIR}/process_upgrade_yaml.sh ${VERRAZZANO_INTERIM_VERSION} ${WORKSPACE}/verrazzano-interim-cr.yaml 379 cp ${WORKSPACE}/verrazzano-interim-cr.yaml ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs 380 fi 381 # Copy upgrade CR where it will be archived 382 cp ${WORKSPACE}/verrazzano-upgrade-cr.yaml ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs 383 384 # Get the install job in verrazzano-install namespace 385 kubectl -n verrazzano-install get job -o yaml --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-pre-upgrade-job.out 386 387 echo "Following is the verrazzano CR file with the new version:" 388 cat ${WORKSPACE}/verrazzano-upgrade-cr.yaml 389 390 echo "CHAOS TEST TYPE = ${params.CHAOS_TEST_TYPE}" 391 392 if [[ "ephemeral.storage.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 393 then 394 # Initiate upgrade 395 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 396 397 # A successful completion of the upgrade means the Keycloak configuration was successfully rebuilt 398 kubectl wait --timeout=45m --for=condition=UpgradeComplete verrazzano/my-verrazzano 399 400 # Kill MySQL and wait for Keycloak to be healthy again 401 ${GO_REPO_PATH}/verrazzano/ci/scripts/chaos_test_ephemeral_storage.sh 402 fi 403 404 if [[ "vpo.killed" == ${params.CHAOS_TEST_TYPE} ]] 405 then 406 # Initiate upgrade 407 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 408 # Wait 15 seconds then kill vpo 409 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 410 echo "Wait 15 seconds and kill VPO \$POD" 411 sleep 15 412 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 413 kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-before-vpo-killed-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 414 echo "dumping out the VZ CR:" 415 kubectl get verrazzano my-verrazzano -o yaml 416 echo "dumping out the VMI" 417 kubectl get vmi system -n verrazzano-system -o yaml 418 kubectl -n verrazzano-install delete po \$POD 419 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 420 kubectl -n verrazzano-install wait --for=condition=ready --timeout=600s pod/\$POD 421 echo "VPO \$POD successfully restarted" 422 fi 423 424 425 if [[ "helm.chart.corrupted" == ${params.CHAOS_TEST_TYPE} ]] 426 then 427 # Move values.yaml so helm install of Verrazzano will fail 428 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 429 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak" 430 # Initiate upgrade 431 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 432 # wait for the expected error message 433 until [ `kubectl logs -l app=verrazzano-platform-operator -n verrazzano-install --tail -1 | grep "Failed running Helm command for release verrazzano" | wc -l` -gt 0 ] 434 do 435 echo "Waiting for the error message ..." 436 sleep 30 437 done 438 # Fix it 439 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml" 440 fi 441 442 if [[ "upgrade.failed.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 443 then 444 # Move values.yaml so helm install of Verrazzano will fail 445 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 446 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak" 447 # Initiate upgrade 448 kubectl apply -f ${WORKSPACE}/verrazzano-interim-cr.yaml 449 # The upgrade will indefinitely remain in the upgrading state. Wait for indication that the Verrazzano component is failing to upgrade 450 until [ `kubectl logs -l app=verrazzano-platform-operator -n verrazzano-install --tail -1 | grep "Failed upgrading component verrazzano" | wc -l` -gt 0 ] 451 do 452 echo "Waiting for Verrazzano to fail upgrade ..." 453 sleep 30 454 done 455 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 456 kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-before-final-upgrade-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 457 # Upgrade to the latest VPO 458 kubectl apply -f ${WORKSPACE}/upgrade-test-operator.yaml 459 # ensure operator pod is up 460 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 461 # need to sleep since the old operator needs to transition to terminating state 462 sleep 45 463 # apply the CR 464 echo "Following is the verrazzano CR file with the updated version:" 465 cat ${WORKSPACE}/verrazzano-upgrade-cr.yaml 466 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 467 fi 468 469 if [[ "upgrade.failing.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 470 then 471 # Move values.yaml so helm install of Verrazzano will fail 472 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 473 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak" 474 # Initiate upgrade 475 kubectl apply -f ${WORKSPACE}/verrazzano-interim-cr.yaml 476 # wait for the expected error 477 until [ `kubectl logs -l app=verrazzano-platform-operator -n verrazzano-install --tail -1 | grep "Failed running Helm command for release verrazzano" | wc -l` -gt 0 ] 478 do 479 echo "Waiting for the error message ..." 480 sleep 30 481 done 482 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 483 kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-before-final-upgrade-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 484 # Make sure the upgrade is still in progress 485 vz_status=\$(kubectl get vz -o jsonpath='{.items[0].status.conditions[?(@.type=="UpgradeStarted")].type}') 486 if [[ \${vz_status} == "UpgradeStarted" ]] 487 then 488 # Upgrade to the latest VPO 489 kubectl apply -f ${WORKSPACE}/upgrade-test-operator.yaml 490 # ensure operator pod is up 491 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 492 # need to sleep since the old operator needs to transition to terminating state 493 sleep 45 494 # apply the CR 495 echo "Following is the verrazzano CR file with the updated version:" 496 cat ${WORKSPACE}/verrazzano-upgrade-cr.yaml 497 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 498 fi 499 fi 500 501 if [[ "uninstall.failed.upgrade" == ${params.CHAOS_TEST_TYPE} ]] 502 then 503 # Move values.yaml so helm install of Verrazzano will fail 504 POD=\$(kubectl get pod -l app=verrazzano-platform-operator -n verrazzano-install -o jsonpath="{.items[0].metadata.name}") 505 kubectl -n verrazzano-install exec \$POD -- /bin/bash -c "mv /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml /verrazzano/platform-operator/helm_config/charts/verrazzano/values.yaml.bak" 506 # Initiate upgrade 507 kubectl apply -f ${WORKSPACE}/verrazzano-upgrade-cr.yaml 508 # wait for the expected error message 509 until [ `kubectl logs -l app=verrazzano-platform-operator -n verrazzano-install --tail -1 | grep "Failed running Helm command for release verrazzano" | wc -l` -gt 0 ] 510 do 511 echo "Waiting for the error message ..." 512 sleep 30 513 done 514 # Make sure the upgrade is still in progress 515 vz_status=\$(kubectl get vz -o jsonpath='{.items[0].status.conditions[?(@.type=="UpgradeStarted")].type}') 516 if [[ \${vz_status} == "UpgradeStarted" ]]; then 517 ${GO_REPO_PATH}/vz uninstall -y --timeout 45m 518 519 ## dump out uninstall logs 520 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/uninstall/build/logs 521 kubectl logs --selector=job-name=verrazzano-uninstall-my-verrazzano -n verrazzano-install > ${WORKSPACE}/verrazzano-platform-operator/scripts/uninstall/build/logs/verrazzano-uninstall.log --tail -1 522 kubectl describe pod --selector=job-name=verrazzano-uninstall-my-verrazzano -n verrazzano-install > ${WORKSPACE}/verrazzano-platform-operator/scripts/uninstall/build/logs/verrazzano-uninstall-job-pod.out 523 echo "Listing all pods in all namespaces after uninstall" 524 kubectl get pods --all-namespaces 525 echo "-----------------------------------------------------" 526 527 528 # verify Uninstall 529 ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/post-uninstall-resources false 530 ${LOOPING_TEST_SCRIPTS_DIR}/verify_uninstall.sh ${WORKSPACE}/verrazzano/build/resources 531 else 532 echo "Upgrade should still be in progress. Exiting with error" 533 exit 1 534 fi 535 fi 536 537 if [[ "uninstall.failed.upgrade" != ${params.CHAOS_TEST_TYPE} ]] 538 then 539 # wait for the upgrade to complete 540 kubectl wait --timeout=45m --for=condition=UpgradeComplete verrazzano/my-verrazzano 541 542 # Dump the resource to debug 543 kubectl get -o yaml verrazzano/my-verrazzano || true 544 545 # ideally we don't need to wait here 546 sleep 15 547 echo "helm list : releases across all namespaces, after upgrading Verrazzano installation ..." 548 helm list -A 549 550 # Get the install job(s) and mke sure the it matches pre-install. If there is more than 1 job or the job changed, then it won't match 551 kubectl -n verrazzano-install get job -o yaml --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-post-upgrade-job.out 552 553 echo "Ensuring that the install job(s) in verrazzzano-system are identical pre and post install" 554 cmp -s ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-pre-upgrade-job.out ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-post-upgrade-job.out 555 fi 556 557 """ 558 } 559 post { 560 always { 561 dumpVerrazzanoPlatformOperatorLogs("after-upgrade") 562 archiveArtifacts artifacts: "verrazzano-upgrade-cr.yaml", allowEmptyArchive: true 563 } 564 failure { 565 script { 566 if (fileExists(env.TESTS_EXECUTED_FILE)) { 567 dumpK8sCluster('upgrade-failure-cluster-snapshot') 568 } 569 } 570 } 571 } 572 } 573 574 stage('Post-upgrade Acceptance Tests') { 575 when { 576 expression{params.CHAOS_TEST_TYPE != "uninstall.failed.upgrade"} 577 } 578 parallel { 579 stage('verify-upgrade post-upgrade') { 580 steps { 581 runGinkgoRandomize('upgrade/post-upgrade/verify') 582 } 583 } 584 stage('examples helidon') { 585 steps { 586 runGinkgoNamespace('examples/helidon', "true", "false", "hello-helidon") 587 } 588 } 589 // run todo example test to verify upgraded app/weblogic/coherence operators are functioning correctly 590 stage('examples todo') { 591 environment { 592 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/todo" 593 } 594 steps { 595 runGinkgoNamespace('examples/todo', "false", "false", "todo-list") 596 } 597 } 598 } 599 post { 600 always { 601 archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true 602 junit testResults: '**/*test-result.xml', allowEmptyResults: true 603 } 604 failure { 605 script { 606 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 607 dumpK8sCluster('post-upgrade-failure-cluster-snapshot') 608 } 609 } 610 } 611 success { 612 script { 613 if (params.DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 614 dumpK8sCluster('post-upgrade-success-cluster-snapshot') 615 } 616 } 617 } 618 } 619 } 620 } 621 622 post { 623 always { 624 script { 625 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 626 dumpVerrazzanoSystemPods() 627 dumpCattleSystemPods() 628 dumpNginxIngressControllerLogs() 629 dumpVerrazzanoApplicationOperatorLogs() 630 dumpOamKubernetesRuntimeLogs() 631 dumpVerrazzanoApiLogs() 632 } 633 } 634 635 sh """ 636 # Copy the generated test reports to WORKSPACE to archive them 637 mkdir -p ${TEST_REPORT_DIR} 638 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 639 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 640 """ 641 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/build/resources/**,**/*full-cluster*/**,**/bug-report/**,**/${TEST_REPORT}", allowEmptyArchive: true 642 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 643 644 sh """ 645 cd ${GO_REPO_PATH}/verrazzano/platform-operator 646 make delete-cluster 647 cd ${WORKSPACE}/verrazzano 648 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 649 echo "Failures seen during dumping of artifacts, treat post as failed" 650 exit 1 651 fi 652 """ 653 } 654 cleanup { 655 deleteDir() 656 } 657 } 658 } 659 660 def runGinkgoRandomize(testSuitePath) { 661 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 662 sh """ 663 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 664 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 665 """ 666 } 667 } 668 669 def runGinkgo(testSuitePath, skipDeploy, skipUndeploy) { 670 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 671 sh """ 672 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 673 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} 674 """ 675 } 676 } 677 678 def runGinkgoNamespace(testSuitePath, skipDeploy, skipUndeploy, namespace) { 679 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 680 sh """ 681 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 682 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace} 683 """ 684 } 685 } 686 687 def dumpK8sCluster(dumpDirectory) { 688 sh """ 689 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 690 """ 691 } 692 693 def dumpVerrazzanoSystemPods() { 694 sh """ 695 cd ${GO_REPO_PATH}/verrazzano/platform-operator 696 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log" 697 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 698 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log" 699 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 700 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log" 701 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 702 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log" 703 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 704 """ 705 } 706 707 def dumpCattleSystemPods() { 708 sh """ 709 cd ${GO_REPO_PATH}/verrazzano/platform-operator 710 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/cattle-system-pods.log" 711 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 712 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/rancher.log" 713 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 714 """ 715 } 716 717 def dumpNginxIngressControllerLogs() { 718 sh """ 719 cd ${GO_REPO_PATH}/verrazzano/platform-operator 720 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log" 721 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 722 """ 723 } 724 725 def dumpVerrazzanoPlatformOperatorLogs(stage) { 726 sh """ 727 ## dump out verrazzano-platform-operator logs 728 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 729 kubectl -n verrazzano-install logs --tail -1 --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 730 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 731 echo "------------------------------------------" 732 """ 733 } 734 735 def dumpVerrazzanoApplicationOperatorLogs() { 736 sh """ 737 ## dump out verrazzano-application-operator logs 738 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 739 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator --tail -1 > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 740 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 741 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 742 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 743 echo "------------------------------------------" 744 """ 745 } 746 747 def dumpOamKubernetesRuntimeLogs() { 748 sh """ 749 ## dump out oam-kubernetes-runtime logs 750 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 751 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime --tail -1 > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 752 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 753 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 754 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 755 echo "------------------------------------------" 756 """ 757 } 758 759 def dumpVerrazzanoApiLogs() { 760 sh """ 761 cd ${GO_REPO_PATH}/verrazzano/platform-operator 762 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log" 763 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 764 """ 765 } 766 767 def setDisplayName() { 768 echo "Start setDisplayName" 769 def causes = currentBuild.getBuildCauses() 770 echo "causes: " + causes.toString() 771 for (cause in causes) { 772 def causeString = cause.toString() 773 echo "current cause: " + causeString 774 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 775 echo "This job was caused by " + causeString 776 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 777 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 778 } else if (causeString.contains("verrazzano-flaky-tests")) { 779 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 780 } 781 } 782 } 783 echo "End setDisplayName" 784 } 785 786 def getEffectiveDumpOnSuccess() { 787 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 788 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 789 effectiveValue = true 790 echo "Forcing dump on success based on global override setting" 791 } 792 return effectiveValue 793 } 794 795 def listNamespacesAndPods(customMessage) { 796 sh """ 797 echo "Listing all the namespaces and pods the namespaces ${customMessage}." 798 kubectl get namespaces 799 kubectl get pods -A 800 echo "-----------------------------------------------------" 801 """ 802 } 803 804 def listHelmReleases(customMessage) { 805 sh """ 806 echo "Listing the releases across all namespaces ${customMessage}." 807 helm list -A 808 echo "-----------------------------------------------------" 809 """ 810 } 811 812 def restartExampleApps() { 813 sh """ 814 kubectl rollout restart deployment -n springboot 815 816 kubectl rollout restart deployment -n hello-helidon 817 """ 818 }