github.com/verrazzano/verrazzano@v1.7.0/ci/ha/JenkinsfileRollingUpgrade (about) 1 // Copyright (c) 2022, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 7 def TESTS_FAILED = false 8 9 def availableRegions = [ "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", 10 "eu-zurich-1", "me-jeddah-1", "sa-saopaulo-1" ] 11 def OKE_CLUSTER_PREFIX = "" 12 def OKE_CLUSTER_ID = "" 13 Collections.shuffle(availableRegions) 14 15 pipeline { 16 options { 17 timeout(time: 5, unit: 'HOURS') 18 skipDefaultCheckout true 19 timestamps () 20 } 21 22 agent { 23 docker { 24 image "${RUNNER_DOCKER_IMAGE}" 25 args "${RUNNER_DOCKER_ARGS}" 26 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 27 registryCredentialsId 'ocir-pull-and-push-account' 28 label "${agentLabel}" 29 } 30 } 31 32 parameters { 33 choice (name: 'KUBERNETES_CLUSTER_VERSION', 34 description: 'Kubernetes Version for OKE Cluster', 35 // 1st choice is the default value, use older version as default to ensure cluster will perform upgrade 36 choices: [ "v1.25.4", "v1.25.12", "v1.26.2", "v1.26.7", "v1.27.2" ]) 37 choice (name: 'OKE_NODE_POOL', 38 description: 'OKE node pool configuration', 39 // 1st choice is the default value 40 choices: [ "VM.Standard.E3.Flex-4-4", "VM.Standard2.4-4", "VM.Standard2.4-2" ]) 41 choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION', 42 // 1st choice is the default value 43 choices: availableRegions ) 44 booleanParam (name: 'SKIP_KUBERNETES_UPGRADE', 45 description: 'Skip Kubernetes version upgrade (simulate upgrade, drain and replace worker nodes only)', 46 defaultValue: false) 47 string (name: 'GIT_COMMIT_TO_USE', 48 defaultValue: 'NONE', 49 description: 'This is the full git commit hash from the source build to be used for all jobs', 50 trim: true) 51 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 52 defaultValue: 'NONE', 53 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator', 54 trim: true) 55 choice (name: 'WILDCARD_DNS_DOMAIN', 56 description: 'This is the wildcard DNS domain', 57 // 1st choice is the default value 58 choices: [ "nip.io", "sslip.io"]) 59 booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 60 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 61 string (name: 'TAGGED_TESTS', 62 defaultValue: '', 63 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 64 trim: true) 65 string (name: 'INCLUDED_TESTS', 66 defaultValue: '.*', 67 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 68 trim: true) 69 string (name: 'EXCLUDED_TESTS', 70 defaultValue: '_excluded_test', 71 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 72 trim: true) 73 string (name: 'CONSOLE_REPO_BRANCH', 74 defaultValue: '', 75 description: 'The branch to check out after cloning the console repository.', 76 trim: true) 77 booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true) 78 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 79 } 80 81 environment { 82 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 83 TF_VAR_tenancy_id = credentials('oci-tenancy') 84 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 85 TF_VAR_user_id = credentials('oci-user-ocid') 86 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 87 TF_VAR_kubernetes_version = "${params.KUBERNETES_CLUSTER_VERSION}" 88 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 89 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 90 TF_VAR_api_private_key_path = credentials('oci-api-key') 91 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 92 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 93 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 94 95 OCI_CLI_TENANCY = credentials('oci-tenancy') 96 OCI_CLI_USER = credentials('oci-user-ocid') 97 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 98 OCI_CLI_KEY_FILE = credentials('oci-api-key') 99 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 100 101 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 102 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 103 GOPATH = '/home/opc/go' 104 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 105 DOCKER_CREDS = credentials('github-packages-credentials-rw') 106 DOCKER_EMAIL = credentials('github-packages-email') 107 DOCKER_REPO = 'ghcr.io' 108 DOCKER_NAMESPACE = 'verrazzano' 109 NETRC_FILE = credentials('netrc') 110 CLUSTER_NAME = 'verrazzano' 111 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 112 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 113 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 114 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 115 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 116 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 117 OCR_CREDS = credentials('ocr-pull-and-push-account') 118 OCR_REPO = 'container-registry.oracle.com' 119 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 120 INSTALL_CONFIG_FILE = "./examples/ha/ha.yaml" 121 INSTALL_PROFILE = "prod" 122 VZ_ENVIRONMENT_NAME = "default" 123 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 124 HELIDON_TEST_NAMESPACE = "ha-hello-helidon" 125 126 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test 127 DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test 128 129 // Environment variables required to capture cluster snapshot and bug report on test failure 130 DUMP_KUBECONFIG="${KUBECONFIG}" 131 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 132 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 133 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 134 135 // Environment variable for Verrazzano CLI executable 136 VZ_COMMAND="${GO_REPO_PATH}/vz" 137 138 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 139 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 140 141 // used for console artifact capture on failure 142 JENKINS_READ = credentials('jenkins-auditor') 143 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 144 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 145 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 146 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 147 148 // used to emit metrics 149 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 150 TEST_ENV_LABEL = "magicdns_oke" 151 TEST_ENV = "magicdns_oke" 152 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 153 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 154 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 155 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 156 157 // used to generate Ginkgo test reports 158 TEST_REPORT = "test-report.xml" 159 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 160 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 161 } 162 163 stages { 164 stage('Clean workspace and checkout') { 165 steps { 166 sh """ 167 echo "${NODE_LABELS}" 168 """ 169 170 script { 171 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 172 if (params.GIT_COMMIT_TO_USE == "NONE") { 173 echo "Specific GIT commit was not specified, use current head" 174 def scmInfo = checkout scm 175 env.GIT_COMMIT = scmInfo.GIT_COMMIT 176 env.GIT_BRANCH = scmInfo.GIT_BRANCH 177 } else { 178 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 179 def scmInfo = checkout([ 180 $class : 'GitSCM', 181 branches : [[name: params.GIT_COMMIT_TO_USE]], 182 doGenerateSubmoduleConfigurations: false, 183 extensions : [], 184 submoduleCfg : [], 185 userRemoteConfigs : [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 186 env.GIT_COMMIT = scmInfo.GIT_COMMIT 187 env.GIT_BRANCH = scmInfo.GIT_BRANCH 188 // If the commit we were handed is not what the SCM says we are using, fail 189 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 190 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 191 exit 1 192 } 193 } 194 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 195 } 196 197 sh """ 198 cp -f "${NETRC_FILE}" $HOME/.netrc 199 chmod 600 $HOME/.netrc 200 """ 201 202 script { 203 try { 204 sh """ 205 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 206 """ 207 } catch (error) { 208 echo "docker login failed, retrying after sleep" 209 retry(4) { 210 sleep(30) 211 sh """ 212 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 213 """ 214 } 215 } 216 } 217 sh """ 218 rm -rf ${GO_REPO_PATH}/verrazzano 219 mkdir -p ${GO_REPO_PATH}/verrazzano 220 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 221 """ 222 223 script { 224 def props = readProperties file: '.verrazzano-development-version' 225 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 226 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 227 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 228 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 229 // update the description with some meaningful info 230 setDisplayName() 231 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 232 233 // derive the prefix for the OKE cluster 234 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 235 } 236 237 script { 238 sh """ 239 echo "Downloading VZ CLI from object storage" 240 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 241 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 242 ${GO_REPO_PATH}/vz version 243 """ 244 } 245 } 246 } 247 248 stage('Tests') { 249 stages { 250 stage('Create OKE Cluster') { 251 steps { 252 sh """ 253 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=rollingupgrade-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/create_oke_cluster.sh 254 kubectl get nodes 255 """ 256 257 script { 258 OKE_CLUSTER_ID = sh(returnStdout: true, script: "cd ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/terraform/cluster && ./terraform output -raw oke_cluster_id").trim() 259 echo "OKE cluster id: ${OKE_CLUSTER_ID}" 260 } 261 } 262 post { 263 failure { 264 script { 265 TESTS_FAILED = true 266 } 267 } 268 always { 269 archiveArtifacts artifacts: '**/*test_kubeconfig*', allowEmptyArchive: true 270 } 271 } 272 } 273 274 stage('Install Verrazzano') { 275 when { 276 expression { TESTS_FAILED == false } 277 } 278 steps { 279 sh """ 280 cd ${GO_REPO_PATH}/verrazzano 281 282 echo "Create the verrazzano-install namespace" 283 kubectl create namespace verrazzano-install 284 285 echo "Create Image Pull Secrets" 286 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 287 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 288 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 289 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 290 291 if [ "NONE" == "${VERRAZZANO_OPERATOR_IMAGE}" ]; then 292 echo "Downloading platform operator from object storage" 293 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/platform-operator.yaml 294 else 295 echo "Generating operator.yaml based on image name provided: ${VERRAZZANO_OPERATOR_IMAGE}" 296 env IMAGE_PULL_SECRETS=${IMAGE_PULL_SECRET} DOCKER_IMAGE=${VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/platform-operator.yaml 297 fi 298 299 echo "Installing Verrazzano" 300 ${GO_REPO_PATH}/vz install --filename ${INSTALL_CONFIG_FILE} --manifests ${WORKSPACE}/platform-operator.yaml --timeout 45m --platform-operator-timeout 10m 301 302 # For now, manually scale the VPO webhook to 3 replicas until we have hooks for it in the product 303 kubectl scale deployment -n verrazzano-install verrazzano-platform-operator-webhook --replicas 3 304 kubectl rollout status deployment -n verrazzano-install verrazzano-platform-operator-webhook -w --timeout 15m 305 """ 306 } 307 post { 308 failure { 309 script { 310 TESTS_FAILED = true 311 dumpK8sCluster('install-fail-cluster-snapshot') 312 } 313 } 314 always { 315 archiveArtifacts artifacts: '**/platform-operator.yaml', allowEmptyArchive: true 316 // enable debug logging of Verrazzano api istio proxy 317 script { 318 if (params.ENABLE_API_ENVOY_LOGGING) { 319 sh ''' 320 vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\") 321 if [ -z "\$vz_api_pod" ]; then 322 echo "Could not find verrazzano-authproxy pod, not enabling debug logging" 323 else 324 kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug 325 fi 326 nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\") 327 if [ -z "\$nginx_ing_pod" ]; then 328 echo "Could not find nginx ingress controller pod, not enabling debug logging" 329 else 330 kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug 331 fi 332 ''' 333 } 334 } 335 } 336 } 337 } 338 339 stage('examples helidon') { 340 when { 341 expression { TESTS_FAILED == false } 342 } 343 environment { 344 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/helidon-install" 345 } 346 steps{ 347 runGinkgo("examples/helidon", "--namespace=${HELIDON_TEST_NAMESPACE} --skipUndeploy=true --appconfig=examples/hello-helidon/hello-helidon-app.yaml") 348 } 349 post { 350 always { 351 sh """ 352 kubectl patch deployment -n ${HELIDON_TEST_NAMESPACE} hello-helidon-deployment -p '{"spec":{"replicas":3}}' --type=merge 353 kubectl wait --for=condition=Ready pods --all -n ${HELIDON_TEST_NAMESPACE} --timeout=300s 354 kubectl get pods -l app=hello-helidon -A 355 """ 356 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 357 junit testResults: '**/*test-result.xml', allowEmptyResults: true 358 } 359 failure { 360 script { 361 TESTS_FAILED = true 362 } 363 } 364 } 365 } 366 367 stage('Verify Install') { 368 when { 369 expression { TESTS_FAILED == false } 370 } 371 environment { 372 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install" 373 } 374 parallel { 375 stage('verify-install') { 376 steps { 377 runGinkgoRandomize('verify-install') 378 } 379 } 380 stage('verify-infra restapi') { 381 steps { 382 runGinkgoRandomize('verify-infra/restapi') 383 } 384 } 385 stage('verify-infra oam') { 386 steps { 387 runGinkgoRandomize('verify-infra/oam') 388 } 389 } 390 stage('verify-infra vmi') { 391 steps { 392 runGinkgoRandomize('verify-infra/vmi') 393 } 394 } 395 stage ('console') { 396 when { 397 expression { false } 398 } 399 environment { 400 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/console" 401 } 402 steps { 403 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 404 sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh" 405 } 406 } 407 } 408 } 409 post { 410 always { 411 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 412 junit testResults: '**/*test-result.xml', allowEmptyResults: true 413 } 414 failure { 415 script { 416 TESTS_FAILED = true 417 dumpK8sCluster('verify-install-fail-cluster-snapshot') 418 } 419 } 420 } 421 } 422 423 stage('Parallel HA Tests and K8s Upgrade') { 424 when { 425 expression { TESTS_FAILED == false } 426 } 427 environment { 428 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/ha-tests" 429 } 430 parallel { 431 stage('K8s In-place Upgrade') { 432 environment { 433 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/inplaceupgrade" 434 OKE_CLUSTER_ID="${OKE_CLUSTER_ID}" 435 } 436 steps { 437 runGinkgoFailFast('ha/inplaceupgrade') 438 } 439 } 440 stage('HA Tests') { 441 steps { 442 runGinkgoRandomize('ha/monitor') 443 } 444 } 445 stage('Access Hello Helidon') { 446 steps { 447 runGinkgo("ha/helidon", "--namespace=${HELIDON_TEST_NAMESPACE}") 448 } 449 } 450 stage ('console') { 451 when { 452 expression { false } 453 } 454 environment { 455 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/console" 456 } 457 steps { 458 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 459 sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh" 460 } 461 } 462 post { 463 failure { 464 sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh" 465 } 466 } 467 } 468 } 469 post { 470 always { 471 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 472 junit testResults: '**/*test-result.xml', allowEmptyResults: true 473 } 474 failure { 475 script { 476 TESTS_FAILED = true 477 dumpK8sCluster('rolling-upgrade-fail-cluster-snapshot') 478 } 479 } 480 } 481 } 482 } 483 484 post { 485 success { 486 script { 487 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 488 dumpK8sCluster('ha-rolling-upgrade-cluster-snapshot') 489 } 490 } 491 } 492 } 493 } 494 } 495 496 post { 497 always { 498 script { 499 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 500 dumpVerrazzanoSystemPods() 501 dumpCattleSystemPods() 502 dumpNginxIngressControllerLogs() 503 dumpVerrazzanoPlatformOperatorLogs() 504 dumpVerrazzanoApplicationOperatorLogs() 505 dumpOamKubernetesRuntimeLogs() 506 dumpVerrazzanoApiLogs() 507 } 508 } 509 510 sh """ 511 # Copy the generated test reports to WORKSPACE to archive them 512 mkdir -p ${TEST_REPORT_DIR} 513 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 514 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 515 """ 516 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true 517 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 518 deleteCluster() 519 } 520 failure { 521 sh """ 522 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 523 """ 524 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 525 sh """ 526 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 527 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 528 rm archive.zip 529 """ 530 } 531 cleanup { 532 sh "VERRAZZANO_KUBECONFIG=${env.KUBECONFIG} TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=rollingupgrade-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/delete_oke_cluster.sh || true" 533 deleteDir() 534 } 535 } 536 } 537 538 def runGinkgoRandomize(testSuitePath) { 539 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 540 sh """ 541 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 542 if [ -d "${testSuitePath}" ]; then 543 ginkgo --timeout=4h -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 544 fi 545 """ 546 } 547 } 548 549 def runGinkgo(testSuitePath, arguments = '') { 550 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 551 sh """ 552 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 553 ginkgo --timeout=4h -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- $arguments 554 """ 555 } 556 } 557 558 def runGinkgoFailFast(testSuitePath, arguments = '') { 559 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 560 sh """ 561 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 562 ginkgo --timeout=4h -v --fail-fast --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- $arguments 563 """ 564 } 565 } 566 567 def dumpK8sCluster(dumpDirectory) { 568 sh """ 569 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 570 """ 571 } 572 573 def dumpVerrazzanoSystemPods() { 574 sh """ 575 cd ${GO_REPO_PATH}/verrazzano/platform-operator 576 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log" 577 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 578 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log" 579 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 580 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log" 581 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 582 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log" 583 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 584 """ 585 } 586 587 def dumpCattleSystemPods() { 588 sh """ 589 cd ${GO_REPO_PATH}/verrazzano/platform-operator 590 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log" 591 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 592 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log" 593 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 594 """ 595 } 596 597 def dumpNginxIngressControllerLogs() { 598 sh """ 599 cd ${GO_REPO_PATH}/verrazzano/platform-operator 600 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log" 601 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 602 """ 603 } 604 605 def dumpVerrazzanoPlatformOperatorLogs() { 606 sh """ 607 ## dump out verrazzano-platform-operator logs 608 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 609 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 610 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 611 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 612 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 613 echo "------------------------------------------" 614 """ 615 } 616 617 def dumpVerrazzanoApplicationOperatorLogs() { 618 sh """ 619 ## dump out verrazzano-application-operator logs 620 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 621 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 622 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 623 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 624 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 625 echo "------------------------------------------" 626 """ 627 } 628 629 def dumpOamKubernetesRuntimeLogs() { 630 sh """ 631 ## dump out oam-kubernetes-runtime logs 632 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 633 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 634 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 635 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 636 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 637 echo "------------------------------------------" 638 """ 639 } 640 641 def dumpVerrazzanoApiLogs() { 642 sh """ 643 cd ${GO_REPO_PATH}/verrazzano/platform-operator 644 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log" 645 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 646 """ 647 } 648 649 def getEffectiveDumpOnSuccess() { 650 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 651 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 652 effectiveValue = true 653 echo "Forcing dump on success based on global override setting" 654 } 655 return effectiveValue 656 } 657 658 659 def deleteCluster() { 660 sh """ 661 cd ${GO_REPO_PATH}/verrazzano/platform-operator 662 make delete-cluster 663 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 664 echo "Failures seen during dumping of artifacts, treat post as failed" 665 exit 1 666 fi 667 """ 668 } 669 670 def setDisplayName() { 671 echo "Start setDisplayName" 672 def causes = currentBuild.getBuildCauses() 673 echo "causes: " + causes.toString() 674 for (cause in causes) { 675 def causeString = cause.toString() 676 echo "current cause: " + causeString 677 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 678 echo "This job was caused by " + causeString 679 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 680 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 681 } else if (causeString.contains("verrazzano-flaky-tests")) { 682 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 683 } 684 } 685 } 686 echo "End setDisplayName" 687 }