github.com/verrazzano/verrazzano@v1.7.1/ci/ha/JenkinsfileRollingUpgrade (about) 1 // Copyright (c) 2022, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 7 def TESTS_FAILED = false 8 9 def availableRegions = [ "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", 10 "eu-zurich-1", "me-jeddah-1", "sa-saopaulo-1" ] 11 def OKE_CLUSTER_PREFIX = "" 12 def OKE_CLUSTER_ID = "" 13 Collections.shuffle(availableRegions) 14 15 pipeline { 16 options { 17 timeout(time: 5, unit: 'HOURS') 18 skipDefaultCheckout true 19 timestamps () 20 } 21 22 agent { 23 docker { 24 image "${RUNNER_DOCKER_IMAGE}" 25 args "${RUNNER_DOCKER_ARGS}" 26 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 27 registryCredentialsId 'ocir-pull-and-push-account' 28 label "${agentLabel}" 29 } 30 } 31 32 parameters { 33 choice (name: 'KUBERNETES_CLUSTER_VERSION', 34 description: 'Kubernetes Version for OKE Cluster', 35 // 1st choice is the default value, use older version as default to ensure cluster will perform upgrade 36 choices: [ "v1.25.4", "v1.25.12", "v1.26.2", "v1.26.7", "v1.27.2" ]) 37 choice (name: 'OKE_NODE_POOL', 38 description: 'OKE node pool configuration', 39 // 1st choice is the default value 40 choices: [ "VM.Standard.E3.Flex-4-4", "VM.Standard2.4-4", "VM.Standard2.4-2" ]) 41 choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION', 42 // 1st choice is the default value 43 choices: availableRegions ) 44 booleanParam (name: 'SKIP_KUBERNETES_UPGRADE', 45 description: 'Skip Kubernetes version upgrade (simulate upgrade, drain and replace worker nodes only)', 46 defaultValue: false) 47 string (name: 'GIT_COMMIT_TO_USE', 48 defaultValue: 'NONE', 49 description: 'This is the full git commit hash from the source build to be used for all jobs', 50 trim: true) 51 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 52 defaultValue: 'NONE', 53 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator', 54 trim: true) 55 choice (name: 'WILDCARD_DNS_DOMAIN', 56 description: 'This is the wildcard DNS domain', 57 // 1st choice is the default value 58 choices: [ "nip.io", "sslip.io"]) 59 booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 60 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 61 string (name: 'TAGGED_TESTS', 62 defaultValue: '', 63 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 64 trim: true) 65 string (name: 'INCLUDED_TESTS', 66 defaultValue: '.*', 67 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 68 trim: true) 69 string (name: 'EXCLUDED_TESTS', 70 defaultValue: '_excluded_test', 71 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 72 trim: true) 73 string (name: 'CONSOLE_REPO_BRANCH', 74 defaultValue: '', 75 description: 'The branch to check out after cloning the console repository.', 76 trim: true) 77 booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true) 78 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 79 } 80 81 environment { 82 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 83 TF_VAR_tenancy_id = credentials('oci-tenancy') 84 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 85 TF_VAR_user_id = credentials('oci-user-ocid') 86 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 87 TF_VAR_kubernetes_version = "${params.KUBERNETES_CLUSTER_VERSION}" 88 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 89 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 90 TF_VAR_api_private_key_path = credentials('oci-api-key') 91 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 92 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 93 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 94 95 OCI_CLI_TENANCY = credentials('oci-tenancy') 96 OCI_CLI_USER = credentials('oci-user-ocid') 97 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 98 OCI_CLI_KEY_FILE = credentials('oci-api-key') 99 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 100 101 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 102 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 103 GOPATH = '/home/opc/go' 104 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 105 DOCKER_CREDS = credentials('github-packages-credentials-rw') 106 DOCKER_EMAIL = credentials('github-packages-email') 107 DOCKER_REPO = 'ghcr.io' 108 DOCKER_NAMESPACE = 'verrazzano' 109 NETRC_FILE = credentials('netrc') 110 CLUSTER_NAME = 'verrazzano' 111 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 112 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 113 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 114 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 115 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 116 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 117 OCR_CREDS = credentials('ocr-pull-and-push-account') 118 OCR_REPO = 'container-registry.oracle.com' 119 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 120 INSTALL_CONFIG_FILE = "./examples/ha/ha.yaml" 121 INSTALL_PROFILE = "prod" 122 VZ_ENVIRONMENT_NAME = "default" 123 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 124 HELIDON_TEST_NAMESPACE = "ha-hello-helidon" 125 126 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test 127 DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test 128 129 // Environment variables required to capture cluster snapshot and bug report on test failure 130 DUMP_KUBECONFIG="${KUBECONFIG}" 131 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 132 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 133 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 134 135 // Environment variable for Verrazzano CLI executable 136 VZ_COMMAND="${GO_REPO_PATH}/vz" 137 138 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 139 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 140 141 // used for console artifact capture on failure 142 JENKINS_READ = credentials('jenkins-auditor') 143 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 144 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 145 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 146 147 // used to emit metrics 148 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 149 TEST_ENV_LABEL = "magicdns_oke" 150 TEST_ENV = "magicdns_oke" 151 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 152 153 // used to generate Ginkgo test reports 154 TEST_REPORT = "test-report.xml" 155 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 156 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 157 } 158 159 stages { 160 stage('Clean workspace and checkout') { 161 steps { 162 sh """ 163 echo "${NODE_LABELS}" 164 """ 165 166 script { 167 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 168 if (params.GIT_COMMIT_TO_USE == "NONE") { 169 echo "Specific GIT commit was not specified, use current head" 170 def scmInfo = checkout scm 171 env.GIT_COMMIT = scmInfo.GIT_COMMIT 172 env.GIT_BRANCH = scmInfo.GIT_BRANCH 173 } else { 174 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 175 def scmInfo = checkout([ 176 $class : 'GitSCM', 177 branches : [[name: params.GIT_COMMIT_TO_USE]], 178 doGenerateSubmoduleConfigurations: false, 179 extensions : [], 180 submoduleCfg : [], 181 userRemoteConfigs : [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 182 env.GIT_COMMIT = scmInfo.GIT_COMMIT 183 env.GIT_BRANCH = scmInfo.GIT_BRANCH 184 // If the commit we were handed is not what the SCM says we are using, fail 185 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 186 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 187 exit 1 188 } 189 } 190 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 191 } 192 193 sh """ 194 cp -f "${NETRC_FILE}" $HOME/.netrc 195 chmod 600 $HOME/.netrc 196 """ 197 198 script { 199 try { 200 sh """ 201 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 202 """ 203 } catch (error) { 204 echo "docker login failed, retrying after sleep" 205 retry(4) { 206 sleep(30) 207 sh """ 208 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 209 """ 210 } 211 } 212 } 213 sh """ 214 rm -rf ${GO_REPO_PATH}/verrazzano 215 mkdir -p ${GO_REPO_PATH}/verrazzano 216 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 217 """ 218 219 script { 220 def props = readProperties file: '.verrazzano-development-version' 221 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 222 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 223 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 224 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 225 // update the description with some meaningful info 226 setDisplayName() 227 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 228 229 // derive the prefix for the OKE cluster 230 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 231 } 232 233 script { 234 sh """ 235 echo "Downloading VZ CLI from object storage" 236 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 237 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 238 ${GO_REPO_PATH}/vz version 239 """ 240 } 241 } 242 } 243 244 stage('Tests') { 245 stages { 246 stage('Create OKE Cluster') { 247 steps { 248 sh """ 249 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=rollingupgrade-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/create_oke_cluster.sh 250 kubectl get nodes 251 """ 252 253 script { 254 OKE_CLUSTER_ID = sh(returnStdout: true, script: "cd ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/terraform/cluster && ./terraform output -raw oke_cluster_id").trim() 255 echo "OKE cluster id: ${OKE_CLUSTER_ID}" 256 } 257 } 258 post { 259 failure { 260 script { 261 TESTS_FAILED = true 262 } 263 } 264 always { 265 archiveArtifacts artifacts: '**/*test_kubeconfig*', allowEmptyArchive: true 266 } 267 } 268 } 269 270 stage('Install Verrazzano') { 271 when { 272 expression { TESTS_FAILED == false } 273 } 274 steps { 275 sh """ 276 cd ${GO_REPO_PATH}/verrazzano 277 278 echo "Create the verrazzano-install namespace" 279 kubectl create namespace verrazzano-install 280 281 echo "Create Image Pull Secrets" 282 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 283 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 284 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 285 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 286 287 if [ "NONE" == "${VERRAZZANO_OPERATOR_IMAGE}" ]; then 288 echo "Downloading platform operator from object storage" 289 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/platform-operator.yaml 290 else 291 echo "Generating operator.yaml based on image name provided: ${VERRAZZANO_OPERATOR_IMAGE}" 292 env IMAGE_PULL_SECRETS=${IMAGE_PULL_SECRET} DOCKER_IMAGE=${VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/platform-operator.yaml 293 fi 294 295 echo "Installing Verrazzano" 296 ${GO_REPO_PATH}/vz install --filename ${INSTALL_CONFIG_FILE} --manifests ${WORKSPACE}/platform-operator.yaml --timeout 45m --platform-operator-timeout 10m 297 298 # For now, manually scale the VPO webhook to 3 replicas until we have hooks for it in the product 299 kubectl scale deployment -n verrazzano-install verrazzano-platform-operator-webhook --replicas 3 300 kubectl rollout status deployment -n verrazzano-install verrazzano-platform-operator-webhook -w --timeout 15m 301 """ 302 } 303 post { 304 failure { 305 script { 306 TESTS_FAILED = true 307 dumpK8sCluster('install-fail-cluster-snapshot') 308 } 309 } 310 always { 311 archiveArtifacts artifacts: '**/platform-operator.yaml', allowEmptyArchive: true 312 // enable debug logging of Verrazzano api istio proxy 313 script { 314 if (params.ENABLE_API_ENVOY_LOGGING) { 315 sh ''' 316 vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\") 317 if [ -z "\$vz_api_pod" ]; then 318 echo "Could not find verrazzano-authproxy pod, not enabling debug logging" 319 else 320 kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug 321 fi 322 nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\") 323 if [ -z "\$nginx_ing_pod" ]; then 324 echo "Could not find nginx ingress controller pod, not enabling debug logging" 325 else 326 kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug 327 fi 328 ''' 329 } 330 } 331 } 332 } 333 } 334 335 stage('examples helidon') { 336 when { 337 expression { TESTS_FAILED == false } 338 } 339 environment { 340 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/helidon-install" 341 } 342 steps{ 343 runGinkgo("examples/helidon", "--namespace=${HELIDON_TEST_NAMESPACE} --skipUndeploy=true --appconfig=examples/hello-helidon/hello-helidon-app.yaml") 344 } 345 post { 346 always { 347 sh """ 348 kubectl patch deployment -n ${HELIDON_TEST_NAMESPACE} hello-helidon-deployment -p '{"spec":{"replicas":3}}' --type=merge 349 kubectl wait --for=condition=Ready pods --all -n ${HELIDON_TEST_NAMESPACE} --timeout=300s 350 kubectl get pods -l app=hello-helidon -A 351 """ 352 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 353 junit testResults: '**/*test-result.xml', allowEmptyResults: true 354 } 355 failure { 356 script { 357 TESTS_FAILED = true 358 } 359 } 360 } 361 } 362 363 stage('Verify Install') { 364 when { 365 expression { TESTS_FAILED == false } 366 } 367 environment { 368 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install" 369 } 370 parallel { 371 stage('verify-install') { 372 steps { 373 runGinkgoRandomize('verify-install') 374 } 375 } 376 stage('verify-infra restapi') { 377 steps { 378 runGinkgoRandomize('verify-infra/restapi') 379 } 380 } 381 stage('verify-infra oam') { 382 steps { 383 runGinkgoRandomize('verify-infra/oam') 384 } 385 } 386 stage('verify-infra vmi') { 387 steps { 388 runGinkgoRandomize('verify-infra/vmi') 389 } 390 } 391 stage ('console') { 392 when { 393 expression { false } 394 } 395 environment { 396 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/console" 397 } 398 steps { 399 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 400 sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh" 401 } 402 } 403 } 404 } 405 post { 406 always { 407 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 408 junit testResults: '**/*test-result.xml', allowEmptyResults: true 409 } 410 failure { 411 script { 412 TESTS_FAILED = true 413 dumpK8sCluster('verify-install-fail-cluster-snapshot') 414 } 415 } 416 } 417 } 418 419 stage('Parallel HA Tests and K8s Upgrade') { 420 when { 421 expression { TESTS_FAILED == false } 422 } 423 environment { 424 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/ha-tests" 425 } 426 parallel { 427 stage('K8s In-place Upgrade') { 428 environment { 429 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/inplaceupgrade" 430 OKE_CLUSTER_ID="${OKE_CLUSTER_ID}" 431 } 432 steps { 433 runGinkgoFailFast('ha/inplaceupgrade') 434 } 435 } 436 stage('HA Tests') { 437 steps { 438 runGinkgoRandomize('ha/monitor') 439 } 440 } 441 stage('Access Hello Helidon') { 442 steps { 443 runGinkgo("ha/helidon", "--namespace=${HELIDON_TEST_NAMESPACE}") 444 } 445 } 446 stage ('console') { 447 when { 448 expression { false } 449 } 450 environment { 451 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/console" 452 } 453 steps { 454 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 455 sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh" 456 } 457 } 458 post { 459 failure { 460 sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh" 461 } 462 } 463 } 464 } 465 post { 466 always { 467 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 468 junit testResults: '**/*test-result.xml', allowEmptyResults: true 469 } 470 failure { 471 script { 472 TESTS_FAILED = true 473 dumpK8sCluster('rolling-upgrade-fail-cluster-snapshot') 474 } 475 } 476 } 477 } 478 } 479 480 post { 481 success { 482 script { 483 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 484 dumpK8sCluster('ha-rolling-upgrade-cluster-snapshot') 485 } 486 } 487 } 488 } 489 } 490 } 491 492 post { 493 always { 494 script { 495 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 496 dumpVerrazzanoSystemPods() 497 dumpCattleSystemPods() 498 dumpNginxIngressControllerLogs() 499 dumpVerrazzanoPlatformOperatorLogs() 500 dumpVerrazzanoApplicationOperatorLogs() 501 dumpOamKubernetesRuntimeLogs() 502 dumpVerrazzanoApiLogs() 503 } 504 } 505 506 sh """ 507 # Copy the generated test reports to WORKSPACE to archive them 508 mkdir -p ${TEST_REPORT_DIR} 509 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 510 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 511 """ 512 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true 513 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 514 deleteCluster() 515 } 516 failure { 517 sh """ 518 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 519 """ 520 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 521 sh """ 522 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 523 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 524 rm archive.zip 525 """ 526 } 527 cleanup { 528 sh "VERRAZZANO_KUBECONFIG=${env.KUBECONFIG} TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=rollingupgrade-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/delete_oke_cluster.sh || true" 529 deleteDir() 530 } 531 } 532 } 533 534 def runGinkgoRandomize(testSuitePath) { 535 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 536 sh """ 537 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 538 if [ -d "${testSuitePath}" ]; then 539 ginkgo --timeout=4h -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 540 fi 541 """ 542 } 543 } 544 545 def runGinkgo(testSuitePath, arguments = '') { 546 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 547 sh """ 548 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 549 ginkgo --timeout=4h -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- $arguments 550 """ 551 } 552 } 553 554 def runGinkgoFailFast(testSuitePath, arguments = '') { 555 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 556 sh """ 557 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 558 ginkgo --timeout=4h -v --fail-fast --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- $arguments 559 """ 560 } 561 } 562 563 def dumpK8sCluster(dumpDirectory) { 564 sh """ 565 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 566 """ 567 } 568 569 def dumpVerrazzanoSystemPods() { 570 sh """ 571 cd ${GO_REPO_PATH}/verrazzano/platform-operator 572 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log" 573 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 574 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log" 575 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 576 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log" 577 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 578 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log" 579 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 580 """ 581 } 582 583 def dumpCattleSystemPods() { 584 sh """ 585 cd ${GO_REPO_PATH}/verrazzano/platform-operator 586 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log" 587 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 588 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log" 589 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 590 """ 591 } 592 593 def dumpNginxIngressControllerLogs() { 594 sh """ 595 cd ${GO_REPO_PATH}/verrazzano/platform-operator 596 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log" 597 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 598 """ 599 } 600 601 def dumpVerrazzanoPlatformOperatorLogs() { 602 sh """ 603 ## dump out verrazzano-platform-operator logs 604 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 605 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 606 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 607 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 608 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 609 echo "------------------------------------------" 610 """ 611 } 612 613 def dumpVerrazzanoApplicationOperatorLogs() { 614 sh """ 615 ## dump out verrazzano-application-operator logs 616 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 617 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 618 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 619 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 620 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 621 echo "------------------------------------------" 622 """ 623 } 624 625 def dumpOamKubernetesRuntimeLogs() { 626 sh """ 627 ## dump out oam-kubernetes-runtime logs 628 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 629 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 630 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 631 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 632 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 633 echo "------------------------------------------" 634 """ 635 } 636 637 def dumpVerrazzanoApiLogs() { 638 sh """ 639 cd ${GO_REPO_PATH}/verrazzano/platform-operator 640 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log" 641 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 642 """ 643 } 644 645 def getEffectiveDumpOnSuccess() { 646 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 647 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 648 effectiveValue = true 649 echo "Forcing dump on success based on global override setting" 650 } 651 return effectiveValue 652 } 653 654 655 def deleteCluster() { 656 sh """ 657 cd ${GO_REPO_PATH}/verrazzano/platform-operator 658 make delete-cluster 659 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 660 echo "Failures seen during dumping of artifacts, treat post as failed" 661 exit 1 662 fi 663 """ 664 } 665 666 def setDisplayName() { 667 echo "Start setDisplayName" 668 def causes = currentBuild.getBuildCauses() 669 echo "causes: " + causes.toString() 670 for (cause in causes) { 671 def causeString = cause.toString() 672 echo "current cause: " + causeString 673 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 674 echo "This job was caused by " + causeString 675 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 676 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 677 } else if (causeString.contains("verrazzano-flaky-tests")) { 678 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 679 } 680 } 681 } 682 echo "End setDisplayName" 683 }