github.com/verrazzano/verrazzano@v1.7.1/ci/backup/JenkinsfileAlllBackupOKETest (about) 1 // Copyright (c) 2022, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 6 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 7 // pulling "ap-*" from the test regions given discovery of image pull issues 8 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ] 9 def OKE_CLUSTER_PREFIX = "" 10 def ociOsBucketName = UUID.randomUUID().toString().substring(0,6).replace('-','') 11 def backup_id = 'dummy' 12 Collections.shuffle(availableRegions) 13 14 pipeline { 15 options { 16 skipDefaultCheckout true 17 } 18 19 agent { 20 docker { 21 image "${RUNNER_DOCKER_IMAGE}" 22 args "${RUNNER_DOCKER_ARGS}" 23 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 24 registryCredentialsId 'ocir-pull-and-push-account' 25 label "$agentLabel" 26 } 27 } 28 29 parameters { 30 string (name: 'VERRAZZANO_BRANCH', 31 defaultValue: 'master', 32 description: 'Branch to clone and checkout the Verrazzano repo', 33 trim: true) 34 choice (name: 'OKE_NODE_POOL', 35 description: 'OKE node pool configuration', 36 // 1st choice is the default value 37 choices: [ "VM.Standard.E2.4","VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2",]) 38 choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION', 39 // 1st choice is the default value 40 choices: availableRegions ) 41 string (name: 'GIT_COMMIT_TO_USE', 42 defaultValue: 'NONE', 43 description: 'This is the full git commit hash from the source build to be used for all jobs', 44 trim: true) 45 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 46 defaultValue: 'NONE', 47 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator', 48 trim: true) 49 choice (name: 'OKE_CLUSTER_VERSION', 50 description: 'Kubernetes Version for OKE Cluster', 51 // 1st choice is the default value 52 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 53 choice (name: 'CRD_API_VERSION', 54 description: 'This is the API crd version.', 55 // 1st choice is the default value 56 choices: [ "v1beta1", "v1alpha1"]) 57 booleanParam (name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', 58 defaultValue: false, 59 description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)') 60 string (name: 'TAGGED_TESTS', 61 defaultValue: '', 62 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 63 trim: true) 64 string (name: 'INCLUDED_TESTS', 65 defaultValue: '.*', 66 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 67 trim: true) 68 string (name: 'EXCLUDED_TESTS', 69 defaultValue: '_excluded_test', 70 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 71 trim: true) 72 booleanParam (description: 'Whether to persist object store bucket', name: 'DELETE_BUCKET', defaultValue: true) 73 booleanParam (description: 'Whether to execute MySQL backup tests', name: 'MYSQL_BACKUP', defaultValue: true) 74 booleanParam (description: 'Whether to execute MySQL backup tests with S3 API. Default is true. If unchecked user principal creds will be used', name: 'MYSQL_BACKUP_S3', defaultValue: true) 75 booleanParam (description: 'Whether to execute OpenSearch backup tests', name: 'OPENSEARCH_BACKUP', defaultValue: true) 76 booleanParam (description: 'Whether to execute Rancher backup tests', name: 'RANCHER_BACKUP', defaultValue: true) 77 } 78 79 environment { 80 OCR_CREDS = credentials('ocr-pull-and-push-account') 81 NETRC_FILE = credentials('netrc') 82 GOPATH = '/home/opc/go' 83 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 84 DOCKER_CREDS = credentials('github-packages-credentials-rw') 85 DOCKER_EMAIL = credentials('github-packages-email') 86 DOCKER_REPO = 'ghcr.io' 87 88 OCR_REPO = 'container-registry.oracle.com' 89 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 90 91 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 92 TF_VAR_tenancy_id = credentials('oci-tenancy') 93 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 94 TF_VAR_user_id = credentials('oci-user-ocid') 95 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 96 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 97 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 98 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 99 TF_VAR_api_private_key_path = credentials('oci-api-key') 100 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 101 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 102 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 103 104 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 105 OCI_CLI_TENANCY = credentials('oci-tenancy') 106 OCI_CLI_USER = credentials('oci-user-ocid') 107 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 108 OCI_CLI_KEY_FILE = credentials('oci-api-key') 109 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 110 DISABLE_SPINNER=1 111 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 112 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 113 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 114 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 115 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 116 KUBECONFIG = "${WORKSPACE}/oke_kubeconfig" 117 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 118 INSTALL_PROFILE = "prod" 119 VZ_ENVIRONMENT_NAME = "default" 120 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 121 LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test" 122 UNINSTALL_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/uninstall-test" 123 INSTALL_CONFIG_FILE = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-oke-backup-enabled.yaml" 124 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') 125 DATABASE_PSW = credentials('todo-mysql-password') 126 127 // used to emit metrics 128 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 129 TEST_ENV_LABEL = "magicdns_oke" 130 TEST_ENV = "OKE" 131 132 // used to generate Ginkgo test reports 133 TEST_REPORT = "test-report.xml" 134 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 135 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 136 137 // Backup parameters 138 VELERO_NAMESPACE = "verrazzano-backup" 139 VELERO_SECRET_NAME = "velero-opensearch-backup-creds" 140 VELERO_MYSQL_SECRET_NAME = "velero-mysql-backup-creds" 141 RANCHER_SECRET_NAME = "rancher-backup-creds" 142 OCI_OS_ACCESS_KEY = credentials('velero-key-id') 143 OCI_OS_ACCESS_SECRET_KEY = credentials('velero-key') 144 OCI_OS_COMPARTMENT_ID = credentials('v8o-velero-test') 145 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 146 OCI_OS_BUCKET_ID = "${ociOsBucketName}" 147 OCI_OS_BUCKET_NAME = "bucket-${OCI_OS_BUCKET_ID}" 148 BACKUP_OPENSEARCH = "backup-opensearch-${OCI_OS_BUCKET_ID}" 149 BACKUP_RANCHER = "backup-rancher-${OCI_OS_BUCKET_ID}" 150 BACKUP_RESOURCE = "vz-hook-resource-${OCI_OS_BUCKET_ID}" 151 RESTORE_OPENSEARCH = "restore-opensearch-${OCI_OS_BUCKET_ID}" 152 RESTORE_RANCHER = "restore-rancher-${OCI_OS_BUCKET_ID}" 153 BACKUP_MYSQL = "backup-mysql-${OCI_OS_BUCKET_ID}" 154 RESTORE_MYSQL = "restore-mysql-${OCI_OS_BUCKET_ID}" 155 BACKUP_OPENSEARCH_STORAGE = "storage-opensearch-${OCI_OS_BUCKET_ID}" 156 BACKUP_MYSQL_STORAGE = "storage-mysql-${OCI_OS_BUCKET_ID}" 157 BACKUP_REGION = "us-phoenix-1" 158 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 159 160 // Environment variable for Verrazzano CLI executable 161 VZ_COMMAND="${GO_REPO_PATH}/vz" 162 } 163 164 stages { 165 stage('Clean workspace and checkout') { 166 steps { 167 sh """ 168 echo "${NODE_LABELS}" 169 """ 170 171 script { 172 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 173 if (params.GIT_COMMIT_TO_USE == "NONE") { 174 echo "Specific GIT commit was not specified, use current head" 175 def scmInfo = checkout scm 176 env.GIT_COMMIT = scmInfo.GIT_COMMIT 177 env.GIT_BRANCH = scmInfo.GIT_BRANCH 178 } else { 179 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 180 def scmInfo = checkout([ 181 $class: 'GitSCM', 182 branches: [[name: params.GIT_COMMIT_TO_USE]], 183 doGenerateSubmoduleConfigurations: false, 184 extensions: [], 185 submoduleCfg: [], 186 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 187 env.GIT_COMMIT = scmInfo.GIT_COMMIT 188 env.GIT_BRANCH = scmInfo.GIT_BRANCH 189 // If the commit we were handed is not what the SCM says we are using, fail 190 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 191 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 192 exit 1 193 } 194 } 195 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 196 } 197 198 sh """ 199 cp -f "${NETRC_FILE}" $HOME/.netrc 200 chmod 600 $HOME/.netrc 201 """ 202 203 script { 204 try { 205 sh """ 206 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 207 """ 208 } catch(error) { 209 echo "docker login failed, retrying after sleep" 210 retry(4) { 211 sleep(30) 212 sh """ 213 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 214 """ 215 } 216 } 217 } 218 sh """ 219 rm -rf ${GO_REPO_PATH}/verrazzano 220 mkdir -p ${GO_REPO_PATH}/verrazzano 221 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 222 """ 223 224 script { 225 def props = readProperties file: '.verrazzano-development-version' 226 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 227 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 228 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 229 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 230 // update the description with some meaningful info 231 setDisplayName() 232 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 233 234 // derive the prefix for the OKE cluster 235 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 236 } 237 script { 238 sh """ 239 echo "Downloading VZ CLI from object storage" 240 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 241 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 242 ${GO_REPO_PATH}/vz version 243 """ 244 } 245 } 246 } 247 248 stage('Create bucket') { 249 steps { 250 createBucket("${OCI_OS_BUCKET_NAME}") 251 } 252 } 253 254 255 stage("Create OKE Cluster") { 256 steps { 257 sh "TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=uninstall-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/create_oke_cluster.sh" 258 } 259 } 260 261 stage('Install Verrazzano') { 262 environment { 263 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 264 } 265 steps { 266 script { 267 listNamepacesAndPods('before installing Verrazzano') 268 } 269 sh """ 270 # Create image pull secret for Verrazzano docker images 271 cd ${GO_REPO_PATH}/verrazzano 272 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 273 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 274 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 275 276 echo "Install Platform Operator" 277 cd ${GO_REPO_PATH}/verrazzano 278 if [ "NONE" = "${VERRAZZANO_OPERATOR_IMAGE}" ]; then 279 echo "Using operator.yaml from object storage" 280 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 281 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 282 else 283 echo "Generating operator.yaml based on image name provided: ${VERRAZZANO_OPERATOR_IMAGE}" 284 env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/acceptance-test-operator.yaml 285 fi 286 kubectl apply -f ${WORKSPACE}/acceptance-test-operator.yaml 287 288 # make sure ns exists 289 ./tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 290 291 # create secret in verrazzano-install ns 292 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 293 ./tests/e2e/config/scripts/process_nipio_install_yaml.sh ${INSTALL_CONFIG_FILE} 294 295 echo "Waiting for Operator to be ready" 296 cd ${GO_REPO_PATH}/verrazzano 297 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 298 299 echo "Installing Verrazzano on OKE" 300 kubectl apply -f ${INSTALL_CONFIG_FILE} 301 302 # wait for Verrazzano install to complete 303 ./tests/e2e/config/scripts/wait-for-verrazzano-install.sh 304 """ 305 } 306 post { 307 308 always { 309 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 310 } 311 success { 312 script { 313 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 314 dumpK8sCluster('verrazzano-install-cluster-dump') 315 } 316 } 317 } 318 failure { 319 script { 320 dumpK8sCluster('verrazzano-install-failure-cluster-dump') 321 sh """ 322 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs 323 ${LOOPING_TEST_SCRIPTS_DIR}/dump_resources.sh > ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs/resources.log 324 """ 325 } 326 } 327 } 328 } 329 330 331 stage('verify-install') { 332 steps { 333 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 334 sh """ 335 cd ${WORKSPACE}/tests/e2e 336 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/... 337 """ 338 } 339 } 340 post { 341 failure { 342 script { 343 dumpK8sCluster('verify-install-cluster-dump') 344 } 345 } 346 } 347 } 348 349 stage('opensearch backup and restore') { 350 when { 351 expression {params.OPENSEARCH_BACKUP == true} 352 } 353 environment { 354 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/opensearch" 355 } 356 steps { 357 println("Checking all pods in namespace verrazzano-logging are ready before start of test") 358 waitForPodsToBeUp("verrazzano-logging") 359 runGinkgoRandomize('backup/opensearch') 360 println("Checking all pods in namespace verrazzano-logging are ready after test completion") 361 waitForPodsToBeUp("verrazzano-logging") 362 } 363 post { 364 always { 365 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 366 junit testResults: '**/*test-result.xml', allowEmptyResults: true 367 } 368 failure { 369 script { 370 dumpK8sCluster('opensearch-backup-restore-cluster-dump') 371 } 372 } 373 } 374 } 375 376 stage('rancher backup and restore') { 377 when { 378 expression {params.RANCHER_BACKUP == true} 379 } 380 environment { 381 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/rancher" 382 } 383 steps { 384 runGinkgoRandomize('backup/rancher') 385 } 386 post { 387 always { 388 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 389 junit testResults: '**/*test-result.xml', allowEmptyResults: true 390 } 391 failure { 392 script { 393 dumpK8sCluster('rancher-backup-restore-cluster-dump') 394 } 395 } 396 } 397 } 398 399 stage('mysql operator backup and restore with OCI user principal') { 400 when { 401 allOf { 402 expression {params.MYSQL_BACKUP == true} 403 expression {params.MYSQL_BACKUP_S3 != true} 404 } 405 } 406 environment { 407 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/mysql" 408 } 409 steps { 410 println("MySQL backup with OCI user principal...") 411 echoAllPodInfo() 412 println("Checking all pods in namespace keycloak are ready before start of test") 413 waitForPodsToBeUp("keycloak") 414 runGinkgoRandomize('backup/mysql') 415 println("Checking all pods in namespace keycloak are ready after completion of test") 416 waitForPodsToBeUp("keycloak") 417 } 418 post { 419 always { 420 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 421 junit testResults: '**/*test-result.xml', allowEmptyResults: true 422 } 423 failure { 424 script { 425 dumpK8sCluster('mysql-backup-restore-cluster-dump') 426 } 427 } 428 } 429 } 430 431 stage('mysql operator backup and restore with S3') { 432 when { 433 allOf { 434 expression {params.MYSQL_BACKUP == true} 435 expression {params.MYSQL_BACKUP_S3 == true} 436 } 437 } 438 environment { 439 MYSQL_BACKUP_MODE = "s3" 440 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/mysql" 441 } 442 steps { 443 println("MySQL backup with S3 credentials ...") 444 echoAllPodInfo() 445 println("Checking all pods in namespace keycloak are ready before start of test") 446 waitForPodsToBeUp("keycloak") 447 runGinkgoRandomize('backup/mysql') 448 println("Checking all pods in namespace keycloak are ready after completion of test") 449 waitForPodsToBeUp("keycloak") 450 } 451 post { 452 always { 453 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 454 junit testResults: '**/*test-result.xml', allowEmptyResults: true 455 } 456 failure { 457 script { 458 dumpK8sCluster('mysql-backup-restore-cluster-dump') 459 } 460 } 461 } 462 } 463 464 } 465 post { 466 always { 467 sh """ 468 # Copy the generated test reports to WORKSPACE to archive them 469 mkdir -p ${TEST_REPORT_DIR} 470 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 471 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 472 """ 473 archiveArtifacts artifacts: "**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/${TEST_REPORT}", allowEmptyArchive: true 474 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 475 script { 476 if (params.DELETE_BUCKET) { 477 println("Delete object store bucket.") 478 deleteBucket("${OCI_OS_BUCKET_NAME}") 479 } else { 480 println("Object store bucket not deleted!") 481 } 482 } 483 } 484 failure { 485 script { 486 archiveArtifacts artifacts: '**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**', allowEmptyArchive: true 487 } 488 } 489 cleanup { 490 sh "VERRAZZANO_KUBECONFIG=${env.KUBECONFIG} TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=uninstall-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/delete_oke_cluster.sh || true" 491 deleteDir() 492 } 493 } 494 } 495 496 def echoAllPodInfo() { 497 sh """ 498 kubectl get pod -A 499 """ 500 } 501 502 def createBucket(bucketName) { 503 sh """ 504 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os bucket create -c ${OCI_OS_COMPARTMENT_ID} --namespace ${OCI_OS_NAMESPACE} --name ${bucketName} 505 """ 506 } 507 508 def deleteBucket(bucketName) { 509 sh """ 510 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os bucket delete -bn ${bucketName} --empty --force 511 """ 512 } 513 514 515 def waitForPodsToBeUp(namespace) { 516 sh """ 517 kubectl wait --selector='!job-name' -n ${namespace} --for=condition=ready pod --all --timeout 10m 518 """ 519 } 520 521 def runGinkgoRandomize(testSuitePath, kubeConfig = '') { 522 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 523 sh """ 524 if [ ! -z "${kubeConfig}" ]; then 525 export KUBECONFIG="${kubeConfig}" 526 fi 527 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 528 if [ -d "${testSuitePath}" ]; then 529 ginkgo -vv --progress --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 530 fi 531 """ 532 } 533 } 534 535 def dumpK8sCluster(dumpDirectory) { 536 sh """ 537 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 538 """ 539 } 540 541 def dumpVerrazzanoSystemPods(logDirectory) { 542 sh """ 543 cd ${GO_REPO_PATH}/verrazzano/platform-operator 544 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-pods.log" 545 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 546 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-certs.log" 547 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 548 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-osd.log" 549 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 550 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-es-master.log" 551 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 552 """ 553 } 554 555 def dumpCattleSystemPods(logDirectory) { 556 sh """ 557 cd ${GO_REPO_PATH}/verrazzano/platform-operator 558 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/cattle-system-pods.log" 559 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 560 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/rancher.log" 561 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 562 """ 563 } 564 565 def dumpNginxIngressControllerLogs(logDirectory) { 566 sh """ 567 cd ${GO_REPO_PATH}/verrazzano/platform-operator 568 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/nginx-ingress-controller.log" 569 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 570 """ 571 } 572 573 def dumpVerrazzanoPlatformOperatorLogs(logDirectory) { 574 sh """ 575 ## dump out verrazzano-platform-operator logs 576 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory} 577 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory}/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 578 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory}/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 579 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 580 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 581 echo "------------------------------------------" 582 """ 583 } 584 585 def dumpVerrazzanoApplicationOperatorLogs(logDirectory) { 586 sh """ 587 ## dump out verrazzano-application-operator logs 588 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory} 589 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 590 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 591 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 592 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 593 echo "------------------------------------------" 594 """ 595 } 596 597 def dumpOamKubernetesRuntimeLogs(logDirectory) { 598 sh """ 599 ## dump out oam-kubernetes-runtime logs 600 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs/${logDirectory} 601 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/${logDirectory}/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 602 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 603 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 604 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 605 echo "------------------------------------------" 606 """ 607 } 608 609 def dumpVerrazzanoApiLogs(logDirectory) { 610 sh """ 611 cd ${GO_REPO_PATH}/verrazzano/platform-operator 612 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-authproxy.log" 613 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 614 """ 615 } 616 617 def listNamepacesAndPods(customMessage) { 618 sh """ 619 echo "Listing all the namespaces and pods the namespaces ${customMessage}." 620 kubectl get namespaces 621 kubectl get pods -A 622 echo "-----------------------------------------------------" 623 """ 624 } 625 626 def listHelmReleases(customMessage) { 627 sh """ 628 echo "Listing the releases across all namespaces ${customMessage}." 629 helm list -A 630 echo "-----------------------------------------------------" 631 """ 632 } 633 634 def getEffectiveDumpOnSuccess() { 635 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 636 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 637 effectiveValue = true 638 echo "Forcing dump on success based on global override setting" 639 } 640 return effectiveValue 641 } 642 643 def setDisplayName() { 644 echo "Start setDisplayName" 645 def causes = currentBuild.getBuildCauses() 646 echo "causes: " + causes.toString() 647 for (cause in causes) { 648 def causeString = cause.toString() 649 echo "current cause: " + causeString 650 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 651 echo "This job was caused by " + causeString 652 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 653 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 654 } else if (causeString.contains("verrazzano-flaky-tests")) { 655 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 656 } 657 } 658 } 659 echo "End setDisplayName" 660 }