github.com/verrazzano/verrazzano@v1.7.0/ci/backup/JenkinsfileAlllBackupOKETest (about) 1 // Copyright (c) 2022, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 6 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 7 // pulling "ap-*" from the test regions given discovery of image pull issues 8 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-frankfurt-1", "eu-zurich-1", "uk-london-1" ] 9 def OKE_CLUSTER_PREFIX = "" 10 def ociOsBucketName = UUID.randomUUID().toString().substring(0,6).replace('-','') 11 def backup_id = 'dummy' 12 Collections.shuffle(availableRegions) 13 14 pipeline { 15 options { 16 skipDefaultCheckout true 17 } 18 19 agent { 20 docker { 21 image "${RUNNER_DOCKER_IMAGE}" 22 args "${RUNNER_DOCKER_ARGS}" 23 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 24 registryCredentialsId 'ocir-pull-and-push-account' 25 label "$agentLabel" 26 } 27 } 28 29 parameters { 30 string (name: 'VERRAZZANO_BRANCH', 31 defaultValue: 'master', 32 description: 'Branch to clone and checkout the Verrazzano repo', 33 trim: true) 34 choice (name: 'OKE_NODE_POOL', 35 description: 'OKE node pool configuration', 36 // 1st choice is the default value 37 choices: [ "VM.Standard.E2.4","VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2",]) 38 choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION', 39 // 1st choice is the default value 40 choices: availableRegions ) 41 string (name: 'GIT_COMMIT_TO_USE', 42 defaultValue: 'NONE', 43 description: 'This is the full git commit hash from the source build to be used for all jobs', 44 trim: true) 45 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 46 defaultValue: 'NONE', 47 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator', 48 trim: true) 49 choice (name: 'OKE_CLUSTER_VERSION', 50 description: 'Kubernetes Version for OKE Cluster', 51 // 1st choice is the default value 52 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 53 choice (name: 'CRD_API_VERSION', 54 description: 'This is the API crd version.', 55 // 1st choice is the default value 56 choices: [ "v1beta1", "v1alpha1"]) 57 booleanParam (name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', 58 defaultValue: false, 59 description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)') 60 string (name: 'TAGGED_TESTS', 61 defaultValue: '', 62 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 63 trim: true) 64 string (name: 'INCLUDED_TESTS', 65 defaultValue: '.*', 66 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 67 trim: true) 68 string (name: 'EXCLUDED_TESTS', 69 defaultValue: '_excluded_test', 70 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 71 trim: true) 72 booleanParam (description: 'Whether to persist object store bucket', name: 'DELETE_BUCKET', defaultValue: true) 73 booleanParam (description: 'Whether to execute MySQL backup tests', name: 'MYSQL_BACKUP', defaultValue: true) 74 booleanParam (description: 'Whether to execute MySQL backup tests with S3 API. Default is true. If unchecked user principal creds will be used', name: 'MYSQL_BACKUP_S3', defaultValue: true) 75 booleanParam (description: 'Whether to execute OpenSearch backup tests', name: 'OPENSEARCH_BACKUP', defaultValue: true) 76 booleanParam (description: 'Whether to execute Rancher backup tests', name: 'RANCHER_BACKUP', defaultValue: true) 77 } 78 79 environment { 80 OCR_CREDS = credentials('ocr-pull-and-push-account') 81 NETRC_FILE = credentials('netrc') 82 GOPATH = '/home/opc/go' 83 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 84 DOCKER_CREDS = credentials('github-packages-credentials-rw') 85 DOCKER_EMAIL = credentials('github-packages-email') 86 DOCKER_REPO = 'ghcr.io' 87 88 OCR_REPO = 'container-registry.oracle.com' 89 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 90 91 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 92 TF_VAR_tenancy_id = credentials('oci-tenancy') 93 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 94 TF_VAR_user_id = credentials('oci-user-ocid') 95 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 96 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 97 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 98 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 99 TF_VAR_api_private_key_path = credentials('oci-api-key') 100 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 101 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 102 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 103 104 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 105 OCI_CLI_TENANCY = credentials('oci-tenancy') 106 OCI_CLI_USER = credentials('oci-user-ocid') 107 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 108 OCI_CLI_KEY_FILE = credentials('oci-api-key') 109 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 110 DISABLE_SPINNER=1 111 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 112 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 113 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 114 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 115 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 116 KUBECONFIG = "${WORKSPACE}/oke_kubeconfig" 117 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 118 INSTALL_PROFILE = "prod" 119 VZ_ENVIRONMENT_NAME = "default" 120 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 121 LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test" 122 UNINSTALL_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/uninstall-test" 123 INSTALL_CONFIG_FILE = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-oke-backup-enabled.yaml" 124 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') 125 DATABASE_PSW = credentials('todo-mysql-password') 126 127 // used to emit metrics 128 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 129 TEST_ENV_LABEL = "magicdns_oke" 130 TEST_ENV = "OKE" 131 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 132 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 133 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 134 135 // used to generate Ginkgo test reports 136 TEST_REPORT = "test-report.xml" 137 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 138 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 139 140 // Backup parameters 141 VELERO_NAMESPACE = "verrazzano-backup" 142 VELERO_SECRET_NAME = "velero-opensearch-backup-creds" 143 VELERO_MYSQL_SECRET_NAME = "velero-mysql-backup-creds" 144 RANCHER_SECRET_NAME = "rancher-backup-creds" 145 OCI_OS_ACCESS_KEY = credentials('velero-key-id') 146 OCI_OS_ACCESS_SECRET_KEY = credentials('velero-key') 147 OCI_OS_COMPARTMENT_ID = credentials('v8o-velero-test') 148 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 149 OCI_OS_BUCKET_ID = "${ociOsBucketName}" 150 OCI_OS_BUCKET_NAME = "bucket-${OCI_OS_BUCKET_ID}" 151 BACKUP_OPENSEARCH = "backup-opensearch-${OCI_OS_BUCKET_ID}" 152 BACKUP_RANCHER = "backup-rancher-${OCI_OS_BUCKET_ID}" 153 BACKUP_RESOURCE = "vz-hook-resource-${OCI_OS_BUCKET_ID}" 154 RESTORE_OPENSEARCH = "restore-opensearch-${OCI_OS_BUCKET_ID}" 155 RESTORE_RANCHER = "restore-rancher-${OCI_OS_BUCKET_ID}" 156 BACKUP_MYSQL = "backup-mysql-${OCI_OS_BUCKET_ID}" 157 RESTORE_MYSQL = "restore-mysql-${OCI_OS_BUCKET_ID}" 158 BACKUP_OPENSEARCH_STORAGE = "storage-opensearch-${OCI_OS_BUCKET_ID}" 159 BACKUP_MYSQL_STORAGE = "storage-mysql-${OCI_OS_BUCKET_ID}" 160 BACKUP_REGION = "us-phoenix-1" 161 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 162 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 163 164 // Environment variable for Verrazzano CLI executable 165 VZ_COMMAND="${GO_REPO_PATH}/vz" 166 } 167 168 stages { 169 stage('Clean workspace and checkout') { 170 steps { 171 sh """ 172 echo "${NODE_LABELS}" 173 """ 174 175 script { 176 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 177 if (params.GIT_COMMIT_TO_USE == "NONE") { 178 echo "Specific GIT commit was not specified, use current head" 179 def scmInfo = checkout scm 180 env.GIT_COMMIT = scmInfo.GIT_COMMIT 181 env.GIT_BRANCH = scmInfo.GIT_BRANCH 182 } else { 183 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 184 def scmInfo = checkout([ 185 $class: 'GitSCM', 186 branches: [[name: params.GIT_COMMIT_TO_USE]], 187 doGenerateSubmoduleConfigurations: false, 188 extensions: [], 189 submoduleCfg: [], 190 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 191 env.GIT_COMMIT = scmInfo.GIT_COMMIT 192 env.GIT_BRANCH = scmInfo.GIT_BRANCH 193 // If the commit we were handed is not what the SCM says we are using, fail 194 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 195 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 196 exit 1 197 } 198 } 199 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 200 } 201 202 sh """ 203 cp -f "${NETRC_FILE}" $HOME/.netrc 204 chmod 600 $HOME/.netrc 205 """ 206 207 script { 208 try { 209 sh """ 210 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 211 """ 212 } catch(error) { 213 echo "docker login failed, retrying after sleep" 214 retry(4) { 215 sleep(30) 216 sh """ 217 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 218 """ 219 } 220 } 221 } 222 sh """ 223 rm -rf ${GO_REPO_PATH}/verrazzano 224 mkdir -p ${GO_REPO_PATH}/verrazzano 225 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 226 """ 227 228 script { 229 def props = readProperties file: '.verrazzano-development-version' 230 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 231 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 232 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 233 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 234 // update the description with some meaningful info 235 setDisplayName() 236 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 237 238 // derive the prefix for the OKE cluster 239 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 240 } 241 script { 242 sh """ 243 echo "Downloading VZ CLI from object storage" 244 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 245 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 246 ${GO_REPO_PATH}/vz version 247 """ 248 } 249 } 250 } 251 252 stage('Create bucket') { 253 steps { 254 createBucket("${OCI_OS_BUCKET_NAME}") 255 } 256 } 257 258 259 stage("Create OKE Cluster") { 260 steps { 261 sh "TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=uninstall-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/create_oke_cluster.sh" 262 } 263 } 264 265 stage('Install Verrazzano') { 266 environment { 267 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 268 } 269 steps { 270 script { 271 listNamepacesAndPods('before installing Verrazzano') 272 } 273 sh """ 274 # Create image pull secret for Verrazzano docker images 275 cd ${GO_REPO_PATH}/verrazzano 276 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 277 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 278 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 279 280 echo "Install Platform Operator" 281 cd ${GO_REPO_PATH}/verrazzano 282 if [ "NONE" = "${VERRAZZANO_OPERATOR_IMAGE}" ]; then 283 echo "Using operator.yaml from object storage" 284 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 285 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 286 else 287 echo "Generating operator.yaml based on image name provided: ${VERRAZZANO_OPERATOR_IMAGE}" 288 env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/acceptance-test-operator.yaml 289 fi 290 kubectl apply -f ${WORKSPACE}/acceptance-test-operator.yaml 291 292 # make sure ns exists 293 ./tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 294 295 # create secret in verrazzano-install ns 296 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 297 ./tests/e2e/config/scripts/process_nipio_install_yaml.sh ${INSTALL_CONFIG_FILE} 298 299 echo "Waiting for Operator to be ready" 300 cd ${GO_REPO_PATH}/verrazzano 301 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 302 303 echo "Installing Verrazzano on OKE" 304 kubectl apply -f ${INSTALL_CONFIG_FILE} 305 306 # wait for Verrazzano install to complete 307 ./tests/e2e/config/scripts/wait-for-verrazzano-install.sh 308 """ 309 } 310 post { 311 312 always { 313 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 314 } 315 success { 316 script { 317 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 318 dumpK8sCluster('verrazzano-install-cluster-dump') 319 } 320 } 321 } 322 failure { 323 script { 324 dumpK8sCluster('verrazzano-install-failure-cluster-dump') 325 sh """ 326 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs 327 ${LOOPING_TEST_SCRIPTS_DIR}/dump_resources.sh > ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs/resources.log 328 """ 329 } 330 } 331 } 332 } 333 334 335 stage('verify-install') { 336 steps { 337 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 338 sh """ 339 cd ${WORKSPACE}/tests/e2e 340 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/... 341 """ 342 } 343 } 344 post { 345 failure { 346 script { 347 dumpK8sCluster('verify-install-cluster-dump') 348 } 349 } 350 } 351 } 352 353 stage('opensearch backup and restore') { 354 when { 355 expression {params.OPENSEARCH_BACKUP == true} 356 } 357 environment { 358 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/opensearch" 359 } 360 steps { 361 println("Checking all pods in namespace verrazzano-logging are ready before start of test") 362 waitForPodsToBeUp("verrazzano-logging") 363 runGinkgoRandomize('backup/opensearch') 364 println("Checking all pods in namespace verrazzano-logging are ready after test completion") 365 waitForPodsToBeUp("verrazzano-logging") 366 } 367 post { 368 always { 369 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 370 junit testResults: '**/*test-result.xml', allowEmptyResults: true 371 } 372 failure { 373 script { 374 dumpK8sCluster('opensearch-backup-restore-cluster-dump') 375 } 376 } 377 } 378 } 379 380 stage('rancher backup and restore') { 381 when { 382 expression {params.RANCHER_BACKUP == true} 383 } 384 environment { 385 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/rancher" 386 } 387 steps { 388 runGinkgoRandomize('backup/rancher') 389 } 390 post { 391 always { 392 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 393 junit testResults: '**/*test-result.xml', allowEmptyResults: true 394 } 395 failure { 396 script { 397 dumpK8sCluster('rancher-backup-restore-cluster-dump') 398 } 399 } 400 } 401 } 402 403 stage('mysql operator backup and restore with OCI user principal') { 404 when { 405 allOf { 406 expression {params.MYSQL_BACKUP == true} 407 expression {params.MYSQL_BACKUP_S3 != true} 408 } 409 } 410 environment { 411 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/mysql" 412 } 413 steps { 414 println("MySQL backup with OCI user principal...") 415 echoAllPodInfo() 416 println("Checking all pods in namespace keycloak are ready before start of test") 417 waitForPodsToBeUp("keycloak") 418 runGinkgoRandomize('backup/mysql') 419 println("Checking all pods in namespace keycloak are ready after completion of test") 420 waitForPodsToBeUp("keycloak") 421 } 422 post { 423 always { 424 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 425 junit testResults: '**/*test-result.xml', allowEmptyResults: true 426 } 427 failure { 428 script { 429 dumpK8sCluster('mysql-backup-restore-cluster-dump') 430 } 431 } 432 } 433 } 434 435 stage('mysql operator backup and restore with S3') { 436 when { 437 allOf { 438 expression {params.MYSQL_BACKUP == true} 439 expression {params.MYSQL_BACKUP_S3 == true} 440 } 441 } 442 environment { 443 MYSQL_BACKUP_MODE = "s3" 444 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/mysql" 445 } 446 steps { 447 println("MySQL backup with S3 credentials ...") 448 echoAllPodInfo() 449 println("Checking all pods in namespace keycloak are ready before start of test") 450 waitForPodsToBeUp("keycloak") 451 runGinkgoRandomize('backup/mysql') 452 println("Checking all pods in namespace keycloak are ready after completion of test") 453 waitForPodsToBeUp("keycloak") 454 } 455 post { 456 always { 457 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 458 junit testResults: '**/*test-result.xml', allowEmptyResults: true 459 } 460 failure { 461 script { 462 dumpK8sCluster('mysql-backup-restore-cluster-dump') 463 } 464 } 465 } 466 } 467 468 } 469 post { 470 always { 471 sh """ 472 # Copy the generated test reports to WORKSPACE to archive them 473 mkdir -p ${TEST_REPORT_DIR} 474 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 475 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 476 """ 477 archiveArtifacts artifacts: "**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/${TEST_REPORT}", allowEmptyArchive: true 478 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 479 script { 480 if (params.DELETE_BUCKET) { 481 println("Delete object store bucket.") 482 deleteBucket("${OCI_OS_BUCKET_NAME}") 483 } else { 484 println("Object store bucket not deleted!") 485 } 486 } 487 } 488 failure { 489 script { 490 archiveArtifacts artifacts: '**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**', allowEmptyArchive: true 491 } 492 } 493 cleanup { 494 sh "VERRAZZANO_KUBECONFIG=${env.KUBECONFIG} TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=uninstall-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/delete_oke_cluster.sh || true" 495 deleteDir() 496 } 497 } 498 } 499 500 def echoAllPodInfo() { 501 sh """ 502 kubectl get pod -A 503 """ 504 } 505 506 def createBucket(bucketName) { 507 sh """ 508 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os bucket create -c ${OCI_OS_COMPARTMENT_ID} --namespace ${OCI_OS_NAMESPACE} --name ${bucketName} 509 """ 510 } 511 512 def deleteBucket(bucketName) { 513 sh """ 514 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os bucket delete -bn ${bucketName} --empty --force 515 """ 516 } 517 518 519 def waitForPodsToBeUp(namespace) { 520 sh """ 521 kubectl wait --selector='!job-name' -n ${namespace} --for=condition=ready pod --all --timeout 10m 522 """ 523 } 524 525 def runGinkgoRandomize(testSuitePath, kubeConfig = '') { 526 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 527 sh """ 528 if [ ! -z "${kubeConfig}" ]; then 529 export KUBECONFIG="${kubeConfig}" 530 fi 531 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 532 if [ -d "${testSuitePath}" ]; then 533 ginkgo -vv --progress --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 534 fi 535 """ 536 } 537 } 538 539 def dumpK8sCluster(dumpDirectory) { 540 sh """ 541 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 542 """ 543 } 544 545 def dumpVerrazzanoSystemPods(logDirectory) { 546 sh """ 547 cd ${GO_REPO_PATH}/verrazzano/platform-operator 548 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-pods.log" 549 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 550 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-certs.log" 551 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 552 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-osd.log" 553 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 554 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-es-master.log" 555 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 556 """ 557 } 558 559 def dumpCattleSystemPods(logDirectory) { 560 sh """ 561 cd ${GO_REPO_PATH}/verrazzano/platform-operator 562 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/cattle-system-pods.log" 563 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 564 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/rancher.log" 565 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 566 """ 567 } 568 569 def dumpNginxIngressControllerLogs(logDirectory) { 570 sh """ 571 cd ${GO_REPO_PATH}/verrazzano/platform-operator 572 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/nginx-ingress-controller.log" 573 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 574 """ 575 } 576 577 def dumpVerrazzanoPlatformOperatorLogs(logDirectory) { 578 sh """ 579 ## dump out verrazzano-platform-operator logs 580 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory} 581 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory}/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 582 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory}/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 583 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 584 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 585 echo "------------------------------------------" 586 """ 587 } 588 589 def dumpVerrazzanoApplicationOperatorLogs(logDirectory) { 590 sh """ 591 ## dump out verrazzano-application-operator logs 592 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory} 593 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 594 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 595 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 596 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 597 echo "------------------------------------------" 598 """ 599 } 600 601 def dumpOamKubernetesRuntimeLogs(logDirectory) { 602 sh """ 603 ## dump out oam-kubernetes-runtime logs 604 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs/${logDirectory} 605 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/${logDirectory}/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 606 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 607 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 608 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 609 echo "------------------------------------------" 610 """ 611 } 612 613 def dumpVerrazzanoApiLogs(logDirectory) { 614 sh """ 615 cd ${GO_REPO_PATH}/verrazzano/platform-operator 616 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-authproxy.log" 617 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 618 """ 619 } 620 621 def listNamepacesAndPods(customMessage) { 622 sh """ 623 echo "Listing all the namespaces and pods the namespaces ${customMessage}." 624 kubectl get namespaces 625 kubectl get pods -A 626 echo "-----------------------------------------------------" 627 """ 628 } 629 630 def listHelmReleases(customMessage) { 631 sh """ 632 echo "Listing the releases across all namespaces ${customMessage}." 633 helm list -A 634 echo "-----------------------------------------------------" 635 """ 636 } 637 638 def getEffectiveDumpOnSuccess() { 639 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 640 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 641 effectiveValue = true 642 echo "Forcing dump on success based on global override setting" 643 } 644 return effectiveValue 645 } 646 647 def setDisplayName() { 648 echo "Start setDisplayName" 649 def causes = currentBuild.getBuildCauses() 650 echo "causes: " + causes.toString() 651 for (cause in causes) { 652 def causeString = cause.toString() 653 echo "current cause: " + causeString 654 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 655 echo "This job was caused by " + causeString 656 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 657 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 658 } else if (causeString.contains("verrazzano-flaky-tests")) { 659 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 660 } 661 } 662 } 663 echo "End setDisplayName" 664 }