github.com/verrazzano/verrazzano@v1.7.1/ci/backup/JenkinsfileAlllBackupKinDTest (about) 1 // Copyright (c) 2022, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 // def agentLabel = "largeexperimental" 7 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 8 def ociOsBucketName = UUID.randomUUID().toString().substring(0,6).replace('-','') 9 10 pipeline { 11 options { 12 timeout(time: 1, unit: 'HOURS') 13 skipDefaultCheckout true 14 timestamps () 15 } 16 17 agent { 18 docker { 19 image "${RUNNER_DOCKER_IMAGE}" 20 args "${RUNNER_DOCKER_ARGS}" 21 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 22 registryCredentialsId 'ocir-pull-and-push-account' 23 label "${agentLabel}" 24 } 25 } 26 27 parameters { 28 choice (name: 'KUBERNETES_CLUSTER_VERSION', 29 description: 'Kubernetes Version for KinD Cluster', 30 // 1st choice is the default value 31 choices: [ "1.27", "1.26", "1.25", "1.24" ]) 32 string (name: 'GIT_COMMIT_TO_USE', 33 defaultValue: 'NONE', 34 description: 'This is the full git commit hash from the source build to be used for all jobs', 35 trim: true) 36 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 37 defaultValue: 'NONE', 38 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator', 39 trim: true) 40 choice (name: 'WILDCARD_DNS_DOMAIN', 41 description: 'This is the wildcard DNS domain', 42 // 1st choice is the default value 43 choices: [ "nip.io", "sslip.io"]) 44 choice (name: 'CRD_API_VERSION', 45 description: 'This is the API crd version.', 46 // 1st choice is the default value 47 choices: [ "v1beta1", "v1alpha1"]) 48 booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 49 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 50 booleanParam (description: 'Whether to use a database for Grafana persistence', name: 'USE_DB_FOR_GRAFANA', defaultValue: false) 51 string (name: 'CONSOLE_REPO_BRANCH', 52 defaultValue: '', 53 description: 'The branch to check out after cloning the console repository.', 54 trim: true) 55 booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true) 56 string (name: 'TAGGED_TESTS', 57 defaultValue: '', 58 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 59 trim: true) 60 string (name: 'INCLUDED_TESTS', 61 defaultValue: '.*', 62 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 63 trim: true) 64 string (name: 'EXCLUDED_TESTS', 65 defaultValue: '_excluded_test', 66 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 67 trim: true) 68 booleanParam (description: 'Whether to run JWT tests', name: 'ENABLE_JWT_TESTING', defaultValue: false) 69 booleanParam (description: 'Whether to run slow tests', name: 'RUN_SLOW_TESTS', defaultValue: false) 70 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 71 booleanParam (description: 'Whether to persist object store bucket', name: 'DELETE_BUCKET', defaultValue: true) 72 booleanParam (description: 'Whether to execute MySQL backup tests', name: 'MYSQL_BACKUP', defaultValue: true) 73 booleanParam (description: 'Whether to execute MySQL backup tests with S3 API. Default is true. If unchecked user principal creds will be used', name: 'MYSQL_BACKUP_S3', defaultValue: true) 74 booleanParam (description: 'Whether to execute OpenSearch backup tests', name: 'OPENSEARCH_BACKUP', defaultValue: true) 75 booleanParam (description: 'Whether to execute Rancher backup tests', name: 'RANCHER_BACKUP', defaultValue: true) 76 } 77 78 environment { 79 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 80 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 81 GOPATH = '/home/opc/go' 82 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 83 DOCKER_CREDS = credentials('github-packages-credentials-rw') 84 DOCKER_EMAIL = credentials('github-packages-email') 85 DOCKER_REPO = 'ghcr.io' 86 DOCKER_NAMESPACE = 'verrazzano' 87 NETRC_FILE = credentials('netrc') 88 CLUSTER_NAME = 'verrazzano' 89 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 90 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 91 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 92 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 93 OCR_CREDS = credentials('ocr-pull-and-push-account') 94 OCR_REPO = 'container-registry.oracle.com' 95 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 96 INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-backup-enabled.yaml" 97 INSTALL_PROFILE = "prod" 98 KIND_NODE_COUNT = "3" 99 VZ_ENVIRONMENT_NAME = "default" 100 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 101 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 102 103 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test 104 DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test 105 106 // Environment variables required to capture cluster snapshot and bug report on test failure 107 DUMP_KUBECONFIG="${KUBECONFIG}" 108 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 109 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 110 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 111 112 // Environment variable for Verrazzano CLI executable 113 VZ_COMMAND="${GO_REPO_PATH}/vz" 114 115 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 116 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 117 118 // used for console artifact capture on failure 119 JENKINS_READ = credentials('jenkins-auditor') 120 // OCI_CLI_AUTH="instance_principal" 121 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 122 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 123 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 124 125 // used to emit metrics 126 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 127 TEST_ENV_LABEL = "kind" 128 TEST_ENV = "kind_oci_dns" 129 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 130 131 // used to generate Ginkgo test reports 132 TEST_REPORT = "test-report.xml" 133 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 134 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 135 136 // Backup parameters 137 VELERO_NAMESPACE = "verrazzano-backup" 138 VELERO_SECRET_NAME = "velero-opensearch-backup-creds" 139 VELERO_MYSQL_SECRET_NAME = "velero-mysql-backup-creds" 140 RANCHER_SECRET_NAME = "rancher-backup-creds" 141 OCI_OS_ACCESS_KEY = credentials('velero-key-id') 142 OCI_OS_ACCESS_SECRET_KEY = credentials('velero-key') 143 OCI_OS_COMPARTMENT_ID = credentials('v8o-velero-test') 144 OCI_OS_BUCKET_ID = "${ociOsBucketName}" 145 OCI_OS_BUCKET_NAME = "bucket-${OCI_OS_BUCKET_ID}" 146 BACKUP_OPENSEARCH = "backup-opensearch-${OCI_OS_BUCKET_ID}" 147 BACKUP_RANCHER = "backup-rancher-${OCI_OS_BUCKET_ID}" 148 BACKUP_MYSQL = "backup-mysql-${OCI_OS_BUCKET_ID}" 149 BACKUP_RESOURCE = "vz-hook-resource-${OCI_OS_BUCKET_ID}" 150 RESTORE_OPENSEARCH = "restore-opensearch-${OCI_OS_BUCKET_ID}" 151 RESTORE_RANCHER = "restore-rancher-${OCI_OS_BUCKET_ID}" 152 RESTORE_MYSQL = "restore-mysql-${OCI_OS_BUCKET_ID}" 153 BACKUP_OPENSEARCH_STORAGE = "storage-opensearch-${OCI_OS_BUCKET_ID}" 154 BACKUP_MYSQL_STORAGE = "storage-mysql-${OCI_OS_BUCKET_ID}" 155 BACKUP_REGION = "us-phoenix-1" 156 157 //OCI parameters 158 OCI_CLI_TENANCY = credentials('oci-tenancy') 159 OCI_CLI_USER = credentials('oci-user-ocid') 160 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 161 OCI_CLI_KEY_FILE = credentials('oci-api-key') 162 } 163 164 stages { 165 stage('Clean workspace and checkout') { 166 steps { 167 sh """ 168 echo "${NODE_LABELS}" 169 """ 170 171 script { 172 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 173 if (params.GIT_COMMIT_TO_USE == "NONE") { 174 echo "Specific GIT commit was not specified, use current head" 175 def scmInfo = checkout scm 176 env.GIT_COMMIT = scmInfo.GIT_COMMIT 177 env.GIT_BRANCH = scmInfo.GIT_BRANCH 178 } else { 179 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 180 def scmInfo = checkout([ 181 $class: 'GitSCM', 182 branches: [[name: params.GIT_COMMIT_TO_USE]], 183 doGenerateSubmoduleConfigurations: false, 184 extensions: [], 185 submoduleCfg: [], 186 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 187 env.GIT_COMMIT = scmInfo.GIT_COMMIT 188 env.GIT_BRANCH = scmInfo.GIT_BRANCH 189 // If the commit we were handed is not what the SCM says we are using, fail 190 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 191 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 192 exit 1 193 } 194 } 195 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 196 } 197 198 sh """ 199 cp -f "${NETRC_FILE}" $HOME/.netrc 200 chmod 600 $HOME/.netrc 201 """ 202 203 script { 204 try { 205 sh """ 206 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 207 """ 208 } catch(error) { 209 echo "docker login failed, retrying after sleep" 210 retry(4) { 211 sleep(30) 212 sh """ 213 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 214 """ 215 } 216 } 217 } 218 sh """ 219 rm -rf ${GO_REPO_PATH}/verrazzano 220 mkdir -p ${GO_REPO_PATH}/verrazzano 221 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 222 """ 223 224 script { 225 def props = readProperties file: '.verrazzano-development-version' 226 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 227 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 228 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 229 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 230 // update the description with some meaningful info 231 setDisplayName() 232 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 233 } 234 script { 235 sh """ 236 echo "Downloading VZ CLI from object storage" 237 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 238 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 239 ${GO_REPO_PATH}/vz version 240 """ 241 } 242 } 243 } 244 245 stage('Create bucket') { 246 steps { 247 createBucket("${OCI_OS_BUCKET_NAME}") 248 } 249 } 250 251 stage('Prepare KinD environment') { 252 environment { 253 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 254 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 255 REALM_USER_PASSWORD = credentials('todo-mysql-password') 256 REALM_NAME = "test-realm" 257 } 258 steps { 259 sh """ 260 cd ${GO_REPO_PATH}/verrazzano 261 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA} 262 """ 263 script { 264 if (params.ENABLE_JWT_TESTING) { 265 sh """ 266 # setup test realm for JWT testing scenarios 267 keycloakPassword=\$(kubectl get secret --namespace keycloak keycloak-http -o jsonpath={.data.password} | base64 --decode; echo) 268 sed -i "s|##KEYCLOAK_PASSWORD##|\$keycloakPassword|g" ci/scripts/create_test_realm.sh 269 sed -i "s|##REALM_USER_PASSWORD##|${env.REALM_USER_PASSWORD}|g" ci/scripts/create_test_realm.sh 270 sed -i "s|##REALM_NAME##|${env.REALM_NAME}|g" ci/scripts/create_test_realm.sh 271 kubectl exec keycloak-0 -n keycloak -- /bin/sh -c "`cat ci/scripts/create_test_realm.sh`" 272 # setup request authentication policy 273 keycloakURI=\$(kubectl get ingress -n keycloak keycloak -o jsonpath="{.spec.rules[0].host}") 274 sed -i "s|##KEYCLOAK_URI##|\$keycloakURI|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml 275 jwks=\$(curl -sk https://\$keycloakURI/auth/realms/test-realm/protocol/openid-connect/certs | jq -c 'del(.keys[] | select( .use == "enc"))') 276 sed -i "s|##JWKS_KEY##|\$jwks|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml 277 kubectl apply -f tests/testdata/jwt/helidon/test-realm-reqauth.yaml 278 """ 279 } 280 } 281 } 282 post { 283 failure { 284 archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true 285 } 286 always { 287 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true 288 // enable debug logging of Verrazzano api istio proxy 289 script { 290 if (params.ENABLE_API_ENVOY_LOGGING) { 291 sh ''' 292 vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\") 293 if [ -z "\$vz_api_pod" ]; then 294 echo "Could not find verrazzano-authproxy pod, not enabling debug logging" 295 else 296 kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug 297 fi 298 nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\") 299 if [ -z "\$nginx_ing_pod" ]; then 300 echo "Could not find nginx ingress controller pod, not enabling debug logging" 301 else 302 kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug 303 fi 304 ''' 305 } 306 } 307 } 308 } 309 } 310 311 stage('verify-install') { 312 steps { 313 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 314 sh """ 315 cd ${WORKSPACE}/tests/e2e 316 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/... 317 """ 318 } 319 } 320 post { 321 failure { 322 script { 323 dumpK8sCluster('verify-install-cluster-dump') 324 } 325 } 326 } 327 } 328 329 stage('opensearch backup and restore') { 330 when { 331 expression {params.OPENSEARCH_BACKUP == true} 332 } 333 environment { 334 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/opensearch" 335 } 336 steps { 337 println("Checking all pods in namespace verrazzano-system are ready before start of test") 338 waitForPodsToBeUp("verrazzano-logging") 339 runGinkgoRandomize('backup/opensearch') 340 println("Checking all pods in namespace verrazzano-system are ready after test completion") 341 waitForPodsToBeUp("verrazzano-logging") 342 } 343 post { 344 always { 345 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 346 junit testResults: '**/*test-result.xml', allowEmptyResults: true 347 } 348 failure { 349 script { 350 dumpK8sCluster('opensearch-backup-restore-cluster-dump') 351 } 352 } 353 } 354 } 355 356 stage('rancher backup and restore') { 357 when { 358 expression {params.RANCHER_BACKUP == true} 359 } 360 environment { 361 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/rancher" 362 } 363 steps { 364 echoAllPodInfo() 365 runGinkgoRandomize('backup/rancher') 366 } 367 post { 368 always { 369 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 370 junit testResults: '**/*test-result.xml', allowEmptyResults: true 371 } 372 failure { 373 script { 374 dumpK8sCluster('rancher-backup-restore-cluster-dump') 375 } 376 } 377 } 378 } 379 380 381 stage('mysql operator backup and restore with OCI user principal') { 382 when { 383 allOf { 384 expression {params.MYSQL_BACKUP == true} 385 expression {params.MYSQL_BACKUP_S3 != true} 386 } 387 } 388 environment { 389 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/mysql" 390 } 391 steps { 392 println("MySQL backup with OCI user principal...") 393 echoAllPodInfo() 394 println("Checking all pods in namespace keycloak are ready before start of test") 395 waitForPodsToBeUp("keycloak") 396 runGinkgoRandomize('backup/mysql') 397 println("Checking all pods in namespace keycloak are ready after completion of test") 398 waitForPodsToBeUp("keycloak") 399 } 400 post { 401 always { 402 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 403 junit testResults: '**/*test-result.xml', allowEmptyResults: true 404 } 405 failure { 406 script { 407 dumpK8sCluster('mysql-backup-restore-cluster-dump') 408 } 409 } 410 } 411 } 412 413 stage('mysql operator backup and restore with S3') { 414 when { 415 allOf { 416 expression {params.MYSQL_BACKUP == true} 417 expression {params.MYSQL_BACKUP_S3 == true} 418 } 419 } 420 environment { 421 MYSQL_BACKUP_MODE = "s3" 422 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/mysql" 423 } 424 steps { 425 println("MySQL backup with S3 credentials ...") 426 echoAllPodInfo() 427 println("Checking all pods in namespace keycloak are ready before start of test") 428 waitForPodsToBeUp("keycloak") 429 runGinkgoRandomize('backup/mysql') 430 println("Checking all pods in namespace keycloak are ready after completion of test") 431 waitForPodsToBeUp("keycloak") 432 } 433 post { 434 always { 435 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 436 junit testResults: '**/*test-result.xml', allowEmptyResults: true 437 } 438 failure { 439 script { 440 dumpK8sCluster('mysql-backup-restore-cluster-dump') 441 } 442 } 443 } 444 } 445 446 } 447 448 post { 449 always { 450 script { 451 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 452 dumpVerrazzanoSystemPods() 453 dumpCattleSystemPods() 454 dumpNginxIngressControllerLogs() 455 dumpVerrazzanoPlatformOperatorLogs() 456 dumpVerrazzanoApplicationOperatorLogs() 457 dumpOamKubernetesRuntimeLogs() 458 dumpVerrazzanoApiLogs() 459 } 460 } 461 462 sh """ 463 # Copy the generated test reports to WORKSPACE to archive them 464 mkdir -p ${TEST_REPORT_DIR} 465 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 466 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 467 """ 468 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true 469 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 470 script { 471 if (params.DELETE_BUCKET) { 472 println("Delete object store bucket.") 473 deleteBucket("${OCI_OS_BUCKET_NAME}") 474 } else { 475 println("Object store bucket not deleted!") 476 } 477 } 478 deleteCluster() 479 } 480 failure { 481 sh """ 482 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 483 """ 484 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 485 sh """ 486 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 487 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 488 rm archive.zip 489 """ 490 } 491 cleanup { 492 deleteDir() 493 } 494 } 495 } 496 497 def runGinkgoRandomize(testSuitePath, kubeConfig = '') { 498 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 499 sh """ 500 if [ ! -z "${kubeConfig}" ]; then 501 export KUBECONFIG="${kubeConfig}" 502 fi 503 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 504 if [ -d "${testSuitePath}" ]; then 505 ginkgo -vv --progress --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 506 fi 507 """ 508 } 509 } 510 511 def createBucket(bucketName) { 512 sh """ 513 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os bucket create -c ${OCI_OS_COMPARTMENT_ID} --namespace ${OCI_OS_NAMESPACE} --name ${bucketName} 514 """ 515 } 516 517 def deleteBucket(bucketName) { 518 sh """ 519 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os bucket delete -bn ${bucketName} --empty --force 520 """ 521 } 522 523 def waitForPodsToBeUp(namespace) { 524 sh """ 525 kubectl wait --selector='!job-name' -n ${namespace} --for=condition=ready pod --all --timeout 10m 526 """ 527 } 528 529 def echoAllPodInfo() { 530 sh """ 531 kubectl get pod -A 532 """ 533 } 534 535 def runSocksVariant(variant) { 536 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 537 sh """ 538 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 539 SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" examples/socks/... 540 """ 541 } 542 } 543 544 def runGinkgo(testSuitePath) { 545 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 546 sh """ 547 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 548 ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 549 """ 550 } 551 } 552 553 def dumpK8sCluster(dumpDirectory) { 554 sh """ 555 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 556 """ 557 } 558 559 def dumpVerrazzanoSystemPods() { 560 sh """ 561 cd ${GO_REPO_PATH}/verrazzano/platform-operator 562 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log" 563 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 564 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log" 565 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 566 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log" 567 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 568 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log" 569 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 570 """ 571 } 572 573 def dumpCattleSystemPods() { 574 sh """ 575 cd ${GO_REPO_PATH}/verrazzano/platform-operator 576 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log" 577 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 578 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log" 579 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 580 """ 581 } 582 583 def dumpNginxIngressControllerLogs() { 584 sh """ 585 cd ${GO_REPO_PATH}/verrazzano/platform-operator 586 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log" 587 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 588 """ 589 } 590 591 def dumpVerrazzanoPlatformOperatorLogs() { 592 sh """ 593 ## dump out verrazzano-platform-operator logs 594 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 595 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 596 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 597 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 598 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 599 echo "------------------------------------------" 600 """ 601 } 602 603 def dumpVerrazzanoApplicationOperatorLogs() { 604 sh """ 605 ## dump out verrazzano-application-operator logs 606 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 607 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 608 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 609 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 610 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 611 echo "------------------------------------------" 612 """ 613 } 614 615 def dumpOamKubernetesRuntimeLogs() { 616 sh """ 617 ## dump out oam-kubernetes-runtime logs 618 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 619 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 620 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 621 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 622 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 623 echo "------------------------------------------" 624 """ 625 } 626 627 def dumpVerrazzanoApiLogs() { 628 sh """ 629 cd ${GO_REPO_PATH}/verrazzano/platform-operator 630 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log" 631 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 632 """ 633 } 634 635 def getEffectiveDumpOnSuccess() { 636 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 637 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 638 effectiveValue = true 639 echo "Forcing dump on success based on global override setting" 640 } 641 return effectiveValue 642 } 643 644 def deleteCluster() { 645 sh """ 646 cd ${GO_REPO_PATH}/verrazzano/platform-operator 647 make delete-cluster 648 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 649 echo "Failures seen during dumping of artifacts, treat post as failed" 650 exit 1 651 fi 652 """ 653 } 654 655 def setDisplayName() { 656 echo "Start setDisplayName" 657 def causes = currentBuild.getBuildCauses() 658 echo "causes: " + causes.toString() 659 for (cause in causes) { 660 def causeString = cause.toString() 661 echo "current cause: " + causeString 662 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 663 echo "This job was caused by " + causeString 664 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 665 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 666 } else if (causeString.contains("verrazzano-flaky-tests")) { 667 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 668 } 669 } 670 } 671 echo "End setDisplayName" 672 }