github.com/verrazzano/verrazzano@v1.7.0/ci/backup/JenkinsfileAlllBackupKinDTest (about) 1 // Copyright (c) 2022, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 // def agentLabel = "largeexperimental" 7 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 8 def ociOsBucketName = UUID.randomUUID().toString().substring(0,6).replace('-','') 9 10 pipeline { 11 options { 12 timeout(time: 1, unit: 'HOURS') 13 skipDefaultCheckout true 14 timestamps () 15 } 16 17 agent { 18 docker { 19 image "${RUNNER_DOCKER_IMAGE}" 20 args "${RUNNER_DOCKER_ARGS}" 21 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 22 registryCredentialsId 'ocir-pull-and-push-account' 23 label "${agentLabel}" 24 } 25 } 26 27 parameters { 28 choice (name: 'KUBERNETES_CLUSTER_VERSION', 29 description: 'Kubernetes Version for KinD Cluster', 30 // 1st choice is the default value 31 choices: [ "1.27", "1.26", "1.25", "1.24" ]) 32 string (name: 'GIT_COMMIT_TO_USE', 33 defaultValue: 'NONE', 34 description: 'This is the full git commit hash from the source build to be used for all jobs', 35 trim: true) 36 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 37 defaultValue: 'NONE', 38 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator', 39 trim: true) 40 choice (name: 'WILDCARD_DNS_DOMAIN', 41 description: 'This is the wildcard DNS domain', 42 // 1st choice is the default value 43 choices: [ "nip.io", "sslip.io"]) 44 choice (name: 'CRD_API_VERSION', 45 description: 'This is the API crd version.', 46 // 1st choice is the default value 47 choices: [ "v1beta1", "v1alpha1"]) 48 booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 49 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 50 booleanParam (description: 'Whether to use a database for Grafana persistence', name: 'USE_DB_FOR_GRAFANA', defaultValue: false) 51 string (name: 'CONSOLE_REPO_BRANCH', 52 defaultValue: '', 53 description: 'The branch to check out after cloning the console repository.', 54 trim: true) 55 booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true) 56 string (name: 'TAGGED_TESTS', 57 defaultValue: '', 58 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 59 trim: true) 60 string (name: 'INCLUDED_TESTS', 61 defaultValue: '.*', 62 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 63 trim: true) 64 string (name: 'EXCLUDED_TESTS', 65 defaultValue: '_excluded_test', 66 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 67 trim: true) 68 booleanParam (description: 'Whether to run JWT tests', name: 'ENABLE_JWT_TESTING', defaultValue: false) 69 booleanParam (description: 'Whether to run slow tests', name: 'RUN_SLOW_TESTS', defaultValue: false) 70 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 71 booleanParam (description: 'Whether to persist object store bucket', name: 'DELETE_BUCKET', defaultValue: true) 72 booleanParam (description: 'Whether to execute MySQL backup tests', name: 'MYSQL_BACKUP', defaultValue: true) 73 booleanParam (description: 'Whether to execute MySQL backup tests with S3 API. Default is true. If unchecked user principal creds will be used', name: 'MYSQL_BACKUP_S3', defaultValue: true) 74 booleanParam (description: 'Whether to execute OpenSearch backup tests', name: 'OPENSEARCH_BACKUP', defaultValue: true) 75 booleanParam (description: 'Whether to execute Rancher backup tests', name: 'RANCHER_BACKUP', defaultValue: true) 76 } 77 78 environment { 79 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 80 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 81 GOPATH = '/home/opc/go' 82 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 83 DOCKER_CREDS = credentials('github-packages-credentials-rw') 84 DOCKER_EMAIL = credentials('github-packages-email') 85 DOCKER_REPO = 'ghcr.io' 86 DOCKER_NAMESPACE = 'verrazzano' 87 NETRC_FILE = credentials('netrc') 88 CLUSTER_NAME = 'verrazzano' 89 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 90 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 91 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 92 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 93 OCR_CREDS = credentials('ocr-pull-and-push-account') 94 OCR_REPO = 'container-registry.oracle.com' 95 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 96 INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-backup-enabled.yaml" 97 INSTALL_PROFILE = "prod" 98 KIND_NODE_COUNT = "3" 99 VZ_ENVIRONMENT_NAME = "default" 100 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 101 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 102 103 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test 104 DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test 105 106 // Environment variables required to capture cluster snapshot and bug report on test failure 107 DUMP_KUBECONFIG="${KUBECONFIG}" 108 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 109 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 110 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 111 112 // Environment variable for Verrazzano CLI executable 113 VZ_COMMAND="${GO_REPO_PATH}/vz" 114 115 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 116 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 117 118 // used for console artifact capture on failure 119 JENKINS_READ = credentials('jenkins-auditor') 120 // OCI_CLI_AUTH="instance_principal" 121 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 122 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 123 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 124 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 125 126 // used to emit metrics 127 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 128 TEST_ENV_LABEL = "kind" 129 TEST_ENV = "kind_oci_dns" 130 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 131 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 132 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 133 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 134 135 // used to generate Ginkgo test reports 136 TEST_REPORT = "test-report.xml" 137 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 138 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 139 140 // Backup parameters 141 VELERO_NAMESPACE = "verrazzano-backup" 142 VELERO_SECRET_NAME = "velero-opensearch-backup-creds" 143 VELERO_MYSQL_SECRET_NAME = "velero-mysql-backup-creds" 144 RANCHER_SECRET_NAME = "rancher-backup-creds" 145 OCI_OS_ACCESS_KEY = credentials('velero-key-id') 146 OCI_OS_ACCESS_SECRET_KEY = credentials('velero-key') 147 OCI_OS_COMPARTMENT_ID = credentials('v8o-velero-test') 148 OCI_OS_BUCKET_ID = "${ociOsBucketName}" 149 OCI_OS_BUCKET_NAME = "bucket-${OCI_OS_BUCKET_ID}" 150 BACKUP_OPENSEARCH = "backup-opensearch-${OCI_OS_BUCKET_ID}" 151 BACKUP_RANCHER = "backup-rancher-${OCI_OS_BUCKET_ID}" 152 BACKUP_MYSQL = "backup-mysql-${OCI_OS_BUCKET_ID}" 153 BACKUP_RESOURCE = "vz-hook-resource-${OCI_OS_BUCKET_ID}" 154 RESTORE_OPENSEARCH = "restore-opensearch-${OCI_OS_BUCKET_ID}" 155 RESTORE_RANCHER = "restore-rancher-${OCI_OS_BUCKET_ID}" 156 RESTORE_MYSQL = "restore-mysql-${OCI_OS_BUCKET_ID}" 157 BACKUP_OPENSEARCH_STORAGE = "storage-opensearch-${OCI_OS_BUCKET_ID}" 158 BACKUP_MYSQL_STORAGE = "storage-mysql-${OCI_OS_BUCKET_ID}" 159 BACKUP_REGION = "us-phoenix-1" 160 161 //OCI parameters 162 OCI_CLI_TENANCY = credentials('oci-tenancy') 163 OCI_CLI_USER = credentials('oci-user-ocid') 164 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 165 OCI_CLI_KEY_FILE = credentials('oci-api-key') 166 } 167 168 stages { 169 stage('Clean workspace and checkout') { 170 steps { 171 sh """ 172 echo "${NODE_LABELS}" 173 """ 174 175 script { 176 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 177 if (params.GIT_COMMIT_TO_USE == "NONE") { 178 echo "Specific GIT commit was not specified, use current head" 179 def scmInfo = checkout scm 180 env.GIT_COMMIT = scmInfo.GIT_COMMIT 181 env.GIT_BRANCH = scmInfo.GIT_BRANCH 182 } else { 183 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 184 def scmInfo = checkout([ 185 $class: 'GitSCM', 186 branches: [[name: params.GIT_COMMIT_TO_USE]], 187 doGenerateSubmoduleConfigurations: false, 188 extensions: [], 189 submoduleCfg: [], 190 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 191 env.GIT_COMMIT = scmInfo.GIT_COMMIT 192 env.GIT_BRANCH = scmInfo.GIT_BRANCH 193 // If the commit we were handed is not what the SCM says we are using, fail 194 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 195 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 196 exit 1 197 } 198 } 199 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 200 } 201 202 sh """ 203 cp -f "${NETRC_FILE}" $HOME/.netrc 204 chmod 600 $HOME/.netrc 205 """ 206 207 script { 208 try { 209 sh """ 210 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 211 """ 212 } catch(error) { 213 echo "docker login failed, retrying after sleep" 214 retry(4) { 215 sleep(30) 216 sh """ 217 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 218 """ 219 } 220 } 221 } 222 sh """ 223 rm -rf ${GO_REPO_PATH}/verrazzano 224 mkdir -p ${GO_REPO_PATH}/verrazzano 225 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 226 """ 227 228 script { 229 def props = readProperties file: '.verrazzano-development-version' 230 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 231 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 232 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 233 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 234 // update the description with some meaningful info 235 setDisplayName() 236 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 237 } 238 script { 239 sh """ 240 echo "Downloading VZ CLI from object storage" 241 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 242 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 243 ${GO_REPO_PATH}/vz version 244 """ 245 } 246 } 247 } 248 249 stage('Create bucket') { 250 steps { 251 createBucket("${OCI_OS_BUCKET_NAME}") 252 } 253 } 254 255 stage('Prepare KinD environment') { 256 environment { 257 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 258 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 259 REALM_USER_PASSWORD = credentials('todo-mysql-password') 260 REALM_NAME = "test-realm" 261 } 262 steps { 263 sh """ 264 cd ${GO_REPO_PATH}/verrazzano 265 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA} 266 """ 267 script { 268 if (params.ENABLE_JWT_TESTING) { 269 sh """ 270 # setup test realm for JWT testing scenarios 271 keycloakPassword=\$(kubectl get secret --namespace keycloak keycloak-http -o jsonpath={.data.password} | base64 --decode; echo) 272 sed -i "s|##KEYCLOAK_PASSWORD##|\$keycloakPassword|g" ci/scripts/create_test_realm.sh 273 sed -i "s|##REALM_USER_PASSWORD##|${env.REALM_USER_PASSWORD}|g" ci/scripts/create_test_realm.sh 274 sed -i "s|##REALM_NAME##|${env.REALM_NAME}|g" ci/scripts/create_test_realm.sh 275 kubectl exec keycloak-0 -n keycloak -- /bin/sh -c "`cat ci/scripts/create_test_realm.sh`" 276 # setup request authentication policy 277 keycloakURI=\$(kubectl get ingress -n keycloak keycloak -o jsonpath="{.spec.rules[0].host}") 278 sed -i "s|##KEYCLOAK_URI##|\$keycloakURI|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml 279 jwks=\$(curl -sk https://\$keycloakURI/auth/realms/test-realm/protocol/openid-connect/certs | jq -c 'del(.keys[] | select( .use == "enc"))') 280 sed -i "s|##JWKS_KEY##|\$jwks|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml 281 kubectl apply -f tests/testdata/jwt/helidon/test-realm-reqauth.yaml 282 """ 283 } 284 } 285 } 286 post { 287 failure { 288 archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true 289 } 290 always { 291 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true 292 // enable debug logging of Verrazzano api istio proxy 293 script { 294 if (params.ENABLE_API_ENVOY_LOGGING) { 295 sh ''' 296 vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\") 297 if [ -z "\$vz_api_pod" ]; then 298 echo "Could not find verrazzano-authproxy pod, not enabling debug logging" 299 else 300 kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug 301 fi 302 nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\") 303 if [ -z "\$nginx_ing_pod" ]; then 304 echo "Could not find nginx ingress controller pod, not enabling debug logging" 305 else 306 kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug 307 fi 308 ''' 309 } 310 } 311 } 312 } 313 } 314 315 stage('verify-install') { 316 steps { 317 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 318 sh """ 319 cd ${WORKSPACE}/tests/e2e 320 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/... 321 """ 322 } 323 } 324 post { 325 failure { 326 script { 327 dumpK8sCluster('verify-install-cluster-dump') 328 } 329 } 330 } 331 } 332 333 stage('opensearch backup and restore') { 334 when { 335 expression {params.OPENSEARCH_BACKUP == true} 336 } 337 environment { 338 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/opensearch" 339 } 340 steps { 341 println("Checking all pods in namespace verrazzano-system are ready before start of test") 342 waitForPodsToBeUp("verrazzano-logging") 343 runGinkgoRandomize('backup/opensearch') 344 println("Checking all pods in namespace verrazzano-system are ready after test completion") 345 waitForPodsToBeUp("verrazzano-logging") 346 } 347 post { 348 always { 349 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 350 junit testResults: '**/*test-result.xml', allowEmptyResults: true 351 } 352 failure { 353 script { 354 dumpK8sCluster('opensearch-backup-restore-cluster-dump') 355 } 356 } 357 } 358 } 359 360 stage('rancher backup and restore') { 361 when { 362 expression {params.RANCHER_BACKUP == true} 363 } 364 environment { 365 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/rancher" 366 } 367 steps { 368 echoAllPodInfo() 369 runGinkgoRandomize('backup/rancher') 370 } 371 post { 372 always { 373 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 374 junit testResults: '**/*test-result.xml', allowEmptyResults: true 375 } 376 failure { 377 script { 378 dumpK8sCluster('rancher-backup-restore-cluster-dump') 379 } 380 } 381 } 382 } 383 384 385 stage('mysql operator backup and restore with OCI user principal') { 386 when { 387 allOf { 388 expression {params.MYSQL_BACKUP == true} 389 expression {params.MYSQL_BACKUP_S3 != true} 390 } 391 } 392 environment { 393 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/mysql" 394 } 395 steps { 396 println("MySQL backup with OCI user principal...") 397 echoAllPodInfo() 398 println("Checking all pods in namespace keycloak are ready before start of test") 399 waitForPodsToBeUp("keycloak") 400 runGinkgoRandomize('backup/mysql') 401 println("Checking all pods in namespace keycloak are ready after completion of test") 402 waitForPodsToBeUp("keycloak") 403 } 404 post { 405 always { 406 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 407 junit testResults: '**/*test-result.xml', allowEmptyResults: true 408 } 409 failure { 410 script { 411 dumpK8sCluster('mysql-backup-restore-cluster-dump') 412 } 413 } 414 } 415 } 416 417 stage('mysql operator backup and restore with S3') { 418 when { 419 allOf { 420 expression {params.MYSQL_BACKUP == true} 421 expression {params.MYSQL_BACKUP_S3 == true} 422 } 423 } 424 environment { 425 MYSQL_BACKUP_MODE = "s3" 426 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/backup/mysql" 427 } 428 steps { 429 println("MySQL backup with S3 credentials ...") 430 echoAllPodInfo() 431 println("Checking all pods in namespace keycloak are ready before start of test") 432 waitForPodsToBeUp("keycloak") 433 runGinkgoRandomize('backup/mysql') 434 println("Checking all pods in namespace keycloak are ready after completion of test") 435 waitForPodsToBeUp("keycloak") 436 } 437 post { 438 always { 439 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 440 junit testResults: '**/*test-result.xml', allowEmptyResults: true 441 } 442 failure { 443 script { 444 dumpK8sCluster('mysql-backup-restore-cluster-dump') 445 } 446 } 447 } 448 } 449 450 } 451 452 post { 453 always { 454 script { 455 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 456 dumpVerrazzanoSystemPods() 457 dumpCattleSystemPods() 458 dumpNginxIngressControllerLogs() 459 dumpVerrazzanoPlatformOperatorLogs() 460 dumpVerrazzanoApplicationOperatorLogs() 461 dumpOamKubernetesRuntimeLogs() 462 dumpVerrazzanoApiLogs() 463 } 464 } 465 466 sh """ 467 # Copy the generated test reports to WORKSPACE to archive them 468 mkdir -p ${TEST_REPORT_DIR} 469 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 470 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 471 """ 472 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true 473 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 474 script { 475 if (params.DELETE_BUCKET) { 476 println("Delete object store bucket.") 477 deleteBucket("${OCI_OS_BUCKET_NAME}") 478 } else { 479 println("Object store bucket not deleted!") 480 } 481 } 482 deleteCluster() 483 } 484 failure { 485 sh """ 486 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 487 """ 488 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 489 sh """ 490 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 491 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 492 rm archive.zip 493 """ 494 } 495 cleanup { 496 deleteDir() 497 } 498 } 499 } 500 501 def runGinkgoRandomize(testSuitePath, kubeConfig = '') { 502 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 503 sh """ 504 if [ ! -z "${kubeConfig}" ]; then 505 export KUBECONFIG="${kubeConfig}" 506 fi 507 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 508 if [ -d "${testSuitePath}" ]; then 509 ginkgo -vv --progress --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 510 fi 511 """ 512 } 513 } 514 515 def createBucket(bucketName) { 516 sh """ 517 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os bucket create -c ${OCI_OS_COMPARTMENT_ID} --namespace ${OCI_OS_NAMESPACE} --name ${bucketName} 518 """ 519 } 520 521 def deleteBucket(bucketName) { 522 sh """ 523 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os bucket delete -bn ${bucketName} --empty --force 524 """ 525 } 526 527 def waitForPodsToBeUp(namespace) { 528 sh """ 529 kubectl wait --selector='!job-name' -n ${namespace} --for=condition=ready pod --all --timeout 10m 530 """ 531 } 532 533 def echoAllPodInfo() { 534 sh """ 535 kubectl get pod -A 536 """ 537 } 538 539 def runSocksVariant(variant) { 540 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 541 sh """ 542 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 543 SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" examples/socks/... 544 """ 545 } 546 } 547 548 def runGinkgo(testSuitePath) { 549 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 550 sh """ 551 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 552 ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 553 """ 554 } 555 } 556 557 def dumpK8sCluster(dumpDirectory) { 558 sh """ 559 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 560 """ 561 } 562 563 def dumpVerrazzanoSystemPods() { 564 sh """ 565 cd ${GO_REPO_PATH}/verrazzano/platform-operator 566 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log" 567 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 568 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log" 569 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 570 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log" 571 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 572 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log" 573 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 574 """ 575 } 576 577 def dumpCattleSystemPods() { 578 sh """ 579 cd ${GO_REPO_PATH}/verrazzano/platform-operator 580 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log" 581 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 582 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log" 583 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 584 """ 585 } 586 587 def dumpNginxIngressControllerLogs() { 588 sh """ 589 cd ${GO_REPO_PATH}/verrazzano/platform-operator 590 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log" 591 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 592 """ 593 } 594 595 def dumpVerrazzanoPlatformOperatorLogs() { 596 sh """ 597 ## dump out verrazzano-platform-operator logs 598 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 599 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 600 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 601 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 602 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 603 echo "------------------------------------------" 604 """ 605 } 606 607 def dumpVerrazzanoApplicationOperatorLogs() { 608 sh """ 609 ## dump out verrazzano-application-operator logs 610 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 611 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 612 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 613 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 614 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 615 echo "------------------------------------------" 616 """ 617 } 618 619 def dumpOamKubernetesRuntimeLogs() { 620 sh """ 621 ## dump out oam-kubernetes-runtime logs 622 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 623 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 624 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 625 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 626 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 627 echo "------------------------------------------" 628 """ 629 } 630 631 def dumpVerrazzanoApiLogs() { 632 sh """ 633 cd ${GO_REPO_PATH}/verrazzano/platform-operator 634 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log" 635 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 636 """ 637 } 638 639 def getEffectiveDumpOnSuccess() { 640 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 641 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 642 effectiveValue = true 643 echo "Forcing dump on success based on global override setting" 644 } 645 return effectiveValue 646 } 647 648 def deleteCluster() { 649 sh """ 650 cd ${GO_REPO_PATH}/verrazzano/platform-operator 651 make delete-cluster 652 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 653 echo "Failures seen during dumping of artifacts, treat post as failed" 654 exit 1 655 fi 656 """ 657 } 658 659 def setDisplayName() { 660 echo "Start setDisplayName" 661 def causes = currentBuild.getBuildCauses() 662 echo "causes: " + causes.toString() 663 for (cause in causes) { 664 def causeString = cause.toString() 665 echo "current cause: " + causeString 666 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 667 echo "This job was caused by " + causeString 668 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 669 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 670 } else if (causeString.contains("verrazzano-flaky-tests")) { 671 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 672 } 673 } 674 } 675 echo "End setDisplayName" 676 }