github.com/verrazzano/verrazzano@v1.7.0/ci/uninstall/Jenkinsfile (about) 1 // Copyright (c) 2020, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 6 7 // pulling "ap-*" from the test regions given discovery of image pull issues 8 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-frankfurt-1", "eu-zurich-1", "uk-london-1" ] 9 def OKE_CLUSTER_PREFIX = "" 10 Collections.shuffle(availableRegions) 11 12 pipeline { 13 options { 14 skipDefaultCheckout true 15 } 16 17 agent { 18 docker { 19 image "${RUNNER_DOCKER_IMAGE}" 20 args "${RUNNER_DOCKER_ARGS}" 21 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 22 registryCredentialsId 'ocir-pull-and-push-account' 23 } 24 } 25 26 parameters { 27 string (name: 'VERRAZZANO_BRANCH', 28 defaultValue: 'master', 29 description: 'Branch to clone and checkout the Verrazzano repo', 30 trim: true) 31 choice (name: 'OKE_NODE_POOL', 32 description: 'OKE node pool configuration', 33 // 1st choice is the default value 34 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2", "VM.Standard.E2.4" ]) 35 choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION', 36 // 1st choice is the default value 37 choices: availableRegions ) 38 string (name: 'GIT_COMMIT_TO_USE', 39 defaultValue: 'NONE', 40 description: 'This is the full git commit hash from the source build to be used for all jobs', 41 trim: true) 42 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 43 defaultValue: 'NONE', 44 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator', 45 trim: true) 46 choice (name: 'OKE_CLUSTER_VERSION', 47 description: 'Kubernetes Version for OKE Cluster', 48 // 1st choice is the default value 49 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 50 choice (name: 'CRD_API_VERSION', 51 description: 'This is the API crd version.', 52 // 1st choice is the default value 53 choices: [ "v1beta1", "v1alpha1"]) 54 booleanParam (name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', 55 defaultValue: false, 56 description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)') 57 string (name: 'TAGGED_TESTS', 58 defaultValue: '', 59 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 60 trim: true) 61 string (name: 'INCLUDED_TESTS', 62 defaultValue: '.*', 63 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 64 trim: true) 65 string (name: 'EXCLUDED_TESTS', 66 defaultValue: '_excluded_test', 67 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 68 trim: true) 69 } 70 71 environment { 72 OCR_CREDS = credentials('ocr-pull-and-push-account') 73 NETRC_FILE = credentials('netrc') 74 GOPATH = '/home/opc/go' 75 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 76 DOCKER_CREDS = credentials('github-packages-credentials-rw') 77 DOCKER_EMAIL = credentials('github-packages-email') 78 DOCKER_REPO = 'ghcr.io' 79 80 OCR_REPO = 'container-registry.oracle.com' 81 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 82 83 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 84 TF_VAR_tenancy_id = credentials('oci-tenancy') 85 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 86 TF_VAR_user_id = credentials('oci-user-ocid') 87 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 88 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 89 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 90 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 91 TF_VAR_api_private_key_path = credentials('oci-api-key') 92 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 93 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 94 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 95 96 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 97 OCI_CLI_TENANCY = credentials('oci-tenancy') 98 OCI_CLI_USER = credentials('oci-user-ocid') 99 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 100 OCI_CLI_KEY_FILE = credentials('oci-api-key') 101 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 102 DISABLE_SPINNER=1 103 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 104 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 105 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 106 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 107 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 108 KUBECONFIG = "${WORKSPACE}/oke_kubeconfig" 109 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 110 111 // Environment variable for Verrazzano CLI executable 112 VZ_COMMAND="${GO_REPO_PATH}/vz" 113 INSTALL_PROFILE = "prod" 114 VZ_ENVIRONMENT_NAME = "default" 115 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 116 LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test" 117 UNINSTALL_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/uninstall-test" 118 INSTALL_CONFIG_FILE = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-default.yaml" 119 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') 120 DATABASE_PSW = credentials('todo-mysql-password') 121 122 // used to emit metrics 123 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 124 TEST_ENV_LABEL = "magicdns_oke" 125 TEST_ENV = "OKE" 126 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 127 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 128 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 129 130 // used to generate Ginkgo test reports 131 TEST_REPORT = "test-report.xml" 132 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 133 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 134 } 135 136 stages { 137 stage('Clean workspace and checkout') { 138 environment { 139 OCI_CLI_AUTH="instance_principal" 140 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 141 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 142 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 143 } 144 steps { 145 sh """ 146 echo "${NODE_LABELS}" 147 """ 148 script { 149 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 150 if (params.GIT_COMMIT_TO_USE == "NONE") { 151 echo "Specific GIT commit was not specified, use current head" 152 def scmInfo = checkout scm 153 env.GIT_COMMIT = scmInfo.GIT_COMMIT 154 env.GIT_BRANCH = scmInfo.GIT_BRANCH 155 } else { 156 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 157 def scmInfo = checkout([ 158 $class: 'GitSCM', 159 branches: [[name: params.GIT_COMMIT_TO_USE]], 160 doGenerateSubmoduleConfigurations: false, 161 extensions: [], 162 submoduleCfg: [], 163 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 164 env.GIT_COMMIT = scmInfo.GIT_COMMIT 165 env.GIT_BRANCH = scmInfo.GIT_BRANCH 166 // If the commit we were handed is not what the SCM says we are using, fail 167 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 168 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 169 exit 1 170 } 171 } 172 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 173 } 174 sh """ 175 cp -f "${NETRC_FILE}" $HOME/.netrc 176 chmod 600 $HOME/.netrc 177 """ 178 179 script { 180 try { 181 sh """ 182 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 183 """ 184 } catch(error) { 185 echo "docker login failed, retrying after sleep" 186 retry(4) { 187 sleep(30) 188 sh """ 189 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 190 """ 191 } 192 } 193 } 194 195 sh """ 196 rm -rf ${GO_REPO_PATH}/verrazzano 197 mkdir -p ${GO_REPO_PATH}/verrazzano 198 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 199 cd ${GO_REPO_PATH}/verrazzano 200 git config --global credential.helper "!f() { echo username=\\$DOCKER_CREDS_USR; echo password=\\$DOCKER_CREDS_PSW; }; f" 201 git config --global user.name $DOCKER_CREDS_USR 202 git config --global user.email "${DOCKER_EMAIL}" 203 git checkout -b ${env.BRANCH_NAME} 204 """ 205 206 script { 207 def props = readProperties file: '.verrazzano-development-version' 208 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 209 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 210 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 211 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 212 // update the description with some meaningful info 213 setDisplayName() 214 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 215 216 // derive the prefix for the OKE cluster 217 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 218 } 219 script { 220 sh """ 221 echo "Downloading VZ CLI from object storage" 222 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 223 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 224 ${GO_REPO_PATH}/vz version 225 """ 226 } 227 } 228 } 229 230 stage("Create Cluster") { 231 steps { 232 sh "TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=uninstall-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/create_oke_cluster.sh" 233 } 234 } 235 236 stage('Install Verrazzano') { 237 environment { 238 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 239 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 240 } 241 steps { 242 script { 243 listNamepacesAndPods('before installing Verrazzano') 244 } 245 sh """ 246 # Create image pull secret for Verrazzano docker images 247 cd ${GO_REPO_PATH}/verrazzano 248 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 249 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 250 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 251 252 echo "Prepare Verrazzano install" 253 cd ${GO_REPO_PATH}/verrazzano 254 if [ "NONE" = "${VERRAZZANO_OPERATOR_IMAGE}" ]; then 255 echo "Using operator.yaml from object storage" 256 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 257 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 258 else 259 echo "Generating operator.yaml based on image name provided: ${VERRAZZANO_OPERATOR_IMAGE}" 260 env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/acceptance-test-operator.yaml 261 fi 262 263 # make sure ns exists 264 kubectl create ns verrazzano-install 265 ./tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 266 267 # create secret in verrazzano-install ns 268 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 269 270 ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/pre-install-resources false 271 272 ./tests/e2e/config/scripts/process_nipio_install_yaml.sh ${INSTALL_CONFIG_FILE} 273 yq -i eval '.spec.components.prometheusAdapter.enabled = true' ${INSTALL_CONFIG_FILE} 274 yq -i eval '.spec.components.kubeStateMetrics.enabled = true' ${INSTALL_CONFIG_FILE} 275 yq -i eval '.spec.components.prometheusPushgateway.enabled = true' ${INSTALL_CONFIG_FILE} 276 277 echo "Installing Verrazzano on OKE" 278 ${GO_REPO_PATH}/vz install --filename ${INSTALL_CONFIG_FILE} --manifests ${WORKSPACE}/acceptance-test-operator.yaml --timeout 45m 279 280 ${TEST_SCRIPTS_DIR}/common-test-setup-script.sh "${GO_REPO_PATH}" "${TEST_CONFIG_FILE}" "${env.DOCKER_REPO}" "${KUBECONFIG}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" "default" "${env.OCR_REPO}" 281 ${TEST_SCRIPTS_DIR}/get_ingress_ip.sh ${TEST_CONFIG_FILE} 282 echo "----------Test config file:-------------" 283 cat ${TEST_CONFIG_FILE} 284 echo "----------------------------------------" 285 """ 286 } 287 post { 288 always { 289 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 290 dumpVerrazzanoSystemPods('install') 291 dumpCattleSystemPods('install') 292 dumpNginxIngressControllerLogs('install') 293 dumpVerrazzanoPlatformOperatorLogs('install') 294 dumpVerrazzanoApplicationOperatorLogs('install') 295 dumpOamKubernetesRuntimeLogs('install') 296 dumpVerrazzanoApiLogs('install') 297 listNamepacesAndPods('after Verrazzano install') 298 listHelmReleases('after Verrazzano install') 299 } 300 success { 301 script { 302 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 303 dumpK8sCluster('verrazzano-install-cluster-snapshot') 304 } 305 } 306 } 307 failure { 308 script { 309 dumpK8sCluster('verrazzano-install-failure-cluster-snapshot') 310 sh """ 311 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs 312 ${LOOPING_TEST_SCRIPTS_DIR}/dump_resources.sh > ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs/resources.log 313 """ 314 } 315 } 316 } 317 } 318 319 stage('Run Test') { 320 stages { 321 stage('verify-install') { 322 steps { 323 runGinkgo('verify-install') 324 } 325 } 326 stage('verify-infra restapi') { 327 steps { 328 runGinkgo('verify-infra/restapi') 329 } 330 } 331 } 332 post { 333 success { 334 script { 335 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 336 dumpK8sCluster('verrazzano-testrun-after-install-cluster-snapshot') 337 } 338 } 339 } 340 failure { 341 dumpK8sCluster('verrazzano-test-failure-cluster-snapshot') 342 343 } 344 } 345 } 346 347 stage('Uninstall Verrazzano') { 348 steps { 349 sh """ 350 ${GO_REPO_PATH}/vz uninstall -y --timeout 60m 351 """ 352 } 353 post { 354 always { 355 sh """ 356 ## dump out uninstall logs 357 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/uninstall/build/logs 358 echo "Listing all pods in all namespaces after uninstall" 359 kubectl get pods --all-namespaces 360 echo "-----------------------------------------------------" 361 """ 362 listNamepacesAndPods('after Verrazzano uninstall') 363 listHelmReleases('after Verrazzano uninstall') 364 } 365 success { 366 script { 367 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 368 dumpK8sCluster('verrazzano-uninstall-cluster-snapshot') 369 } 370 } 371 } 372 failure { 373 dumpK8sCluster('verrazzano-uninstall-failure-cluster-snapshot') 374 } 375 } 376 } 377 378 stage("Verify Uninstall") { 379 steps { 380 retry(6) { 381 sleep time: 300, unit: 'SECONDS' 382 sh """ 383 ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/post-uninstall-resources false 384 ${LOOPING_TEST_SCRIPTS_DIR}/verify_uninstall.sh ${WORKSPACE}/verrazzano/build/resources 385 """ 386 } 387 } 388 post { 389 success { 390 script { 391 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 392 dumpK8sCluster('verify-uninstall-cluster-snapshot') 393 } 394 } 395 } 396 failure { 397 dumpK8sCluster('verify-uninstall-cluster-snapshot') 398 } 399 } 400 } 401 402 stage("Reinstall Verrazzano") { 403 steps { 404 sh """ 405 # sleep for a period to ensure async deletion of Verrazzano components from uninstall above has completed 406 sleep 90 407 # make sure ns exists 408 kubectl create ns verrazzano-install 409 ./tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 410 411 # create secret in verrazzano-install ns 412 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 413 414 415 ${GO_REPO_PATH}/vz install --filename ${INSTALL_CONFIG_FILE} --manifests ${WORKSPACE}/acceptance-test-operator.yaml --timeout 45m 416 417 ${TEST_SCRIPTS_DIR}/common-test-setup-script.sh "${GO_REPO_PATH}" "${TEST_CONFIG_FILE}" "${env.DOCKER_REPO}" "${KUBECONFIG}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" "default" 418 ${TEST_SCRIPTS_DIR}/get_ingress_ip.sh ${TEST_CONFIG_FILE} 419 echo "----------Test config file:-------------" 420 cat ${TEST_CONFIG_FILE} 421 echo "----------------------------------------" 422 """ 423 } 424 post { 425 always { 426 dumpVerrazzanoSystemPods('reinstall') 427 dumpCattleSystemPods('reinstall') 428 dumpNginxIngressControllerLogs('reinstall') 429 dumpVerrazzanoPlatformOperatorLogs('reinstall') 430 dumpVerrazzanoApplicationOperatorLogs('reinstall') 431 dumpOamKubernetesRuntimeLogs('reinstall') 432 dumpVerrazzanoApiLogs('reinstall') 433 listNamepacesAndPods('after reinstalling Verrazzano') 434 listHelmReleases('after reinstalling Verrazzano') 435 } 436 success { 437 script { 438 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 439 dumpK8sCluster('verrazzano-reinstall-cluster-snapshot') 440 } 441 } 442 } 443 failure { 444 dumpK8sCluster('verrazzano-reinstall-failure-cluster-snapshot') 445 sh """ 446 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/reinstall/build/logs 447 ${LOOPING_TEST_SCRIPTS_DIR}/dump_resources.sh > ${WORKSPACE}/verrazzano-platform-operator/scripts/reinstall/build/logs/resources.log 448 """ 449 } 450 } 451 } 452 453 stage('Rerun Test') { 454 stages { 455 stage('verify-install') { 456 steps { 457 runGinkgo('verify-install') 458 } 459 } 460 stage('verify-infra restapi') { 461 steps { 462 runGinkgo('verify-infra/restapi') 463 } 464 } 465 } 466 post { 467 success { 468 script { 469 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 470 dumpK8sCluster('verrazzano-testrun-after-reinstall-cluster-snapshot') 471 } 472 } 473 } 474 failure { 475 dumpK8sCluster('verrazzano-test-failure-after-reinstall-cluster-snapshot') 476 } 477 } 478 } 479 } 480 post { 481 always { 482 sh """ 483 # Copy the generated test reports to WORKSPACE to archive them 484 mkdir -p ${TEST_REPORT_DIR} 485 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 486 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 487 """ 488 archiveArtifacts artifacts: "**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/${TEST_REPORT}", allowEmptyArchive: true 489 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 490 } 491 failure { 492 script { 493 archiveArtifacts artifacts: '**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**', allowEmptyArchive: true 494 } 495 } 496 cleanup { 497 sh "VERRAZZANO_KUBECONFIG=${env.KUBECONFIG} TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=uninstall-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/delete_oke_cluster.sh || true" 498 deleteDir() 499 } 500 } 501 } 502 503 def runGinkgo(testSuitePath) { 504 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 505 sh """ 506 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 507 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 508 """ 509 } 510 } 511 512 def dumpK8sCluster(dumpDirectory) { 513 sh """ 514 515 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 516 """ 517 } 518 519 def dumpVerrazzanoSystemPods(logDirectory) { 520 sh """ 521 cd ${GO_REPO_PATH}/verrazzano/platform-operator 522 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-pods.log" 523 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 524 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-certs.log" 525 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 526 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-osd.log" 527 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 528 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-es-master.log" 529 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 530 """ 531 } 532 533 def dumpCattleSystemPods(logDirectory) { 534 sh """ 535 cd ${GO_REPO_PATH}/verrazzano/platform-operator 536 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/cattle-system-pods.log" 537 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 538 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/rancher.log" 539 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 540 """ 541 } 542 543 def dumpNginxIngressControllerLogs(logDirectory) { 544 sh """ 545 cd ${GO_REPO_PATH}/verrazzano/platform-operator 546 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/nginx-ingress-controller.log" 547 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 548 """ 549 } 550 551 def dumpVerrazzanoPlatformOperatorLogs(logDirectory) { 552 sh """ 553 ## dump out verrazzano-platform-operator logs 554 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory} 555 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory}/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 556 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory}/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 557 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 558 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 559 echo "------------------------------------------" 560 """ 561 } 562 563 def dumpVerrazzanoApplicationOperatorLogs(logDirectory) { 564 sh """ 565 ## dump out verrazzano-application-operator logs 566 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory} 567 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 568 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 569 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 570 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 571 echo "------------------------------------------" 572 """ 573 } 574 575 def dumpOamKubernetesRuntimeLogs(logDirectory) { 576 sh """ 577 ## dump out oam-kubernetes-runtime logs 578 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs/${logDirectory} 579 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/${logDirectory}/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 580 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 581 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 582 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 583 echo "------------------------------------------" 584 """ 585 } 586 587 def dumpVerrazzanoApiLogs(logDirectory) { 588 sh """ 589 cd ${GO_REPO_PATH}/verrazzano/platform-operator 590 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-authproxy.log" 591 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 592 """ 593 } 594 595 def listNamepacesAndPods(customMessage) { 596 sh """ 597 echo "Listing all the namespaces and pods the namespaces ${customMessage}." 598 kubectl get namespaces 599 kubectl get pods -A 600 echo "-----------------------------------------------------" 601 """ 602 } 603 604 def listHelmReleases(customMessage) { 605 sh """ 606 echo "Listing the releases across all namespaces ${customMessage}." 607 helm list -A 608 echo "-----------------------------------------------------" 609 """ 610 } 611 612 def getEffectiveDumpOnSuccess() { 613 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 614 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 615 effectiveValue = true 616 echo "Forcing dump on success based on global override setting" 617 } 618 return effectiveValue 619 } 620 621 def setDisplayName() { 622 echo "Start setDisplayName" 623 def causes = currentBuild.getBuildCauses() 624 echo "causes: " + causes.toString() 625 for (cause in causes) { 626 def causeString = cause.toString() 627 echo "current cause: " + causeString 628 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 629 echo "This job was caused by " + causeString 630 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 631 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 632 } else if (causeString.contains("verrazzano-flaky-tests")) { 633 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 634 } 635 } 636 } 637 echo "End setDisplayName" 638 }