github.com/verrazzano/verrazzano@v1.7.1/ci/uninstall/Jenkinsfile (about) 1 // Copyright (c) 2020, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 6 7 // pulling "ap-*" from the test regions given discovery of image pull issues 8 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ] 9 def OKE_CLUSTER_PREFIX = "" 10 Collections.shuffle(availableRegions) 11 12 pipeline { 13 options { 14 skipDefaultCheckout true 15 } 16 17 agent { 18 docker { 19 image "${RUNNER_DOCKER_IMAGE}" 20 args "${RUNNER_DOCKER_ARGS}" 21 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 22 registryCredentialsId 'ocir-pull-and-push-account' 23 } 24 } 25 26 parameters { 27 string (name: 'VERRAZZANO_BRANCH', 28 defaultValue: 'master', 29 description: 'Branch to clone and checkout the Verrazzano repo', 30 trim: true) 31 choice (name: 'OKE_NODE_POOL', 32 description: 'OKE node pool configuration', 33 // 1st choice is the default value 34 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2", "VM.Standard.E2.4" ]) 35 choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION', 36 // 1st choice is the default value 37 choices: availableRegions ) 38 string (name: 'GIT_COMMIT_TO_USE', 39 defaultValue: 'NONE', 40 description: 'This is the full git commit hash from the source build to be used for all jobs', 41 trim: true) 42 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 43 defaultValue: 'NONE', 44 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator', 45 trim: true) 46 choice (name: 'OKE_CLUSTER_VERSION', 47 description: 'Kubernetes Version for OKE Cluster', 48 // 1st choice is the default value 49 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 50 choice (name: 'CRD_API_VERSION', 51 description: 'This is the API crd version.', 52 // 1st choice is the default value 53 choices: [ "v1beta1", "v1alpha1"]) 54 booleanParam (name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', 55 defaultValue: false, 56 description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)') 57 string (name: 'TAGGED_TESTS', 58 defaultValue: '', 59 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 60 trim: true) 61 string (name: 'INCLUDED_TESTS', 62 defaultValue: '.*', 63 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 64 trim: true) 65 string (name: 'EXCLUDED_TESTS', 66 defaultValue: '_excluded_test', 67 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 68 trim: true) 69 } 70 71 environment { 72 OCR_CREDS = credentials('ocr-pull-and-push-account') 73 NETRC_FILE = credentials('netrc') 74 GOPATH = '/home/opc/go' 75 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 76 DOCKER_CREDS = credentials('github-packages-credentials-rw') 77 DOCKER_EMAIL = credentials('github-packages-email') 78 DOCKER_REPO = 'ghcr.io' 79 80 OCR_REPO = 'container-registry.oracle.com' 81 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 82 83 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 84 TF_VAR_tenancy_id = credentials('oci-tenancy') 85 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 86 TF_VAR_user_id = credentials('oci-user-ocid') 87 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 88 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 89 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 90 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 91 TF_VAR_api_private_key_path = credentials('oci-api-key') 92 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 93 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 94 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 95 96 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 97 OCI_CLI_TENANCY = credentials('oci-tenancy') 98 OCI_CLI_USER = credentials('oci-user-ocid') 99 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 100 OCI_CLI_KEY_FILE = credentials('oci-api-key') 101 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 102 DISABLE_SPINNER=1 103 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 104 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 105 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 106 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 107 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 108 KUBECONFIG = "${WORKSPACE}/oke_kubeconfig" 109 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 110 111 // Environment variable for Verrazzano CLI executable 112 VZ_COMMAND="${GO_REPO_PATH}/vz" 113 INSTALL_PROFILE = "prod" 114 VZ_ENVIRONMENT_NAME = "default" 115 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 116 LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test" 117 UNINSTALL_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/uninstall-test" 118 INSTALL_CONFIG_FILE = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-default.yaml" 119 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') 120 DATABASE_PSW = credentials('todo-mysql-password') 121 122 // used to emit metrics 123 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 124 TEST_ENV_LABEL = "magicdns_oke" 125 TEST_ENV = "OKE" 126 127 // used to generate Ginkgo test reports 128 TEST_REPORT = "test-report.xml" 129 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 130 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 131 } 132 133 stages { 134 stage('Clean workspace and checkout') { 135 environment { 136 OCI_CLI_AUTH="instance_principal" 137 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 138 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 139 } 140 steps { 141 sh """ 142 echo "${NODE_LABELS}" 143 """ 144 script { 145 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 146 if (params.GIT_COMMIT_TO_USE == "NONE") { 147 echo "Specific GIT commit was not specified, use current head" 148 def scmInfo = checkout scm 149 env.GIT_COMMIT = scmInfo.GIT_COMMIT 150 env.GIT_BRANCH = scmInfo.GIT_BRANCH 151 } else { 152 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 153 def scmInfo = checkout([ 154 $class: 'GitSCM', 155 branches: [[name: params.GIT_COMMIT_TO_USE]], 156 doGenerateSubmoduleConfigurations: false, 157 extensions: [], 158 submoduleCfg: [], 159 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 160 env.GIT_COMMIT = scmInfo.GIT_COMMIT 161 env.GIT_BRANCH = scmInfo.GIT_BRANCH 162 // If the commit we were handed is not what the SCM says we are using, fail 163 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 164 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 165 exit 1 166 } 167 } 168 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 169 } 170 sh """ 171 cp -f "${NETRC_FILE}" $HOME/.netrc 172 chmod 600 $HOME/.netrc 173 """ 174 175 script { 176 try { 177 sh """ 178 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 179 """ 180 } catch(error) { 181 echo "docker login failed, retrying after sleep" 182 retry(4) { 183 sleep(30) 184 sh """ 185 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 186 """ 187 } 188 } 189 } 190 191 sh """ 192 rm -rf ${GO_REPO_PATH}/verrazzano 193 mkdir -p ${GO_REPO_PATH}/verrazzano 194 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 195 cd ${GO_REPO_PATH}/verrazzano 196 git config --global credential.helper "!f() { echo username=\\$DOCKER_CREDS_USR; echo password=\\$DOCKER_CREDS_PSW; }; f" 197 git config --global user.name $DOCKER_CREDS_USR 198 git config --global user.email "${DOCKER_EMAIL}" 199 git checkout -b ${env.BRANCH_NAME} 200 """ 201 202 script { 203 def props = readProperties file: '.verrazzano-development-version' 204 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 205 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 206 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 207 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 208 // update the description with some meaningful info 209 setDisplayName() 210 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 211 212 // derive the prefix for the OKE cluster 213 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 214 } 215 script { 216 sh """ 217 echo "Downloading VZ CLI from object storage" 218 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 219 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 220 ${GO_REPO_PATH}/vz version 221 """ 222 } 223 } 224 } 225 226 stage("Create Cluster") { 227 steps { 228 sh "TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=uninstall-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/create_oke_cluster.sh" 229 } 230 } 231 232 stage('Install Verrazzano') { 233 environment { 234 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 235 } 236 steps { 237 script { 238 listNamepacesAndPods('before installing Verrazzano') 239 } 240 sh """ 241 # Create image pull secret for Verrazzano docker images 242 cd ${GO_REPO_PATH}/verrazzano 243 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 244 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 245 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 246 247 echo "Prepare Verrazzano install" 248 cd ${GO_REPO_PATH}/verrazzano 249 if [ "NONE" = "${VERRAZZANO_OPERATOR_IMAGE}" ]; then 250 echo "Using operator.yaml from object storage" 251 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 252 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 253 else 254 echo "Generating operator.yaml based on image name provided: ${VERRAZZANO_OPERATOR_IMAGE}" 255 env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/acceptance-test-operator.yaml 256 fi 257 258 # make sure ns exists 259 kubectl create ns verrazzano-install 260 ./tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 261 262 # create secret in verrazzano-install ns 263 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 264 265 ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/pre-install-resources false 266 267 ./tests/e2e/config/scripts/process_nipio_install_yaml.sh ${INSTALL_CONFIG_FILE} 268 yq -i eval '.spec.components.prometheusAdapter.enabled = true' ${INSTALL_CONFIG_FILE} 269 yq -i eval '.spec.components.kubeStateMetrics.enabled = true' ${INSTALL_CONFIG_FILE} 270 yq -i eval '.spec.components.prometheusPushgateway.enabled = true' ${INSTALL_CONFIG_FILE} 271 272 echo "Installing Verrazzano on OKE" 273 ${GO_REPO_PATH}/vz install --filename ${INSTALL_CONFIG_FILE} --manifests ${WORKSPACE}/acceptance-test-operator.yaml --timeout 45m 274 275 ${TEST_SCRIPTS_DIR}/common-test-setup-script.sh "${GO_REPO_PATH}" "${TEST_CONFIG_FILE}" "${env.DOCKER_REPO}" "${KUBECONFIG}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" "default" "${env.OCR_REPO}" 276 ${TEST_SCRIPTS_DIR}/get_ingress_ip.sh ${TEST_CONFIG_FILE} 277 echo "----------Test config file:-------------" 278 cat ${TEST_CONFIG_FILE} 279 echo "----------------------------------------" 280 """ 281 } 282 post { 283 always { 284 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 285 dumpVerrazzanoSystemPods('install') 286 dumpCattleSystemPods('install') 287 dumpNginxIngressControllerLogs('install') 288 dumpVerrazzanoPlatformOperatorLogs('install') 289 dumpVerrazzanoApplicationOperatorLogs('install') 290 dumpOamKubernetesRuntimeLogs('install') 291 dumpVerrazzanoApiLogs('install') 292 listNamepacesAndPods('after Verrazzano install') 293 listHelmReleases('after Verrazzano install') 294 } 295 success { 296 script { 297 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 298 dumpK8sCluster('verrazzano-install-cluster-snapshot') 299 } 300 } 301 } 302 failure { 303 script { 304 dumpK8sCluster('verrazzano-install-failure-cluster-snapshot') 305 sh """ 306 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs 307 ${LOOPING_TEST_SCRIPTS_DIR}/dump_resources.sh > ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs/resources.log 308 """ 309 } 310 } 311 } 312 } 313 314 stage('Run Test') { 315 stages { 316 stage('verify-install') { 317 steps { 318 runGinkgo('verify-install') 319 } 320 } 321 stage('verify-infra restapi') { 322 steps { 323 runGinkgo('verify-infra/restapi') 324 } 325 } 326 } 327 post { 328 success { 329 script { 330 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 331 dumpK8sCluster('verrazzano-testrun-after-install-cluster-snapshot') 332 } 333 } 334 } 335 failure { 336 dumpK8sCluster('verrazzano-test-failure-cluster-snapshot') 337 338 } 339 } 340 } 341 342 stage('Uninstall Verrazzano') { 343 steps { 344 sh """ 345 ${GO_REPO_PATH}/vz uninstall -y --timeout 60m 346 """ 347 } 348 post { 349 always { 350 sh """ 351 ## dump out uninstall logs 352 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/uninstall/build/logs 353 echo "Listing all pods in all namespaces after uninstall" 354 kubectl get pods --all-namespaces 355 echo "-----------------------------------------------------" 356 """ 357 listNamepacesAndPods('after Verrazzano uninstall') 358 listHelmReleases('after Verrazzano uninstall') 359 } 360 success { 361 script { 362 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 363 dumpK8sCluster('verrazzano-uninstall-cluster-snapshot') 364 } 365 } 366 } 367 failure { 368 dumpK8sCluster('verrazzano-uninstall-failure-cluster-snapshot') 369 } 370 } 371 } 372 373 stage("Verify Uninstall") { 374 steps { 375 retry(6) { 376 sleep time: 300, unit: 'SECONDS' 377 sh """ 378 ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/post-uninstall-resources false 379 ${LOOPING_TEST_SCRIPTS_DIR}/verify_uninstall.sh ${WORKSPACE}/verrazzano/build/resources 380 """ 381 } 382 } 383 post { 384 success { 385 script { 386 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 387 dumpK8sCluster('verify-uninstall-cluster-snapshot') 388 } 389 } 390 } 391 failure { 392 dumpK8sCluster('verify-uninstall-cluster-snapshot') 393 } 394 } 395 } 396 397 stage("Reinstall Verrazzano") { 398 steps { 399 sh """ 400 # sleep for a period to ensure async deletion of Verrazzano components from uninstall above has completed 401 sleep 90 402 # make sure ns exists 403 kubectl create ns verrazzano-install 404 ./tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 405 406 # create secret in verrazzano-install ns 407 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 408 409 410 ${GO_REPO_PATH}/vz install --filename ${INSTALL_CONFIG_FILE} --manifests ${WORKSPACE}/acceptance-test-operator.yaml --timeout 45m 411 412 ${TEST_SCRIPTS_DIR}/common-test-setup-script.sh "${GO_REPO_PATH}" "${TEST_CONFIG_FILE}" "${env.DOCKER_REPO}" "${KUBECONFIG}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" "default" 413 ${TEST_SCRIPTS_DIR}/get_ingress_ip.sh ${TEST_CONFIG_FILE} 414 echo "----------Test config file:-------------" 415 cat ${TEST_CONFIG_FILE} 416 echo "----------------------------------------" 417 """ 418 } 419 post { 420 always { 421 dumpVerrazzanoSystemPods('reinstall') 422 dumpCattleSystemPods('reinstall') 423 dumpNginxIngressControllerLogs('reinstall') 424 dumpVerrazzanoPlatformOperatorLogs('reinstall') 425 dumpVerrazzanoApplicationOperatorLogs('reinstall') 426 dumpOamKubernetesRuntimeLogs('reinstall') 427 dumpVerrazzanoApiLogs('reinstall') 428 listNamepacesAndPods('after reinstalling Verrazzano') 429 listHelmReleases('after reinstalling Verrazzano') 430 } 431 success { 432 script { 433 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 434 dumpK8sCluster('verrazzano-reinstall-cluster-snapshot') 435 } 436 } 437 } 438 failure { 439 dumpK8sCluster('verrazzano-reinstall-failure-cluster-snapshot') 440 sh """ 441 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/reinstall/build/logs 442 ${LOOPING_TEST_SCRIPTS_DIR}/dump_resources.sh > ${WORKSPACE}/verrazzano-platform-operator/scripts/reinstall/build/logs/resources.log 443 """ 444 } 445 } 446 } 447 448 stage('Rerun Test') { 449 stages { 450 stage('verify-install') { 451 steps { 452 runGinkgo('verify-install') 453 } 454 } 455 stage('verify-infra restapi') { 456 steps { 457 runGinkgo('verify-infra/restapi') 458 } 459 } 460 } 461 post { 462 success { 463 script { 464 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 465 dumpK8sCluster('verrazzano-testrun-after-reinstall-cluster-snapshot') 466 } 467 } 468 } 469 failure { 470 dumpK8sCluster('verrazzano-test-failure-after-reinstall-cluster-snapshot') 471 } 472 } 473 } 474 } 475 post { 476 always { 477 sh """ 478 # Copy the generated test reports to WORKSPACE to archive them 479 mkdir -p ${TEST_REPORT_DIR} 480 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 481 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 482 """ 483 archiveArtifacts artifacts: "**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/${TEST_REPORT}", allowEmptyArchive: true 484 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 485 } 486 failure { 487 script { 488 archiveArtifacts artifacts: '**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**', allowEmptyArchive: true 489 } 490 } 491 cleanup { 492 sh "VERRAZZANO_KUBECONFIG=${env.KUBECONFIG} TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=uninstall-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/delete_oke_cluster.sh || true" 493 deleteDir() 494 } 495 } 496 } 497 498 def runGinkgo(testSuitePath) { 499 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 500 sh """ 501 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 502 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 503 """ 504 } 505 } 506 507 def dumpK8sCluster(dumpDirectory) { 508 sh """ 509 510 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 511 """ 512 } 513 514 def dumpVerrazzanoSystemPods(logDirectory) { 515 sh """ 516 cd ${GO_REPO_PATH}/verrazzano/platform-operator 517 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-pods.log" 518 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 519 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-certs.log" 520 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 521 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-osd.log" 522 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 523 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-system-es-master.log" 524 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 525 """ 526 } 527 528 def dumpCattleSystemPods(logDirectory) { 529 sh """ 530 cd ${GO_REPO_PATH}/verrazzano/platform-operator 531 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/cattle-system-pods.log" 532 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 533 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/rancher.log" 534 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 535 """ 536 } 537 538 def dumpNginxIngressControllerLogs(logDirectory) { 539 sh """ 540 cd ${GO_REPO_PATH}/verrazzano/platform-operator 541 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/nginx-ingress-controller.log" 542 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 543 """ 544 } 545 546 def dumpVerrazzanoPlatformOperatorLogs(logDirectory) { 547 sh """ 548 ## dump out verrazzano-platform-operator logs 549 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory} 550 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory}/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 551 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/${logDirectory}/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 552 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 553 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 554 echo "------------------------------------------" 555 """ 556 } 557 558 def dumpVerrazzanoApplicationOperatorLogs(logDirectory) { 559 sh """ 560 ## dump out verrazzano-application-operator logs 561 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory} 562 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 563 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 564 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 565 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 566 echo "------------------------------------------" 567 """ 568 } 569 570 def dumpOamKubernetesRuntimeLogs(logDirectory) { 571 sh """ 572 ## dump out oam-kubernetes-runtime logs 573 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs/${logDirectory} 574 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/${logDirectory}/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 575 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/${logDirectory}/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 576 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 577 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 578 echo "------------------------------------------" 579 """ 580 } 581 582 def dumpVerrazzanoApiLogs(logDirectory) { 583 sh """ 584 cd ${GO_REPO_PATH}/verrazzano/platform-operator 585 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano-platform-operator/scripts/${logDirectory}/build/logs/verrazzano-authproxy.log" 586 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 587 """ 588 } 589 590 def listNamepacesAndPods(customMessage) { 591 sh """ 592 echo "Listing all the namespaces and pods the namespaces ${customMessage}." 593 kubectl get namespaces 594 kubectl get pods -A 595 echo "-----------------------------------------------------" 596 """ 597 } 598 599 def listHelmReleases(customMessage) { 600 sh """ 601 echo "Listing the releases across all namespaces ${customMessage}." 602 helm list -A 603 echo "-----------------------------------------------------" 604 """ 605 } 606 607 def getEffectiveDumpOnSuccess() { 608 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 609 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 610 effectiveValue = true 611 echo "Forcing dump on success based on global override setting" 612 } 613 return effectiveValue 614 } 615 616 def setDisplayName() { 617 echo "Start setDisplayName" 618 def causes = currentBuild.getBuildCauses() 619 echo "causes: " + causes.toString() 620 for (cause in causes) { 621 def causeString = cause.toString() 622 echo "current cause: " + causeString 623 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 624 echo "This job was caused by " + causeString 625 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 626 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 627 } else if (causeString.contains("verrazzano-flaky-tests")) { 628 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 629 } 630 } 631 } 632 echo "End setDisplayName" 633 }