github.com/verrazzano/verrazzano@v1.7.0/ci/multiplatform/Jenkinsfile (about) 1 // Copyright (c) 2020, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def SKIP_ACCEPTANCE_TESTS = false 6 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-frankfurt-1", "eu-zurich-1", "uk-london-1" ] 7 Collections.shuffle(availableRegions) 8 def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','') 9 def dns_zone_ocid = 'dummy' 10 def OKE_CLUSTER_PREFIX = "" 11 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 12 13 pipeline { 14 options { 15 skipDefaultCheckout true 16 } 17 18 agent { 19 docker { 20 image "${RUNNER_DOCKER_IMAGE}" 21 args "${RUNNER_DOCKER_ARGS}" 22 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 23 registryCredentialsId 'ocir-pull-and-push-account' 24 label "${agentLabel}" 25 } 26 } 27 28 parameters { 29 choice (description: 'Verrazzano Test Environment', name: 'TEST_ENV', 30 choices: ["kind", "magicdns_oke", "ocidns_oke"]) 31 choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION', 32 // 1st choice is the default value 33 choices: availableRegions ) 34 choice (description: 'OKE node pool configuration', name: 'OKE_NODE_POOL', 35 // 1st choice is the default value 36 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2" ]) 37 choice (description: 'Kubernetes Version for OKE Cluster', name: 'OKE_CLUSTER_VERSION', 38 // 1st choice is the default value 39 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 40 string defaultValue: 'dev', description: 'Verrazzano install profile name', name: "INSTALL_PROFILE", trim: true 41 choice (name: 'CRD_API_VERSION', 42 description: 'This is the API crd version.', 43 // 1st choice is the default value 44 choices: [ "v1beta1", "v1alpha1"]) 45 booleanParam (description: 'Whether to kick off acceptance test run at the end of this build', name: 'RUN_ACCEPTANCE_TESTS', defaultValue: true) 46 booleanParam (description: 'Whether to run example tests', name: 'RUN_EXAMPLE_TESTS', defaultValue: true) 47 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 48 } 49 50 environment { 51 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 52 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 53 DOCKER_PLATFORM_IMAGE_NAME = "${env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' ? env.DOCKER_PLATFORM_PUBLISH_IMAGE_NAME : env.DOCKER_PLATFORM_CI_IMAGE_NAME}" 54 DOCKER_OAM_CI_IMAGE_NAME = 'verrazzano-application-operator-jenkins' 55 DOCKER_OAM_PUBLISH_IMAGE_NAME = 'verrazzano-application-operator' 56 DOCKER_OAM_IMAGE_NAME = "${env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' ? env.DOCKER_OAM_PUBLISH_IMAGE_NAME : env.DOCKER_OAM_CI_IMAGE_NAME}" 57 CREATE_LATEST_TAG = "${env.BRANCH_NAME == 'master' ? '1' : '0'}" 58 GOPATH = '/home/opc/go' 59 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 60 DOCKER_CREDS = credentials('github-packages-credentials-rw') 61 DOCKER_EMAIL = credentials('github-packages-email') 62 DOCKER_REPO = 'ghcr.io' 63 DOCKER_NAMESPACE = 'verrazzano' 64 NETRC_FILE = credentials('netrc') 65 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 66 67 CLUSTER_NAME = 'verrazzano' 68 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 69 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 70 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 71 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 72 73 // Environment variable for Verrazzano CLI executable 74 VZ_COMMAND="${GO_REPO_PATH}/vz" 75 76 OCR_CREDS = credentials('ocr-pull-and-push-account') 77 OCR_REPO = 'container-registry.oracle.com' 78 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 79 INSTALL_CONFIG_FILE_KIND = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml" 80 81 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test 82 DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test 83 84 TEST_ENV = "${params.TEST_ENV}" 85 INSTALL_PROFILE = "${params.INSTALL_PROFILE}" 86 87 TF_VAR_tenancy_id = credentials('oci-tenancy') 88 TF_VAR_user_id = credentials('oci-user-ocid') 89 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 90 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 91 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 92 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 93 TF_VAR_api_private_key_path = credentials('oci-api-key') 94 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 95 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 96 97 OCI_CLI_TENANCY = credentials('oci-tenancy') 98 OCI_CLI_USER = credentials('oci-user-ocid') 99 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 100 OCI_CLI_KEY_FILE = credentials('oci-api-key') 101 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 102 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 103 INSTALL_CONFIG_FILE_OCIDNS = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-ocidns.yaml" 104 INSTALL_CONFIG_FILE_NIPIO = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-nipio.yaml" 105 OCI_DNS_ZONE_SUFFIX=credentials('oci-dns-zone') 106 OCI_DNS_ZONE_NAME="z${zoneId}." + "${OCI_DNS_ZONE_SUFFIX}" 107 VZ_ENVIRONMENT_NAME = "${params.TEST_ENV == 'ocidns_oke' ? 'b' + env.BUILD_NUMBER : 'default'}" 108 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 109 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 110 111 // used to emit metrics 112 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 113 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 114 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 115 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 116 } 117 118 stages { 119 stage('Clean workspace and checkout') { 120 environment { 121 OCI_CLI_AUTH="instance_principal" 122 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 123 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 124 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 125 } 126 steps { 127 sh """ 128 echo "${NODE_LABELS}" 129 """ 130 131 script { 132 checkout scm 133 } 134 sh """ 135 cp -f "${NETRC_FILE}" $HOME/.netrc 136 chmod 600 $HOME/.netrc 137 """ 138 139 script { 140 try { 141 sh """ 142 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 143 """ 144 } catch(error) { 145 echo "docker login failed, retrying after sleep" 146 retry(4) { 147 sleep(30) 148 sh """ 149 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 150 """ 151 } 152 } 153 } 154 sh """ 155 rm -rf ${GO_REPO_PATH}/verrazzano 156 mkdir -p ${GO_REPO_PATH}/verrazzano 157 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 158 """ 159 160 script { 161 def props = readProperties file: '.verrazzano-development-version' 162 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 163 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 164 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 165 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 166 167 if (params.TEST_ENV != "kind") { 168 // derive the prefix for the OKE cluster 169 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${GO_REPO_PATH}/verrazzano/ci/scripts/derive_oke_cluster_name.sh").trim() 170 } 171 } 172 script { 173 sh """ 174 echo "Downloading VZ CLI from object storage" 175 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 176 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 177 ${GO_REPO_PATH}/vz version 178 """ 179 } 180 } 181 } 182 183 stage('Acceptance Tests') { 184 185 stages { 186 187 stage('Prepare AT environment') { 188 stages { 189 stage('Create Cluster') { 190 steps { 191 script { 192 sh """ 193 echo "tests will execute" > ${TESTS_EXECUTED_FILE} 194 if [ "${env.TEST_ENV}" == "kind" ] 195 then 196 echo "Create Kind cluster" 197 cd ${GO_REPO_PATH}/verrazzano/platform-operator 198 make create-cluster 199 200 echo "Install metallb" 201 cd ${GO_REPO_PATH}/verrazzano 202 ./tests/e2e/config/scripts/install-metallb.sh 203 else 204 echo "Create OKE cluster" 205 cd ${GO_REPO_PATH}/verrazzano 206 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} ./tests/e2e/config/scripts/create_oke_cluster.sh 207 fi 208 """ 209 } 210 } 211 } 212 213 stage('Configure Verrazzano prerequisites') { 214 environment { 215 OCI_CLI_AUTH="instance_principal" 216 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 217 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 218 } 219 parallel { 220 stage ('Install platform operator') { 221 steps { 222 script { 223 sh """ 224 echo "Create Image Pull Secrets" 225 cd ${GO_REPO_PATH}/verrazzano 226 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 227 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 228 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 229 230 # Install the verrazzano-platform-operator 231 echo "Install Platform Operator" 232 if [ "NONE" = "${VERRAZZANO_OPERATOR_IMAGE}" ]; then 233 echo "Using operator.yaml from object storage" 234 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 235 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 236 else 237 echo "Generating operator.yaml based on image name provided: ${VERRAZZANO_OPERATOR_IMAGE}" 238 env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/acceptance-test-operator.yaml 239 fi 240 kubectl apply -f ${WORKSPACE}/acceptance-test-operator.yaml 241 242 243 # make sure ns exists 244 ./tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 245 246 # create secret in verrazzano-install ns 247 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 248 """ 249 } 250 } 251 } 252 253 stage("Configure OCI DNS install prerequisites") { 254 when { expression { return params.TEST_ENV == 'ocidns_oke' } } 255 stages { 256 stage('Create dns zone') { 257 steps { 258 script { 259 dns_zone_ocid = sh(script: "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/oci_dns_ops.sh -o create -c ${TF_VAR_compartment_id} -s z${zoneId}", returnStdout: true) 260 } 261 } 262 } 263 stage('Configure Install profile') { 264 environment { 265 OCI_DNS_COMPARTMENT_OCID = credentials('oci-dns-compartment') 266 OCI_PRIVATE_KEY_FILE = credentials('oci-api-key') 267 OCI_DNS_ZONE_OCID = "${dns_zone_ocid}" 268 } 269 steps { 270 script { 271 sh """ 272 export PATH=${HOME}/go/bin:${PATH} 273 cd ${GO_REPO_PATH}/verrazzano 274 ./tests/e2e/config/scripts/process_oci_dns_install_yaml.sh $INSTALL_CONFIG_FILE_OCIDNS 275 ./tests/e2e/config/scripts/create-test-oci-config-secret.sh 276 """ 277 } 278 } 279 } 280 } 281 } 282 283 stage("Configure Magic DNS install prerequisites") { 284 when { expression { return params.TEST_ENV == 'magicdns_oke' } } 285 steps { 286 script { 287 sh """ 288 export PATH=${HOME}/go/bin:${PATH} 289 cd ${GO_REPO_PATH}/verrazzano 290 ./tests/e2e/config/scripts/process_nipio_install_yaml.sh $INSTALL_CONFIG_FILE_NIPIO 291 """ 292 } 293 } 294 } 295 296 stage("Configure Kind install prerequisites") { 297 when { expression { return params.TEST_ENV == 'kind' } } 298 steps { 299 script { 300 sh """ 301 export PATH=${HOME}/go/bin:${PATH} 302 cd ${GO_REPO_PATH}/verrazzano 303 ./tests/e2e/config/scripts/process_kind_install_yaml.sh ${INSTALL_CONFIG_FILE_KIND} 304 """ 305 } 306 } 307 } 308 } 309 } 310 311 stage ('Install Verrazzano') { 312 steps { 313 script { 314 sh """ 315 echo "Waiting for Operator to be ready" 316 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 317 echo "Installing Verrazzano on ${TEST_ENV}" 318 # apply config to create cluster 319 if [ "${TEST_ENV}" == "magicdns_oke" ]; then 320 kubectl apply -f ${INSTALL_CONFIG_FILE_NIPIO} 321 elif [ "${TEST_ENV}" == "ocidns_oke" ]; then 322 kubectl apply -f ${INSTALL_CONFIG_FILE_OCIDNS} 323 elif [ "${TEST_ENV}" == "kind" ]; then 324 kubectl apply -f ${INSTALL_CONFIG_FILE_KIND} 325 fi 326 327 cd ${GO_REPO_PATH}/verrazzano 328 ./tests/e2e/config/scripts/wait-for-verrazzano-install.sh 329 330 """ 331 } 332 } 333 } 334 } 335 post { 336 always { 337 sh """ 338 ## dump out install logs 339 mkdir -p ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs 340 kubectl -n verrazzano-install logs --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-install.log --tail -1 341 kubectl -n verrazzano-install describe pod --selector=job-name=verrazzano-install-my-verrazzano > ${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-install-job-pod.out 342 echo "Verrazzano Installation logs dumped to verrazzano-install.log" 343 echo "Verrazzano Install pod description dumped to verrazzano-install-job-pod.out" 344 echo "------------------------------------------" 345 """ 346 } 347 } 348 } 349 350 stage('Run Acceptance Tests') { 351 352 parallel { 353 stage('verify-install') { 354 steps { 355 runGinkgoRandomize('verify-install') 356 } 357 } 358 stage('verify-infra restapi') { 359 steps { 360 runGinkgoRandomize('verify-infra/restapi') 361 } 362 } 363 stage('verify-infra oam') { 364 steps { 365 runGinkgoRandomize('verify-infra/oam') 366 } 367 } 368 stage('verify-infra vmi') { 369 steps { 370 runGinkgoRandomize('verify-infra/vmi') 371 } 372 } 373 // yes i know this is ugly - working on cleaning it up 374 stage('examples todo') { 375 when { 376 expression {params.RUN_EXAMPLE_TESTS == true} 377 } 378 steps { 379 runGinkgo('examples/todo-list') 380 } 381 } 382 stage('examples socks') { 383 when { 384 expression {params.RUN_EXAMPLE_TESTS == true} 385 } 386 steps { 387 runGinkgo('examples/sock-shop') 388 } 389 } 390 stage('examples springboot') { 391 when { 392 expression {params.RUN_EXAMPLE_TESTS == true} 393 } 394 steps { 395 runGinkgo('examples/springboot-app') 396 } 397 } 398 stage('examples helidon') { 399 when { 400 expression {params.RUN_EXAMPLE_TESTS == true} 401 } 402 steps { 403 runGinkgo('examples/hello-helidon') 404 } 405 } 406 stage('examples bobs books') { 407 when { 408 expression {params.RUN_EXAMPLE_TESTS == true} 409 } 410 steps { 411 runGinkgo('examples/bobs-books') 412 } 413 } 414 } 415 post { 416 always { 417 archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true 418 junit testResults: '**/*test-result.xml', allowEmptyResults: true 419 } 420 } 421 } 422 } 423 424 post { 425 failure { 426 script { 427 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 428 dumpK8sCluster('new-acceptance-tests-cluster-snapshot') 429 } 430 } 431 } 432 success { 433 script { 434 if (params.DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 435 dumpK8sCluster('new-acceptance-tests-cluster-snapshot') 436 } 437 } 438 } 439 } 440 } 441 } 442 443 post { 444 always { 445 script { 446 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 447 dumpVerrazzanoSystemPods() 448 dumpCattleSystemPods() 449 dumpNginxIngressControllerLogs() 450 dumpVerrazzanoPlatformOperatorLogs() 451 dumpVerrazzanoApplicationOperatorLogs() 452 dumpOamKubernetesRuntimeLogs() 453 dumpVerrazzanoApiLogs() 454 } 455 } 456 archiveArtifacts artifacts: '**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**', allowEmptyArchive: true 457 junit testResults: '**/*test-result.xml', allowEmptyResults: true 458 459 sh """ 460 if [ "${TEST_ENV}" == "ocidns_oke" ]; then 461 cd ${GO_REPO_PATH}/verrazzano 462 ./tests/e2e/config/scripts/oci_dns_ops.sh -o delete -s z${zoneId} || echo "Failed to delete DNS zone z${zoneId}" 463 fi 464 if [ "${TEST_ENV}" == "kind" ]; then 465 cd ${GO_REPO_PATH}/verrazzano/platform-operator 466 make delete-cluster 467 else 468 cd ${GO_REPO_PATH}/verrazzano 469 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} ./tests/e2e/config/scripts/delete_oke_cluster.sh || true 470 fi 471 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 472 echo "Failures seen during dumping of artifacts, treat post as failed" 473 exit 1 474 fi 475 """ 476 deleteDir() 477 } 478 } 479 } 480 481 def runGinkgoRandomize(testSuitePath) { 482 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 483 sh """ 484 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 485 ginkgo -p --randomize-all -v --keep-going --no-color ${testSuitePath}/... 486 """ 487 } 488 } 489 490 def runGinkgo(testSuitePath) { 491 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 492 sh """ 493 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 494 ginkgo -v --keep=going --no-color ${testSuitePath}/... 495 """ 496 } 497 } 498 499 def dumpK8sCluster(dumpDirectory) { 500 sh """ 501 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 502 """ 503 } 504 505 def dumpVerrazzanoSystemPods() { 506 sh """ 507 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log" 508 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 509 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log" 510 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 511 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log" 512 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 513 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log" 514 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 515 """ 516 } 517 518 def dumpCattleSystemPods() { 519 sh """ 520 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/cattle-system-pods.log" 521 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 522 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/rancher.log" 523 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 524 """ 525 } 526 527 def dumpNginxIngressControllerLogs() { 528 sh """ 529 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log" 530 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 531 """ 532 } 533 534 def dumpVerrazzanoPlatformOperatorLogs() { 535 sh """ 536 ## dump out verrazzano-platform-operator logs 537 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 538 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 539 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 540 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 541 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 542 echo "------------------------------------------" 543 """ 544 } 545 546 def dumpVerrazzanoApplicationOperatorLogs() { 547 sh """ 548 ## dump out verrazzano-application-operator logs 549 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 550 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 551 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 552 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 553 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 554 echo "------------------------------------------" 555 """ 556 } 557 558 def dumpOamKubernetesRuntimeLogs() { 559 sh """ 560 ## dump out oam-kubernetes-runtime logs 561 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 562 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 563 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 564 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 565 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 566 echo "------------------------------------------" 567 """ 568 } 569 570 def dumpVerrazzanoApiLogs() { 571 sh """ 572 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log" 573 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 574 """ 575 }