github.com/verrazzano/verrazzano@v1.7.0/ci/vz-analyze/Jenkinsfile (about) 1 // Copyright (c) 2020, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 7 8 pipeline { 9 options { 10 timeout(time: 1, unit: 'HOURS') 11 skipDefaultCheckout true 12 timestamps () 13 } 14 15 agent { 16 docker { 17 image "${RUNNER_DOCKER_IMAGE}" 18 args "${RUNNER_DOCKER_ARGS}" 19 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 20 registryCredentialsId 'ocir-pull-and-push-account' 21 label "${agentLabel}" 22 } 23 } 24 25 parameters { 26 choice (name: 'KUBERNETES_CLUSTER_VERSION', 27 description: 'Kubernetes Version for KinD Cluster', 28 // 1st choice is the default value 29 choices: [ "1.27", "1.26", "1.25", "1.24" ]) 30 string (name: 'GIT_COMMIT_TO_USE', 31 defaultValue: 'NONE', 32 description: 'This is the full git commit hash from the source build to be used for all jobs', 33 trim: true) 34 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 35 defaultValue: 'NONE', 36 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator', 37 trim: true) 38 choice (name: 'WILDCARD_DNS_DOMAIN', 39 description: 'This is the wildcard DNS domain', 40 // 1st choice is the default value 41 choices: [ "nip.io", "sslip.io"]) 42 choice (name: 'CRD_API_VERSION', 43 description: 'This is the API crd version.', 44 // 1st choice is the default value 45 choices: [ "v1beta1", "v1alpha1"]) 46 booleanParam (description: 'Whether to install Verrazzano using examples/ha/ha.yaml and 3 kind nodes (defaults to false)', name: 'INSTALL_HA', defaultValue: false) 47 booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 48 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 49 booleanParam (description: 'Whether to use a database for Grafana persistence', name: 'USE_DB_FOR_GRAFANA', defaultValue: false) 50 string (name: 'CONSOLE_REPO_BRANCH', 51 defaultValue: '', 52 description: 'The branch to check out after cloning the console repository.', 53 trim: true) 54 booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true) 55 string (name: 'TAGGED_TESTS', 56 defaultValue: '', 57 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 58 trim: true) 59 string (name: 'INCLUDED_TESTS', 60 defaultValue: '.*', 61 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 62 trim: true) 63 string (name: 'EXCLUDED_TESTS', 64 defaultValue: '_excluded_test', 65 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 66 trim: true) 67 booleanParam (description: 'Whether to run JWT tests', name: 'ENABLE_JWT_TESTING', defaultValue: false) 68 booleanParam (description: 'Whether to run slow tests', name: 'RUN_SLOW_TESTS', defaultValue: false) 69 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 70 } 71 72 environment { 73 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 74 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 75 GOPATH = '/home/opc/go' 76 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 77 DOCKER_CREDS = credentials('github-packages-credentials-rw') 78 DOCKER_EMAIL = credentials('github-packages-email') 79 DOCKER_REPO = 'ghcr.io' 80 DOCKER_NAMESPACE = 'verrazzano' 81 NETRC_FILE = credentials('netrc') 82 CLUSTER_NAME = 'verrazzano' 83 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 84 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 85 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 86 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 87 OCR_CREDS = credentials('ocr-pull-and-push-account') 88 OCR_REPO = 'container-registry.oracle.com' 89 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 90 INSTALL_CONFIG_FILE_KIND_TESTS = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml" 91 INSTALL_CONFIG_FILE_KIND_HA = "./examples/ha/ha.yaml" 92 INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/install-verrazzano.yaml" 93 INSTALL_PROFILE = "${params.INSTALL_HA == true ? "prod" : "dev"}" 94 KIND_NODE_COUNT = "${params.INSTALL_HA == true ? "3" : "1"}" 95 VZ_ENVIRONMENT_NAME = "default" 96 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 97 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 98 99 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test 100 DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test 101 102 // Environment variables required to capture cluster snapshot and bug report on test failure 103 DUMP_KUBECONFIG="${KUBECONFIG}" 104 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 105 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 106 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 107 108 // Environment variable for Verrazzano CLI executable 109 VZ_COMMAND="${GO_REPO_PATH}/vz" 110 111 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 112 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 113 114 // used for console artifact capture on failure 115 JENKINS_READ = credentials('jenkins-auditor') 116 OCI_CLI_AUTH="instance_principal" 117 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 118 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 119 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 120 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 121 122 // used to emit metrics from Ginkgo suites 123 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 124 TEST_ENV_LABEL = "kind" 125 TEST_ENV = "KIND" 126 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 127 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 128 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 129 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 130 131 // used to generate Ginkgo test reports 132 TEST_REPORT = "test-report.xml" 133 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 134 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 135 } 136 137 stages { 138 stage('Clean workspace and checkout') { 139 steps { 140 sh """ 141 echo "${NODE_LABELS}" 142 """ 143 144 script { 145 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 146 if (params.GIT_COMMIT_TO_USE == "NONE") { 147 echo "Specific GIT commit was not specified, use current head" 148 def scmInfo = checkout scm 149 env.GIT_COMMIT = scmInfo.GIT_COMMIT 150 env.GIT_BRANCH = scmInfo.GIT_BRANCH 151 } else { 152 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 153 def scmInfo = checkout([ 154 $class: 'GitSCM', 155 branches: [[name: params.GIT_COMMIT_TO_USE]], 156 doGenerateSubmoduleConfigurations: false, 157 extensions: [], 158 submoduleCfg: [], 159 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 160 env.GIT_COMMIT = scmInfo.GIT_COMMIT 161 env.GIT_BRANCH = scmInfo.GIT_BRANCH 162 // If the commit we were handed is not what the SCM says we are using, fail 163 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 164 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 165 exit 1 166 } 167 } 168 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 169 } 170 171 sh """ 172 cp -f "${NETRC_FILE}" $HOME/.netrc 173 chmod 600 $HOME/.netrc 174 """ 175 176 script { 177 try { 178 sh """ 179 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 180 """ 181 } catch(error) { 182 echo "docker login failed, retrying after sleep" 183 retry(4) { 184 sleep(30) 185 sh """ 186 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 187 """ 188 } 189 } 190 } 191 sh """ 192 rm -rf ${GO_REPO_PATH}/verrazzano 193 mkdir -p ${GO_REPO_PATH}/verrazzano 194 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 195 """ 196 197 script { 198 def props = readProperties file: '.verrazzano-development-version' 199 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 200 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 201 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 202 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 203 // update the description with some meaningful info 204 setDisplayName() 205 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 206 } 207 script { 208 sh """ 209 echo "Downloading VZ CLI from object storage" 210 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 211 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 212 ${GO_REPO_PATH}/vz version 213 """ 214 } 215 } 216 } 217 218 stage('Acceptance Tests') { 219 stages { 220 stage('Prepare AT environment') { 221 environment { 222 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 223 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 224 REALM_USER_PASSWORD = credentials('todo-mysql-password') 225 REALM_NAME = "test-realm" 226 } 227 steps { 228 script { 229 if (params.INSTALL_HA) { 230 sh """ 231 cp ${env.INSTALL_CONFIG_FILE_KIND_HA} ${env.INSTALL_CONFIG_FILE_KIND} 232 cd ${GO_REPO_PATH}/verrazzano 233 ci/scripts/prepare_ha_test_at_environment.sh ${env.INSTALL_CONFIG_FILE_KIND} 234 """ 235 } else { 236 sh """ 237 cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND} 238 """ 239 } 240 } 241 sh """ 242 cd ${GO_REPO_PATH}/verrazzano 243 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA} 244 """ 245 script { 246 if (params.ENABLE_JWT_TESTING) { 247 sh """ 248 # setup test realm for JWT testing scenarios 249 keycloakPassword=\$(kubectl get secret --namespace keycloak keycloak-http -o jsonpath={.data.password} | base64 --decode; echo) 250 sed -i "s|##KEYCLOAK_PASSWORD##|\$keycloakPassword|g" ci/scripts/create_test_realm.sh 251 sed -i "s|##REALM_USER_PASSWORD##|${env.REALM_USER_PASSWORD}|g" ci/scripts/create_test_realm.sh 252 sed -i "s|##REALM_NAME##|${env.REALM_NAME}|g" ci/scripts/create_test_realm.sh 253 kubectl exec keycloak-0 -n keycloak -- /bin/sh -c "`cat ci/scripts/create_test_realm.sh`" 254 # setup request authentication policy 255 keycloakURI=\$(kubectl get ingress -n keycloak keycloak -o jsonpath="{.spec.rules[0].host}") 256 sed -i "s|##KEYCLOAK_URI##|\$keycloakURI|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml 257 jwks=\$(curl -sk https://\$keycloakURI/auth/realms/test-realm/protocol/openid-connect/certs | jq -c 'del(.keys[] | select( .use == "enc"))') 258 sed -i "s|##JWKS_KEY##|\$jwks|g" tests/testdata/jwt/helidon/test-realm-reqauth.yaml 259 kubectl apply -f tests/testdata/jwt/helidon/test-realm-reqauth.yaml 260 """ 261 } 262 } 263 } 264 post { 265 failure { 266 archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true 267 } 268 always { 269 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true 270 // enable debug logging of Verrazzano api istio proxy 271 script { 272 if (params.ENABLE_API_ENVOY_LOGGING) { 273 sh ''' 274 vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\") 275 if [ -z "\$vz_api_pod" ]; then 276 echo "Could not find verrazzano-authproxy pod, not enabling debug logging" 277 else 278 kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug 279 fi 280 nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\") 281 if [ -z "\$nginx_ing_pod" ]; then 282 echo "Could not find nginx ingress controller pod, not enabling debug logging" 283 else 284 kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug 285 fi 286 ''' 287 } 288 } 289 } 290 } 291 } 292 293 stage('Verify analyze-tool') { 294 environment { 295 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-analyze-tool" 296 } 297 steps { 298 runGinkgo('verify-analyze-tool') 299 } 300 post { 301 always { 302 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 303 junit testResults: '**/*test-result.xml', allowEmptyResults: true 304 } 305 } 306 } 307 } 308 309 post { 310 aborted { 311 script { 312 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 313 dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot') 314 } 315 } 316 } 317 failure { 318 script { 319 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 320 dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot') 321 } 322 } 323 } 324 success { 325 script { 326 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 327 dumpK8sCluster('new-kind-acceptance-tests-cluster-snapshot') 328 } 329 } 330 } 331 } 332 } 333 334 } 335 336 post { 337 always { 338 script { 339 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 340 dumpVerrazzanoSystemPods() 341 dumpCattleSystemPods() 342 dumpNginxIngressControllerLogs() 343 dumpVerrazzanoPlatformOperatorLogs() 344 dumpVerrazzanoApplicationOperatorLogs() 345 dumpOamKubernetesRuntimeLogs() 346 dumpVerrazzanoApiLogs() 347 } 348 } 349 350 sh """ 351 # Copy the generated test reports to WORKSPACE to archive them 352 mkdir -p ${TEST_REPORT_DIR} 353 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 354 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 355 """ 356 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true 357 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 358 deleteCluster() 359 } 360 failure { 361 sh """ 362 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 363 """ 364 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 365 sh """ 366 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 367 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 368 rm archive.zip 369 """ 370 } 371 cleanup { 372 deleteDir() 373 } 374 } 375 } 376 377 378 def runGinkgoRandomize(testSuitePath, kubeConfig = '') { 379 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 380 sh """ 381 if [ ! -z "${kubeConfig}" ]; then 382 export KUBECONFIG="${kubeConfig}" 383 fi 384 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 385 if [ -d "${testSuitePath}" ]; then 386 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 387 fi 388 """ 389 } 390 } 391 392 def runSocksVariant(variant) { 393 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 394 sh """ 395 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 396 SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" examples/socks/... 397 """ 398 } 399 } 400 401 def runGinkgo(testSuitePath) { 402 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 403 sh """ 404 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 405 ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 406 """ 407 } 408 } 409 410 def dumpK8sCluster(dumpDirectory) { 411 sh """ 412 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 413 """ 414 } 415 416 def dumpVerrazzanoSystemPods() { 417 sh """ 418 cd ${GO_REPO_PATH}/verrazzano/platform-operator 419 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log" 420 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 421 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log" 422 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 423 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log" 424 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 425 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log" 426 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 427 """ 428 } 429 430 def dumpCattleSystemPods() { 431 sh """ 432 cd ${GO_REPO_PATH}/verrazzano/platform-operator 433 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log" 434 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 435 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log" 436 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 437 """ 438 } 439 440 def dumpNginxIngressControllerLogs() { 441 sh """ 442 cd ${GO_REPO_PATH}/verrazzano/platform-operator 443 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log" 444 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 445 """ 446 } 447 448 def dumpVerrazzanoPlatformOperatorLogs() { 449 sh """ 450 ## dump out verrazzano-platform-operator logs 451 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 452 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 453 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 454 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 455 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 456 echo "------------------------------------------" 457 """ 458 } 459 460 def dumpVerrazzanoApplicationOperatorLogs() { 461 sh """ 462 ## dump out verrazzano-application-operator logs 463 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 464 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 465 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 466 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 467 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 468 echo "------------------------------------------" 469 """ 470 } 471 472 def dumpOamKubernetesRuntimeLogs() { 473 sh """ 474 ## dump out oam-kubernetes-runtime logs 475 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 476 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 477 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 478 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 479 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 480 echo "------------------------------------------" 481 """ 482 } 483 484 def dumpVerrazzanoApiLogs() { 485 sh """ 486 cd ${GO_REPO_PATH}/verrazzano/platform-operator 487 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log" 488 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 489 """ 490 } 491 492 def getEffectiveDumpOnSuccess() { 493 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 494 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 495 effectiveValue = true 496 echo "Forcing dump on success based on global override setting" 497 } 498 return effectiveValue 499 } 500 501 def deleteCluster() { 502 sh """ 503 cd ${GO_REPO_PATH}/verrazzano/platform-operator 504 make delete-cluster 505 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 506 echo "Failures seen during dumping of artifacts, treat post as failed" 507 exit 1 508 fi 509 """ 510 } 511 512 def setDisplayName() { 513 echo "Start setDisplayName" 514 def causes = currentBuild.getBuildCauses() 515 echo "causes: " + causes.toString() 516 for (cause in causes) { 517 def causeString = cause.toString() 518 echo "current cause: " + causeString 519 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 520 echo "This job was caused by " + causeString 521 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 522 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 523 } else if (causeString.contains("verrazzano-flaky-tests")) { 524 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 525 } 526 } 527 } 528 echo "End setDisplayName" 529 }