github.com/verrazzano/verrazzano@v1.7.1/ci/clusterAPI/JenkinsfileCAPIQC (about) 1 // Copyright (c) 2020, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 7 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ] 8 def okeCNITypes = ['FLANNEL_OVERLAY', 'OCI_VCN_IP_NATIVE'] 9 Collections.shuffle(availableRegions) 10 Collections.shuffle(okeCNITypes) 11 12 pipeline { 13 options { 14 timeout(time: 90, unit: 'MINUTES') 15 skipDefaultCheckout true 16 timestamps () 17 } 18 19 agent { 20 docker { 21 image "${RUNNER_DOCKER_IMAGE}" 22 args "${RUNNER_DOCKER_ARGS}" 23 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 24 registryCredentialsId 'ocir-pull-and-push-account' 25 label "${agentLabel}" 26 } 27 } 28 29 parameters { 30 choice(name: 'KUBERNETES_CLUSTER_VERSION', 31 description: 'Kubernetes Version for KinD Cluster', 32 // 1st choice is the default value 33 choices: ["1.27", "1.26", "1.25", "1.24"]) 34 booleanParam(description: 'Deploy OCNE OCI Cluster', name: 'DEPLOY_OCNEOCI_CLUSTER', defaultValue: false) 35 booleanParam(description: 'Deploy OKE Cluster', name: 'DEPLOY_OKE_CLUSTER', defaultValue: true) 36 choice (description: 'CNI Type for OKE', name: 'OKE_CNI_TYPE', choices: okeCNITypes ) 37 choice (description: 'OCI region to launch CAPI clusters in', name: 'CAPI_CLUSTER_REGION', choices: availableRegions ) 38 string(name: 'GIT_COMMIT_TO_USE', 39 defaultValue: 'NONE', 40 description: 'This is the full git commit hash from the source build to be used for all jobs', 41 trim: true) 42 string(name: 'VERRAZZANO_OPERATOR_IMAGE', 43 defaultValue: 'NONE', 44 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator', 45 trim: true) 46 choice(name: 'WILDCARD_DNS_DOMAIN', 47 description: 'This is the wildcard DNS domain', 48 // 1st choice is the default value 49 choices: ["nip.io", "sslip.io"]) 50 choice(name: 'CRD_API_VERSION', 51 description: 'This is the API crd version.', 52 // 1st choice is the default value 53 choices: ["v1beta1", "v1alpha1"]) 54 booleanParam(description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 55 booleanParam(description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 56 booleanParam(description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true) 57 string(name: 'TAGGED_TESTS', 58 defaultValue: '', 59 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 60 trim: true) 61 string(name: 'INCLUDED_TESTS', 62 defaultValue: '.*', 63 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 64 trim: true) 65 string(name: 'EXCLUDED_TESTS', 66 defaultValue: '_excluded_test', 67 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 68 trim: true) 69 booleanParam(description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 70 } 71 environment { 72 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 73 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 74 GOPATH = '/home/opc/go' 75 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 76 DOCKER_CREDS = credentials('github-packages-credentials-rw') 77 DOCKER_EMAIL = credentials('github-packages-email') 78 DOCKER_REPO = 'ghcr.io' 79 DOCKER_NAMESPACE = 'verrazzano' 80 NETRC_FILE = credentials('netrc') 81 CLUSTER_NAME = 'verrazzano' 82 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 83 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 84 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 85 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 86 OCR_CREDS = credentials('ocr-pull-and-push-account') 87 OCR_REPO = 'container-registry.oracle.com' 88 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 89 INSTALL_CONFIG_FILE_KIND_TESTS = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml" 90 INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/install-verrazzano.yaml" 91 INSTALL_PROFILE = "dev" 92 KIND_NODE_COUNT = "1" 93 VZ_ENVIRONMENT_NAME = "default" 94 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 95 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 96 97 // Environment variables required to capture cluster snapshot and bug report on test failure 98 DUMP_KUBECONFIG="${KUBECONFIG}" 99 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 100 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 101 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 102 103 // Environment variable for Verrazzano CLI executable 104 VZ_COMMAND="${GO_REPO_PATH}/vz" 105 106 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 107 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 108 109 // used for console artifact capture on failure 110 JENKINS_READ = credentials('jenkins-auditor') 111 OCI_CLI_AUTH="instance_principal" 112 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 113 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 114 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 115 116 // used to emit metrics from Ginkgo suites 117 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 118 TEST_ENV_LABEL = "kind" 119 TEST_ENV = "KIND" 120 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 121 122 // used to generate Ginkgo test reports 123 TEST_REPORT = "test-report.xml" 124 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 125 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 126 127 // CAPI variables 128 OCI_USER_ID = credentials('oci-user-ocid') 129 OCI_CREDENTIALS_FINGERPRINT = credentials('oci-api-key-fingerprint') 130 OCI_TENANCY_ID = credentials('oci-tenancy') 131 OCI_REGION = "${params.CAPI_CLUSTER_REGION}" 132 OCI_COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid') 133 CAPI_NODE_SSH_KEY_PATH = credentials('oci-tf-pub-ssh-key') 134 CAPI_OCI_PRIVATE_KEY_PATH = credentials('oci-api-key') 135 CNI_TYPE = "${params.OKE_CNI_TYPE}" 136 } 137 138 stages { 139 stage('Clean workspace and checkout') { 140 steps { 141 sh """ 142 echo "${NODE_LABELS}" 143 """ 144 145 script { 146 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 147 if (params.GIT_COMMIT_TO_USE == "NONE") { 148 echo "Specific GIT commit was not specified, use current head" 149 def scmInfo = checkout scm 150 env.GIT_COMMIT = scmInfo.GIT_COMMIT 151 env.GIT_BRANCH = scmInfo.GIT_BRANCH 152 } else { 153 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 154 def scmInfo = checkout([ 155 $class: 'GitSCM', 156 branches: [[name: params.GIT_COMMIT_TO_USE]], 157 doGenerateSubmoduleConfigurations: false, 158 extensions: [], 159 submoduleCfg: [], 160 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 161 env.GIT_COMMIT = scmInfo.GIT_COMMIT 162 env.GIT_BRANCH = scmInfo.GIT_BRANCH 163 // If the commit we were handed is not what the SCM says we are using, fail 164 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 165 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 166 exit 1 167 } 168 } 169 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 170 } 171 172 sh """ 173 cp -f "${NETRC_FILE}" $HOME/.netrc 174 chmod 600 $HOME/.netrc 175 """ 176 177 performDockerLogin() 178 179 sh """ 180 rm -rf ${GO_REPO_PATH}/verrazzano 181 mkdir -p ${GO_REPO_PATH}/verrazzano 182 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 183 """ 184 185 script { 186 def props = readProperties file: '.verrazzano-development-version' 187 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 188 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 189 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 190 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 191 // update the description with some meaningful info 192 setDisplayName() 193 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 194 } 195 script { 196 sh """ 197 echo "Downloading VZ CLI from object storage" 198 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 199 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 200 ${GO_REPO_PATH}/vz version 201 """ 202 } 203 } 204 } 205 206 stage('Tests') { 207 stages { 208 stage('Prepare AT environment') { 209 environment { 210 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 211 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 212 REALM_USER_PASSWORD = credentials('todo-mysql-password') 213 REALM_NAME = "test-realm" 214 GITHUB_TOKEN = credentials('github-api-token-release-process') 215 } 216 steps { 217 script { 218 sh """ 219 cp ${env.INSTALL_CONFIG_FILE_KIND_TESTS} ${env.INSTALL_CONFIG_FILE_KIND} 220 """ 221 } 222 sh """ 223 cd ${GO_REPO_PATH}/verrazzano 224 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA} 225 """ 226 } 227 post { 228 failure { 229 archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true 230 } 231 always { 232 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true 233 // enable debug logging of Verrazzano api istio proxy 234 script { 235 if (params.ENABLE_API_ENVOY_LOGGING) { 236 sh ''' 237 vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\") 238 if [ -z "\$vz_api_pod" ]; then 239 echo "Could not find verrazzano-authproxy pod, not enabling debug logging" 240 else 241 kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug 242 fi 243 nginx_ing_pod=\$(kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\") 244 if [ -z "\$nginx_ing_pod" ]; then 245 echo "Could not find nginx ingress controller pod, not enabling debug logging" 246 else 247 kubectl exec \$nginx_ing_pod -c istio-proxy -n ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug 248 fi 249 ''' 250 } 251 } 252 } 253 } 254 } 255 256 stage('Verify Install') { 257 environment { 258 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/verify-install" 259 } 260 steps { 261 runGinkgoRandomize('verify-install') 262 } 263 post { 264 always { 265 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 266 junit testResults: '**/*test-result.xml', allowEmptyResults: true 267 } 268 } 269 } 270 stage('Deploy Quick Create Clusters in parallel') { 271 parallel { 272 stage('OCNEOCI Cluster - Run verrazzano addon tests') { 273 when { 274 expression { params.DEPLOY_OCNEOCI_CLUSTER == true } 275 } 276 steps { 277 runQuickCreateSuite('ocneoci') 278 } 279 post { 280 always { 281 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 282 junit testResults: '**/*test-result.xml', allowEmptyResults: true 283 } 284 failure { 285 script { 286 dumpK8sCluster('ocne-deploy-failure') 287 } 288 } 289 } 290 } 291 stage('OKE Cluster - Run verrazzano addon tests') { 292 when { 293 expression { params.DEPLOY_OKE_CLUSTER == true } 294 } 295 steps { 296 runQuickCreateSuite('oke') 297 } 298 post { 299 always { 300 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 301 junit testResults: '**/*test-result.xml', allowEmptyResults: true 302 } 303 failure { 304 script { 305 dumpK8sCluster('oke-deploy-failure') 306 } 307 } 308 } 309 } 310 } 311 } 312 } 313 314 post { 315 aborted { 316 script { 317 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 318 dumpK8sCluster('clusterAPI-qc-kind-tests-cluster-snapshot') 319 } 320 } 321 } 322 failure { 323 script { 324 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 325 dumpK8sCluster('clusterAPI-qc-kind-tests-cluster-snapshot') 326 } 327 } 328 } 329 success { 330 script { 331 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 332 dumpK8sCluster('clusterAPI-qc-kind-tests-cluster-snapshot') 333 } 334 } 335 } 336 } 337 } 338 } 339 340 post { 341 always { 342 script { 343 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 344 dumpVerrazzanoSystemPods() 345 dumpCattleSystemPods() 346 dumpNginxIngressControllerLogs() 347 dumpVerrazzanoPlatformOperatorLogs() 348 dumpVerrazzanoApplicationOperatorLogs() 349 dumpOamKubernetesRuntimeLogs() 350 dumpVerrazzanoApiLogs() 351 } 352 } 353 354 sh """ 355 # Copy the generated test reports to WORKSPACE to archive them 356 mkdir -p ${TEST_REPORT_DIR} 357 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 358 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 359 """ 360 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true 361 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 362 deleteCluster() 363 } 364 failure { 365 sh """ 366 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 367 """ 368 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 369 sh """ 370 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 371 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 372 rm archive.zip 373 """ 374 } 375 cleanup { 376 deleteDir() 377 } 378 } 379 } 380 381 def runQuickCreateSuite(clusterType) { 382 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 383 script { 384 sh """ 385 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 386 if [ -d "quickcreate" ]; then 387 ginkgo -vv --progress --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" quickcreate/... -- --clusterType="${clusterType}" 388 fi 389 """ 390 } 391 } 392 } 393 394 def runGinkgoRandomize(testSuitePath, kubeConfig = '') { 395 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 396 sh """ 397 if [ ! -z "${kubeConfig}" ]; then 398 export KUBECONFIG="${kubeConfig}" 399 fi 400 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 401 if [ -d "${testSuitePath}" ]; then 402 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 403 fi 404 """ 405 } 406 } 407 408 def dumpK8sCluster(dumpDirectory) { 409 sh """ 410 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 411 """ 412 } 413 414 def dumpVerrazzanoSystemPods() { 415 sh """ 416 cd ${GO_REPO_PATH}/verrazzano/platform-operator 417 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log" 418 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 419 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log" 420 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 421 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log" 422 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 423 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log" 424 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 425 """ 426 } 427 428 def dumpCattleSystemPods() { 429 sh """ 430 cd ${GO_REPO_PATH}/verrazzano/platform-operator 431 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log" 432 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 433 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log" 434 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 435 """ 436 } 437 438 def dumpNginxIngressControllerLogs() { 439 sh """ 440 cd ${GO_REPO_PATH}/verrazzano/platform-operator 441 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log" 442 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 443 """ 444 } 445 446 def dumpVerrazzanoPlatformOperatorLogs() { 447 sh """ 448 ## dump out verrazzano-platform-operator logs 449 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 450 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 451 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 452 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 453 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 454 echo "------------------------------------------" 455 """ 456 } 457 458 def dumpVerrazzanoApplicationOperatorLogs() { 459 sh """ 460 ## dump out verrazzano-application-operator logs 461 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 462 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 463 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 464 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 465 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 466 echo "------------------------------------------" 467 """ 468 } 469 470 def dumpOamKubernetesRuntimeLogs() { 471 sh """ 472 ## dump out oam-kubernetes-runtime logs 473 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 474 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 475 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 476 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 477 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 478 echo "------------------------------------------" 479 """ 480 } 481 482 def dumpVerrazzanoApiLogs() { 483 sh """ 484 cd ${GO_REPO_PATH}/verrazzano/platform-operator 485 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log" 486 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 487 """ 488 } 489 490 def getEffectiveDumpOnSuccess() { 491 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 492 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 493 effectiveValue = true 494 echo "Forcing dump on success based on global override setting" 495 } 496 return effectiveValue 497 } 498 499 def deleteCluster() { 500 sh """ 501 cd ${GO_REPO_PATH}/verrazzano/platform-operator 502 make delete-cluster 503 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 504 echo "Failures seen during dumping of artifacts, treat post as failed" 505 exit 1 506 fi 507 """ 508 } 509 510 def setDisplayName() { 511 echo "Start setDisplayName" 512 def causes = currentBuild.getBuildCauses() 513 echo "causes: " + causes.toString() 514 for (cause in causes) { 515 def causeString = cause.toString() 516 echo "current cause: " + causeString 517 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 518 echo "This job was caused by " + causeString 519 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 520 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 521 } else if (causeString.contains("verrazzano-flaky-tests")) { 522 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 523 } 524 } 525 } 526 echo "End setDisplayName" 527 } 528 529 def performDockerLogin() { 530 script { 531 try { 532 sh """ 533 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 534 """ 535 } catch(error) { 536 echo "docker login failed, retrying after sleep" 537 retry(4) { 538 sleep(30) 539 sh """ 540 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 541 """ 542 } 543 } 544 } 545 }