github.com/verrazzano/verrazzano@v1.7.1/ci/dynamic-updates/JenkinsfileUpdateDuringInstall (about) 1 // Copyright (c) 2023, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 7 8 pipeline { 9 options { 10 skipDefaultCheckout true 11 timestamps () 12 } 13 14 agent { 15 docker { 16 image "${RUNNER_DOCKER_IMAGE}" 17 args "${RUNNER_DOCKER_ARGS}" 18 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 19 registryCredentialsId 'ocir-pull-and-push-account' 20 label "${agentLabel}" 21 } 22 } 23 24 parameters { 25 choice (name: 'KUBERNETES_CLUSTER_VERSION', 26 description: 'Kubernetes Version for KinD Cluster', 27 // 1st choice is the default value 28 choices: ["1.27", "1.26", "1.25", "1.24" ]) 29 string (name: 'GIT_COMMIT_TO_USE', 30 defaultValue: 'NONE', 31 description: 'This is the full git commit hash from the source build to be used for all jobs', 32 trim: true) 33 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 34 defaultValue: 'NONE', 35 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator', 36 trim: true) 37 choice (name: 'WILDCARD_DNS_DOMAIN', 38 description: 'This is the wildcard DNS domain', 39 // 1st choice is the default value 40 choices: [ "nip.io", "sslip.io"]) 41 choice (name: 'CRD_API_VERSION', 42 description: 'This is the API crd version.', 43 // 1st choice is the default value 44 choices: [ "v1beta1", "v1alpha1"]) 45 string (name: 'TAGGED_TESTS', 46 defaultValue: '', 47 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 48 trim: true) 49 string (name: 'INCLUDED_TESTS', 50 defaultValue: '.*', 51 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 52 trim: true) 53 string (name: 'EXCLUDED_TESTS', 54 defaultValue: '_excluded_test', 55 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 56 trim: true) 57 booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 58 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 59 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 60 choice (name:'WAIT_FOR_RESOURCE_BEFORE_UPDATE', 61 description: 'The update to the VZ CR will occur after this specified resource is created. This affects the timing of the update.', 62 choices: [ "mysql statefulset", "rancher bootstrap secret"]) 63 } 64 65 environment { 66 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 67 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 68 GOPATH = '/home/opc/go' 69 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 70 DOCKER_CREDS = credentials('github-packages-credentials-rw') 71 DOCKER_EMAIL = credentials('github-packages-email') 72 DOCKER_REPO = 'ghcr.io' 73 DOCKER_NAMESPACE = 'verrazzano' 74 NETRC_FILE = credentials('netrc') 75 CLUSTER_NAME = 'verrazzano' 76 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 77 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 78 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 79 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 80 OCR_CREDS = credentials('ocr-pull-and-push-account') 81 OCR_REPO = 'container-registry.oracle.com' 82 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 83 INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-with-persistence.yaml" 84 INSTALL_PROFILE = "dev" 85 VZ_ENVIRONMENT_NAME = "default" 86 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 87 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 88 89 // Environment variables required to capture cluster snapshot and bug report on test failure 90 DUMP_KUBECONFIG="${KUBECONFIG}" 91 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 92 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 93 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 94 95 // Environment variable for Verrazzano CLI executable 96 VZ_COMMAND="${GO_REPO_PATH}/vz" 97 98 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 99 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 100 101 // used for console artifact capture on failure 102 JENKINS_READ = credentials('jenkins-auditor') 103 OCI_CLI_AUTH="instance_principal" 104 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 105 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 106 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 107 108 // used to emit metrics 109 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 110 TEST_ENV_LABEL = "kind" 111 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 112 TEST_ENV = "KIND" 113 114 // used to generate Ginkgo test reports 115 TEST_REPORT = "test-report.xml" 116 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 117 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 118 119 // how many nodes in the KIND cluster 120 KIND_NODE_COUNT = 3 121 } 122 123 stages { 124 stage('Clean workspace and checkout') { 125 steps { 126 sh """ 127 echo "${NODE_LABELS}" 128 """ 129 130 script { 131 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 132 if (params.GIT_COMMIT_TO_USE == "NONE") { 133 echo "Specific GIT commit was not specified, use current head" 134 def scmInfo = checkout scm 135 env.GIT_COMMIT = scmInfo.GIT_COMMIT 136 env.GIT_BRANCH = scmInfo.GIT_BRANCH 137 } else { 138 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 139 def scmInfo = checkout([ 140 $class: 'GitSCM', 141 branches: [[name: params.GIT_COMMIT_TO_USE]], 142 doGenerateSubmoduleConfigurations: false, 143 extensions: [], 144 submoduleCfg: [], 145 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 146 env.GIT_COMMIT = scmInfo.GIT_COMMIT 147 env.GIT_BRANCH = scmInfo.GIT_BRANCH 148 // If the commit we were handed is not what the SCM says we are using, fail 149 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 150 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 151 exit 1 152 } 153 } 154 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 155 } 156 157 sh """ 158 cp -f "${NETRC_FILE}" $HOME/.netrc 159 chmod 600 $HOME/.netrc 160 """ 161 162 script { 163 try { 164 sh """ 165 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 166 """ 167 } catch(error) { 168 echo "docker login failed, retrying after sleep" 169 retry(4) { 170 sleep(30) 171 sh """ 172 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 173 """ 174 } 175 } 176 } 177 sh """ 178 rm -rf ${GO_REPO_PATH}/verrazzano 179 mkdir -p ${GO_REPO_PATH}/verrazzano 180 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 181 """ 182 183 script { 184 def props = readProperties file: '.verrazzano-development-version' 185 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 186 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 187 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 188 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 189 // update the description with some meaningful info 190 setDisplayName() 191 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 192 } 193 script { 194 sh """ 195 echo "Downloading VZ CLI from object storage" 196 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 197 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 198 ${VZ_COMMAND} version 199 """ 200 } 201 } 202 } 203 204 // This kicks off two parallel stages: 205 // 1. A stage to install Verrazzano, with a typo in the Verrazzano CRD 206 // 2. A stage that waits a certain amount of time, then updates the Verrazzano CRD to fix the typo while the first stage is installing 207 // This aims to test that updates to the Verrazzano CRD during initial install does not break the installation. 208 stage('Update During Install') { 209 parallel { 210 stage('Install Verrazzano with Wildcard DNS Typo') { 211 environment { 212 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 213 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 214 } 215 steps { 216 // This installs Verrazzano with an invalid WildcardDNS domain of "typodomain.io" 217 // This also makes sure that certain components relevant to testing are enabled 218 sh """ 219 cd ${GO_REPO_PATH}/verrazzano 220 yq -i eval '.spec.components.keycloak.enabled = true' ${INSTALL_CONFIG_FILE_KIND} 221 yq -i eval '.spec.components.rancher.enabled = true' ${INSTALL_CONFIG_FILE_KIND} 222 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} "typodomain.io" 223 """ 224 } 225 post { 226 success { 227 script { 228 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 229 dumpK8sCluster('dynamic-install-post-install-cluster-snapshot') 230 } 231 } 232 } 233 failure { 234 archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true 235 } 236 always { 237 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 238 } 239 } 240 } 241 242 stage('Update the VZ CR') { 243 options { 244 timeout(time: 30, unit: "MINUTES") 245 } 246 steps { 247 script { 248 // first wait for certain resources to be created, then update the Verrazzano CRD to fix the WildcardDNS typo 249 if (params.WAIT_FOR_RESOURCE_BEFORE_UPDATE == 'mysql statefulset') { 250 waitForKeycloakMySqlStatefulset() 251 } else if (params.WAIT_FOR_RESOURCE_BEFORE_UPDATE == 'rancher bootstrap secret') { 252 waitForRancherBootstrapSecret() 253 } 254 patchWildcardDNSDomain() 255 } 256 } 257 } 258 } 259 260 post { 261 failure { 262 script { 263 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 264 dumpK8sCluster('dynamic-install-failed-cluster-snapshot') 265 } 266 } 267 } 268 success { 269 script { 270 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 271 dumpK8sCluster('dynamic-install-success-cluster-snapshot') 272 } 273 } 274 } 275 } 276 } 277 278 stage('Verify Tests') { 279 environment { 280 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-install-verify-infra" 281 } 282 steps { 283 runGinkgoRandomize('verify-install') 284 } 285 post { 286 failure { 287 script { 288 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 289 dumpK8sCluster('dynamic-install-verify-install-failed-cluster-snapshot') 290 } 291 } 292 } 293 always { 294 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 295 junit testResults: '**/*test-result.xml', allowEmptyResults: true 296 } 297 } 298 } 299 300 stage('Infra Tests') { 301 environment { 302 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/post-install-infra-tests" 303 } 304 steps { 305 script { 306 parallel generateVerifyInfraStages("${TEST_DUMP_ROOT}/post-install-infra-tests") 307 } 308 } 309 post { 310 failure { 311 script { 312 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 313 dumpK8sCluster('dynamic-install-verify-infra-failed-cluster-snapshot') 314 } 315 } 316 } 317 always { 318 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 319 junit testResults: '**/*test-result.xml', allowEmptyResults: true 320 } 321 } 322 } 323 } 324 325 post { 326 always { 327 script { 328 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 329 dumpVerrazzanoSystemPods() 330 dumpCattleSystemPods() 331 dumpNginxIngressControllerLogs() 332 dumpVerrazzanoPlatformOperatorLogs("post-run") 333 dumpVerrazzanoApplicationOperatorLogs() 334 dumpOamKubernetesRuntimeLogs() 335 dumpVerrazzanoApiLogs() 336 } 337 } 338 sh """ 339 # Copy the generated test reports to WORKSPACE to archive them 340 mkdir -p ${TEST_REPORT_DIR} 341 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 342 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 343 """ 344 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true 345 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 346 deleteCluster() 347 } 348 failure { 349 sh """ 350 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 351 """ 352 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 353 sh """ 354 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 355 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 356 rm archive.zip 357 """ 358 } 359 cleanup { 360 deleteDir() 361 } 362 } 363 } 364 365 def generateVerifyInfraStages(dumpRoot) { 366 return [ 367 "verify-infra restapi": { 368 runGinkgoRandomize('verify-infra/restapi', "${dumpRoot}/verify-infra-restapi") 369 }, 370 "verify-infra oam": { 371 runGinkgoRandomize('verify-infra/oam', "${dumpRoot}/verify-infra-oam") 372 }, 373 ] 374 } 375 376 def runGinkgoRandomize(testSuitePath, dumpDir = '') { 377 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 378 sh """ 379 if [ ! -z "${dumpDir}" ]; then 380 export DUMP_DIRECTORY=${dumpDir} 381 fi 382 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 383 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 384 """ 385 } 386 } 387 388 def dumpK8sCluster(dumpDirectory) { 389 sh """ 390 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 391 """ 392 } 393 394 def dumpVerrazzanoSystemPods() { 395 sh """ 396 cd ${GO_REPO_PATH}/verrazzano/platform-operator 397 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log" 398 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 399 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log" 400 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 401 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log" 402 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 403 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log" 404 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 405 """ 406 } 407 408 def dumpCattleSystemPods() { 409 sh """ 410 cd ${GO_REPO_PATH}/verrazzano/platform-operator 411 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log" 412 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 413 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log" 414 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 415 """ 416 } 417 418 def dumpNginxIngressControllerLogs() { 419 sh """ 420 cd ${GO_REPO_PATH}/verrazzano/platform-operator 421 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log" 422 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 423 """ 424 } 425 426 def dumpVerrazzanoPlatformOperatorLogs(stage) { 427 sh """ 428 ## dump out verrazzano-platform-operator logs 429 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 430 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 431 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-${stage}-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 432 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 433 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 434 echo "------------------------------------------" 435 """ 436 } 437 438 def dumpVerrazzanoApplicationOperatorLogs() { 439 sh """ 440 ## dump out verrazzano-application-operator logs 441 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 442 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 443 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 444 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 445 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 446 echo "------------------------------------------" 447 """ 448 } 449 450 def dumpOamKubernetesRuntimeLogs() { 451 sh """ 452 ## dump out oam-kubernetes-runtime logs 453 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 454 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 455 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 456 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 457 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 458 echo "------------------------------------------" 459 """ 460 } 461 462 def dumpVerrazzanoApiLogs() { 463 sh """ 464 cd ${GO_REPO_PATH}/verrazzano/platform-operator 465 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-api.log" 466 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-api-*" -m "verrazzano api" -c verrazzano-api -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 467 """ 468 } 469 470 def getEffectiveDumpOnSuccess() { 471 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 472 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 473 effectiveValue = true 474 echo "Forcing dump on success based on global override setting" 475 } 476 return effectiveValue 477 } 478 479 def deleteCluster() { 480 sh """ 481 cd ${GO_REPO_PATH}/verrazzano/platform-operator 482 make delete-cluster 483 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 484 echo "Failures seen during dumping of artifacts, treat post as failed" 485 exit 1 486 fi 487 """ 488 } 489 490 def setDisplayName() { 491 echo "Start setDisplayName" 492 def causes = currentBuild.getBuildCauses() 493 echo "causes: " + causes.toString() 494 for (cause in causes) { 495 def causeString = cause.toString() 496 echo "current cause: " + causeString 497 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 498 echo "This job was caused by " + causeString 499 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 500 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 501 } else if (causeString.contains("verrazzano-flaky-tests")) { 502 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 503 } 504 } 505 } 506 echo "End setDisplayName" 507 } 508 509 def patchWildcardDNSDomain() { 510 sh """ 511 echo "WildcardDNS Configuration before patching:" 512 kubectl get vz my-verrazzano -o yaml | grep 'domain:' 513 514 # change the Wildcard DNS domain 515 kubectl patch vz my-verrazzano -p '{"spec":{"components":{"dns":{"wildcard":{"domain":"${params.WILDCARD_DNS_DOMAIN}"}}}}}' --type=merge 516 517 echo "WildcardDNSConfiguration after patching:" 518 kubectl get vz my-verrazzano -o yaml | grep 'domain:' 519 """ 520 } 521 522 def waitForRancherBootstrapSecret() { 523 sh """ 524 while ! kubectl get vz 525 do 526 echo "Waiting for verrazzano resource to be created..." 527 sleep 30s 528 done 529 530 while ! kubectl get secret bootstrap-secret --namespace cattle-system 531 do 532 echo "Waiting for bootstrap-secret Secret to be created..." 533 sleep 1m 534 done 535 """ 536 } 537 538 def waitForKeycloakMySqlStatefulset() { 539 sh """ 540 while ! kubectl get vz 541 do 542 echo "Waiting for verrazzano resource to be created..." 543 sleep 30s 544 done 545 546 while ! kubectl get statefulset mysql -n keycloak 547 do 548 echo "Waiting for keycloak/mysql statefuleset to be created..." 549 sleep 15s 550 done 551 """ 552 }