github.com/verrazzano/verrazzano@v1.7.1/ci/rolebased/Jenkinsfile (about) 1 // Copyright (c) 2020, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def SKIP_ACCEPTANCE_TESTS = false 6 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 7 8 pipeline { 9 options { 10 skipDefaultCheckout true 11 } 12 13 agent { 14 docker { 15 image "${RUNNER_DOCKER_IMAGE}" 16 args "${RUNNER_DOCKER_ARGS}" 17 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 18 registryCredentialsId 'ocir-pull-and-push-account' 19 label "${agentLabel}" 20 } 21 } 22 23 parameters { 24 booleanParam (description: 'Whether to kick off acceptance test run at the end of this build', name: 'RUN_ACCEPTANCE_TESTS', defaultValue: true) 25 booleanParam (description: 'Whether to run example tests', name: 'RUN_EXAMPLE_TESTS', defaultValue: true) 26 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 27 defaultValue: 'NONE', 28 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from VZ repo will be leveraged to create VZ platform operator', 29 trim: true) 30 choice (name: 'WILDCARD_DNS_DOMAIN', 31 description: 'This is the wildcard DNS domain', 32 // 1st choice is the default value 33 choices: [ "nip.io", "sslip.io"]) 34 choice (name: 'CRD_API_VERSION', 35 description: 'This is the API crd version.', 36 // 1st choice is the default value 37 choices: [ "v1beta1", "v1alpha1"]) 38 booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 39 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 40 } 41 42 environment { 43 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 44 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 45 DOCKER_PLATFORM_IMAGE_NAME = "${env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' ? env.DOCKER_PLATFORM_PUBLISH_IMAGE_NAME : env.DOCKER_PLATFORM_CI_IMAGE_NAME}" 46 DOCKER_OAM_CI_IMAGE_NAME = 'verrazzano-application-operator-jenkins' 47 DOCKER_OAM_PUBLISH_IMAGE_NAME = 'verrazzano-application-operator' 48 DOCKER_OAM_IMAGE_NAME = "${env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'master' ? env.DOCKER_OAM_PUBLISH_IMAGE_NAME : env.DOCKER_OAM_CI_IMAGE_NAME}" 49 CREATE_LATEST_TAG = "${env.BRANCH_NAME == 'master' ? '1' : '0'}" 50 GOPATH = '/home/opc/go' 51 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 52 DOCKER_CREDS = credentials('github-packages-credentials-rw') 53 DOCKER_EMAIL = credentials('github-packages-email') 54 DOCKER_REPO = 'ghcr.io' 55 DOCKER_NAMESPACE = 'verrazzano' 56 NETRC_FILE = credentials('netrc') 57 SERVICE_KEY = credentials('PAGERDUTY_SERVICE_KEY') 58 59 CLUSTER_NAME = 'verrazzano' 60 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 61 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 62 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 63 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 64 65 // Environment variable for Verrazzano CLI executable 66 VZ_COMMAND="${GO_REPO_PATH}/vz" 67 68 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 69 OCR_CREDS = credentials('ocr-pull-and-push-account') 70 OCR_REPO = 'container-registry.oracle.com' 71 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 72 INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml" 73 INSTALL_PROFILE = "dev" 74 VZ_ENVIRONMENT_NAME = "default" 75 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 76 77 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // Needed by ToDoList example test 78 DATABASE_PSW = credentials('todo-mysql-password') // Needed by ToDoList example test 79 80 // used to emit metrics 81 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 82 TEST_ENV = "KIND" 83 } 84 85 stages { 86 stage('Clean workspace and checkout') { 87 environment { 88 OCI_CLI_AUTH="instance_principal" 89 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 90 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 91 } 92 steps { 93 sh """ 94 echo "${NODE_LABELS}" 95 """ 96 97 script { 98 checkout scm 99 } 100 sh """ 101 cp -f "${NETRC_FILE}" $HOME/.netrc 102 chmod 600 $HOME/.netrc 103 """ 104 105 script { 106 try { 107 sh """ 108 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 109 """ 110 } catch(error) { 111 echo "docker login failed, retrying after sleep" 112 retry(4) { 113 sleep(30) 114 sh """ 115 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 116 """ 117 } 118 } 119 } 120 sh """ 121 rm -rf ${GO_REPO_PATH}/verrazzano 122 mkdir -p ${GO_REPO_PATH}/verrazzano 123 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 124 """ 125 126 script { 127 def props = readProperties file: '.verrazzano-development-version' 128 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 129 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 130 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 131 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 132 } 133 script { 134 sh """ 135 echo "Downloading VZ CLI from object storage" 136 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 137 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 138 ${GO_REPO_PATH}/vz version 139 """ 140 } 141 } 142 } 143 144 stage('Acceptance Tests') { 145 146 stages { 147 148 stage('Prepare AT environment') { 149 environment { 150 VERRAZZANO_OPERATOR_IMAGE="NONE" 151 KIND_KUBERNETES_CLUSTER_VERSION="1.24" 152 OCI_CLI_AUTH="instance_principal" 153 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 154 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 155 } 156 steps { 157 sh """ 158 cd ${GO_REPO_PATH}/verrazzano 159 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} 160 """ 161 } 162 post { 163 always { 164 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 165 } 166 failure { 167 archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true 168 } 169 } 170 } 171 172 stage('Run Acceptance Tests') { 173 environment { 174 PROJECT_ADMIN_ROLE = "verrazzano-project-admin" 175 TEST_ROLE = "verrazzano-test-role" 176 } 177 stages { 178 stage ('Create test role') { 179 steps { 180 script { 181 sh """ 182 cd ${GO_REPO_PATH}/verrazzano 183 kubectl apply -f ./tools/scripts/testrole.yaml || true 184 """ 185 } 186 } 187 } 188 stage ('Run Tests') { 189 parallel { 190 stage('verify-install istio') { 191 environment { 192 TEST_ID = UUID.randomUUID().toString() 193 TEST_NAMESPACE = "installistio" 194 TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}" 195 } 196 steps { 197 runGinkgoWithKubeClient('verify-install/istio') 198 } 199 post { 200 always { 201 sh """ 202 cd ${GO_REPO_PATH}/verrazzano 203 ./tools/scripts/remove-test-kube-client.sh 204 """ 205 } 206 } 207 } 208 stage('verify-install web') { 209 environment { 210 TEST_ID = UUID.randomUUID().toString() 211 TEST_NAMESPACE = "installweb" 212 TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}" 213 } 214 steps { 215 runGinkgoWithKubeClient('verify-install/web') 216 } 217 post { 218 always { 219 sh """ 220 cd ${GO_REPO_PATH}/verrazzano 221 ./tools/scripts/remove-test-kube-client.sh 222 """ 223 } 224 } 225 } 226 stage('verify-install verrazzano') { 227 steps { 228 runGinkgoRandomize('verify-install/verrazzano') 229 } 230 } 231 stage('verify-install kubernetes') { 232 steps { 233 runGinkgoRandomize('verify-install/kubernetes') 234 } 235 } 236 stage('verify-infra restapi') { 237 environment { 238 TEST_ID = UUID.randomUUID().toString() 239 TEST_NAMESPACE = "infrarestapi" 240 TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}" 241 } 242 steps { 243 runGinkgoWithKubeClient('verify-infra/restapi') 244 } 245 post { 246 always { 247 sh """ 248 cd ${GO_REPO_PATH}/verrazzano 249 ./tools/scripts/remove-test-kube-client.sh 250 """ 251 } 252 } 253 } 254 stage('verify-infra oam') { 255 environment { 256 TEST_ID = UUID.randomUUID().toString() 257 TEST_NAMESPACE = "infraoam" 258 TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}" 259 } 260 steps { 261 runGinkgoWithKubeClient('verify-infra/oam') 262 } 263 post { 264 always { 265 sh """ 266 cd ${GO_REPO_PATH}/verrazzano 267 ./tools/scripts/remove-test-kube-client.sh 268 """ 269 } 270 } 271 } 272 stage('verify-infra vmi') { 273 steps { 274 runGinkgoRandomize('verify-infra/vmi') 275 } 276 } 277 stage('examples todo') { 278 when { 279 expression {params.RUN_EXAMPLE_TESTS == true} 280 } 281 steps { 282 runGinkgo('examples/todo') 283 } 284 } 285 stage('examples socks') { 286 environment { 287 TEST_ID = UUID.randomUUID().toString() 288 TEST_NAMESPACE = "sockshop" 289 TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}" 290 } 291 when { 292 expression {params.RUN_EXAMPLE_TESTS == true} 293 } 294 steps { 295 runGinkgoWithKubeClient('examples/socks') 296 } 297 post { 298 always { 299 sh """ 300 cd ${GO_REPO_PATH}/verrazzano 301 ./tools/scripts/remove-test-kube-client.sh 302 """ 303 } 304 } 305 } 306 stage('examples springboot') { 307 environment { 308 TEST_ID = UUID.randomUUID().toString() 309 TEST_NAMESPACE = "springboot" 310 TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}" 311 } 312 when { 313 expression {params.RUN_EXAMPLE_TESTS == true} 314 } 315 steps { 316 runGinkgoWithKubeClient('examples/springboot') 317 } 318 post { 319 always { 320 sh """ 321 cd ${GO_REPO_PATH}/verrazzano 322 ./tools/scripts/remove-test-kube-client.sh 323 """ 324 } 325 } 326 } 327 stage('examples helidon') { 328 environment { 329 TEST_ID = UUID.randomUUID().toString() 330 TEST_NAMESPACE = "hello-helidon" 331 TEST_KUBECONFIG= "/tmp/" + "${TEST_ID}" 332 } 333 when { 334 expression {params.RUN_EXAMPLE_TESTS == true} 335 } 336 steps { 337 runGinkgoWithKubeClient('examples/helidon') 338 } 339 post { 340 always { 341 sh """ 342 cd ${GO_REPO_PATH}/verrazzano 343 ./tools/scripts/remove-test-kube-client.sh 344 """ 345 } 346 } 347 } 348 stage('examples bobs books') { 349 when { 350 expression {params.RUN_EXAMPLE_TESTS == true} 351 } 352 steps { 353 runGinkgo('examples/bobsbooks') 354 } 355 } 356 } 357 } 358 } 359 post { 360 always { 361 sh """ 362 cd ${GO_REPO_PATH}/verrazzano 363 kubectl delete -f ./tools/scripts/testrole.yaml || true 364 """ 365 366 archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true 367 junit testResults: '**/*test-result.xml', allowEmptyResults: true 368 } 369 } 370 } 371 } 372 373 post { 374 failure { 375 script { 376 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 377 dumpK8sCluster('new-acceptance-tests-cluster-snapshot') 378 } 379 } 380 } 381 success { 382 script { 383 if (params.DUMP_K8S_CLUSTER_ON_SUCCESS == true && fileExists(env.TESTS_EXECUTED_FILE) ) { 384 dumpK8sCluster('new-acceptance-tests-cluster-snapshot') 385 } 386 } 387 } 388 } 389 } 390 } 391 392 post { 393 always { 394 script { 395 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 396 dumpVerrazzanoSystemPods() 397 dumpCattleSystemPods() 398 dumpNginxIngressControllerLogs() 399 dumpVerrazzanoPlatformOperatorLogs() 400 dumpVerrazzanoApplicationOperatorLogs() 401 dumpOamKubernetesRuntimeLogs() 402 dumpVerrazzanoApiLogs() 403 } 404 } 405 archiveArtifacts artifacts: '**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**', allowEmptyArchive: true 406 junit testResults: '**/*test-result.xml', allowEmptyResults: true 407 408 sh """ 409 cd ${GO_REPO_PATH}/verrazzano/platform-operator 410 make delete-cluster 411 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 412 echo "Failures seen during dumping of artifacts, treat post as failed" 413 exit 1 414 fi 415 """ 416 deleteDir() 417 } 418 } 419 } 420 421 def runGinkgoRandomize(testSuitePath) { 422 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 423 sh """ 424 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 425 ginkgo -p --randomize-all -v --keep-going --no-color ${testSuitePath}/... 426 """ 427 } 428 } 429 430 def runGinkgo(testSuitePath) { 431 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 432 sh """ 433 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 434 ginkgo -v --keep-going --no-color ${testSuitePath}/... 435 """ 436 } 437 } 438 439 def dumpK8sCluster(dumpDirectory) { 440 sh """ 441 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 442 """ 443 } 444 445 def dumpVerrazzanoSystemPods() { 446 sh """ 447 cd ${GO_REPO_PATH}/verrazzano/platform-operator 448 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log" 449 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 450 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log" 451 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 452 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log" 453 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 454 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log" 455 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 456 """ 457 } 458 459 def dumpCattleSystemPods() { 460 sh """ 461 cd ${GO_REPO_PATH}/verrazzano/platform-operator 462 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/cattle-system-pods.log" 463 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 464 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/rancher.log" 465 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 466 """ 467 } 468 469 def dumpNginxIngressControllerLogs() { 470 sh """ 471 cd ${GO_REPO_PATH}/verrazzano/platform-operator 472 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log" 473 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 474 """ 475 } 476 477 def dumpVerrazzanoPlatformOperatorLogs() { 478 sh """ 479 ## dump out verrazzano-platform-operator logs 480 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 481 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 482 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 483 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 484 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 485 echo "------------------------------------------" 486 """ 487 } 488 489 def dumpVerrazzanoApplicationOperatorLogs() { 490 sh """ 491 ## dump out verrazzano-application-operator logs 492 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 493 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 494 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 495 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 496 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 497 echo "------------------------------------------" 498 """ 499 } 500 501 def dumpOamKubernetesRuntimeLogs() { 502 sh """ 503 ## dump out oam-kubernetes-runtime logs 504 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 505 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 506 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 507 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 508 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 509 echo "------------------------------------------" 510 """ 511 } 512 513 def dumpVerrazzanoApiLogs() { 514 sh """ 515 cd ${GO_REPO_PATH}/verrazzano/platform-operator 516 export DIAGNOSTIC_LOG="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log" 517 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 518 """ 519 } 520 521 def runGinkgoWithKubeClient(testSuitePath) { 522 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 523 sh """ 524 cd ${GO_REPO_PATH}/verrazzano 525 ./tools/scripts/create-test-kube-client.sh 526 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 527 ginkgo -v --keep-going --no-color ${testSuitePath}/... 528 """ 529 } 530 }