github.com/verrazzano/verrazzano@v1.7.0/ci/oci-integration/Jenkinsfile (about) 1 // Copyright (c) 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 // This runs tests for OCI service integrations 5 6 def DEFAULT_REPO_URL 7 def testEnvironments = [ "magicdns_oke" ] 8 def installProfiles = [ "dev", "prod", "managed-cluster" ] 9 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 10 11 // pulling "ap-*" from the test regions given discovery of image pull issues 12 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-frankfurt-1", "eu-zurich-1", "uk-london-1" ] 13 Collections.shuffle(availableRegions) 14 def keepOKEClusterOnFailure = "false" 15 def OKE_CLUSTER_PREFIX = "" 16 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 17 18 pipeline { 19 options { 20 skipDefaultCheckout true 21 copyArtifactPermission('*'); 22 timestamps () 23 } 24 25 agent { 26 docker { 27 image "${RUNNER_DOCKER_IMAGE}" 28 args "${RUNNER_DOCKER_ARGS} --cap-add=NET_ADMIN" 29 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 30 label "${agentLabel}" 31 } 32 } 33 34 parameters { 35 choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION', 36 // 1st choice is the default value 37 choices: availableRegions ) 38 choice (description: 'OKE node pool configuration', name: 'OKE_NODE_POOL', 39 // 1st choice is the default value 40 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2", "VM.Standard.E2.2" ]) 41 choice (description: 'Kubernetes Version for OKE Cluster', name: 'OKE_CLUSTER_VERSION', 42 // 1st choice is the default value 43 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 44 choice (description: 'Kubernetes Version for KinD Cluster', 45 name: 'KIND_CLUSTER_VERSION', 46 // 1st choice is the default value 47 choices: ["1.27", "1.26", "1.25", "1.24" ]) 48 choice(description: 'Verrazzano Install Profile', name: "INSTALL_PROFILE", choices: installProfiles ) 49 string defaultValue: 'NONE', description: 'Verrazzano platform operator image name (within ghcr.io/verrazzano repo)', name: 'VERRAZZANO_OPERATOR_IMAGE', trim: true 50 choice (name: 'WILDCARD_DNS_DOMAIN', 51 description: 'This is the wildcard DNS domain', 52 // 1st choice is the default value 53 choices: [ "nip.io", "sslip.io"]) 54 choice (name: 'CRD_API_VERSION', 55 description: 'This is the API crd version.', 56 // 1st choice is the default value 57 choices: [ "v1beta1", "v1alpha1"]) 58 string (name: 'GIT_COMMIT_TO_USE', 59 defaultValue: 'NONE', 60 description: 'This is the full git commit hash from the source build to be used for all jobs', 61 trim: true) 62 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 63 string (name: 'TAGGED_TESTS', 64 defaultValue: '', 65 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 66 trim: true) 67 string (name: 'INCLUDED_TESTS', 68 defaultValue: '.*', 69 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 70 trim: true) 71 string (name: 'EXCLUDED_TESTS', 72 defaultValue: '_excluded_test', 73 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 74 trim: true) 75 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 76 } 77 78 environment { 79 OCR_CREDS = credentials('ocr-pull-and-push-account') 80 NETRC_FILE = credentials('netrc') 81 OCR_REPO = 'container-registry.oracle.com' 82 GHCR_REPO = 'ghcr.io' 83 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 84 TEST_ENV = "magicdns_oke" 85 INSTALL_PROFILE = "${params.INSTALL_PROFILE}" 86 GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw') 87 OCIR_CREDS = credentials('ocir-pull-and-push-account') 88 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 89 OCIR_PHX_REPO = 'phx.ocir.io' 90 POST_DUMP_FAILED = 'false' 91 GOPATH = '/home/opc/go' 92 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 93 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 94 95 DOCKER_CREDS = credentials('github-packages-credentials-rw') 96 DOCKER_EMAIL = credentials('github-packages-email') 97 DOCKER_REPO = 'ghcr.io' 98 DOCKER_NAMESPACE = 'verrazzano' 99 100 TF_VAR_tenancy_id = credentials('oci-tenancy') 101 TF_VAR_user_id = credentials('oci-user-ocid') 102 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 103 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 104 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 105 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 106 TF_VAR_api_private_key_path = credentials('oci-api-key') 107 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 108 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 109 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 110 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 111 112 OCI_CLI_TENANCY = credentials('oci-tenancy') 113 OCI_CLI_USER = credentials('oci-user-ocid') 114 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 115 OCI_CLI_KEY_FILE = credentials('oci-api-key') 116 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 117 118 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 119 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 120 OKE_KUBECONFIG = "${WORKSPACE}/test_oke_kubeconfig" 121 KIND_KUBECONFIG = "${WORKSPACE}/test_kind_kubeconfig" 122 123 DISABLE_SPINNER=1 124 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 125 126 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 127 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 128 129 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 130 131 INSTALL_CONFIG_FILE_NIPIO = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-nipio.yaml" 132 INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-no-persistence.yaml" 133 134 VZ_ENVIRONMENT_NAME = "default" 135 136 // Environment variables required to capture cluster snapshot and bug report on test failure 137 DUMP_COMMAND="${WORKSPACE}/tools/scripts/k8s-dump-cluster.sh" 138 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 139 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 140 141 // Environment variable for Verrazzano CLI executable 142 VZ_COMMAND="${GO_REPO_PATH}/vz" 143 144 // used for console artifact capture on failure and for downloading the operator.yaml file 145 JENKINS_READ = credentials('jenkins-auditor') 146 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 147 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 148 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 149 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 150 151 COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid') 152 153 // used to emit metrics 154 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 155 TEST_ENV_LABEL = "${TEST_ENV}" 156 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 157 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 158 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 159 160 // used to generate Ginkgo test reports 161 TEST_REPORT = "test-report.xml" 162 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 163 } 164 165 stages { 166 stage('Initialize') { 167 steps { 168 script { 169 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 170 if (params.GIT_COMMIT_TO_USE == "NONE") { 171 echo "Specific GIT commit was not specified, use current head" 172 def scmInfo = checkout scm 173 env.GIT_COMMIT = scmInfo.GIT_COMMIT 174 env.GIT_BRANCH = scmInfo.GIT_BRANCH 175 } else { 176 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 177 def scmInfo = checkout([ 178 $class: 'GitSCM', 179 branches: [[name: params.GIT_COMMIT_TO_USE]], 180 doGenerateSubmoduleConfigurations: false, 181 extensions: [], 182 submoduleCfg: [], 183 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 184 env.GIT_COMMIT = scmInfo.GIT_COMMIT 185 env.GIT_BRANCH = scmInfo.GIT_BRANCH 186 // If the commit we were handed is not what the SCM says we are using, fail 187 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 188 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 189 exit 1 190 } 191 } 192 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 193 } 194 195 sh """ 196 cp -f "${NETRC_FILE}" $HOME/.netrc 197 chmod 600 $HOME/.netrc 198 """ 199 println("${params.OKE_CLUSTER_REGION}") 200 println("agentlabel: ${agentLabel}") 201 sh """ 202 echo "${NODE_LABELS}" 203 """ 204 205 sh """ 206 rm -rf ${GO_REPO_PATH}/verrazzano 207 mkdir -p ${GO_REPO_PATH}/verrazzano 208 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 209 """ 210 211 script { 212 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 213 // update the description with some meaningful info 214 setDisplayName() 215 currentBuild.description = SHORT_COMMIT_HASH + " : " + params.OKE_CLUSTER_REGION + " : " + params.OKE_CLUSTER_VERSION 216 217 // derive the prefix for the OKE cluster 218 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 219 } 220 script { 221 sh """ 222 echo "Downloading VZ CLI from object storage" 223 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 224 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 225 ${GO_REPO_PATH}/vz version 226 """ 227 } 228 } 229 } 230 231 stage("Download platform operator") { 232 environment { 233 OCI_CLI_AUTH="instance_principal" 234 } 235 steps { 236 sh """ 237 echo "Download Platform Operator" 238 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 239 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 240 """ 241 } 242 post { 243 always { 244 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 245 } 246 } 247 } 248 249 stage("Create clusters and run tests") { 250 parallel { 251 stage("OKE") { 252 environment { 253 KUBECONFIG="${OKE_KUBECONFIG}" 254 DUMP_KUBECONFIG="${OKE_KUBECONFIG}" 255 } 256 stages { 257 stage("Create OCI test resources") { 258 steps { 259 script { 260 env.OKE_LOG_IDS = sh(script:"${WORKSPACE}/tests/e2e/config/scripts/create_oci_logging_resources.sh ${COMPARTMENT_ID}", returnStdout: true).trim() 261 262 env.OKE_LOG_GROUP_ID = sh(script:"echo \'${OKE_LOG_IDS}\' | jq -r '.logGroupId'", returnStdout: true).trim() 263 env.OKE_SYSTEM_LOG_ID = sh(script:"echo \'${OKE_LOG_IDS}\' | jq -r '.systemLogId'", returnStdout: true).trim() 264 env.OKE_APP_LOG_ID = sh(script:"echo \'${OKE_LOG_IDS}\' | jq -r '.appLogId'", returnStdout: true).trim() 265 env.OKE_NS_LOG_ID = sh(script:"echo \'${OKE_LOG_IDS}\' | jq -r '.nsLogId'", returnStdout: true).trim() 266 } 267 } 268 } 269 270 stage("Create OKE cluster") { 271 environment { 272 TF_VAR_label_prefix="${OKE_CLUSTER_PREFIX}" 273 } 274 steps { 275 script { 276 withCredentials([sshUserPrivateKey(credentialsId: '5fcc03de-31ce-4566-b11f-9de38e5d98fd', keyFileVariable: 'OPC_USER_KEY_FILE', passphraseVariable: 'OPC_USER_PASSPHRASE', usernameVariable: 'OPC_USERNAME')]) { 277 sh """ 278 # get the ssh public key 279 ssh-keygen -y -e -f ${OPC_USER_KEY_FILE} > /tmp/opc_ssh2.pub 280 # convert SSH2 public key into an OpenSSH format 281 ssh-keygen -i -f /tmp/opc_ssh2.pub > /tmp/opc_ssh.pub 282 # set the ssh public key value for terraform 283 export TF_VAR_ssh_public_key_path=/tmp/opc_ssh.pub 284 export TF_VAR_state_name=${env.BUILD_NUMBER}-${env.TIMESTAMP}-${env.BRANCH_NAME} 285 # call create_oke_cluster with cluster access private 286 ${WORKSPACE}/tests/e2e/config/scripts/create_oke_cluster.sh true false 287 """ 288 } 289 } 290 } 291 post { 292 failure { 293 script { 294 echo "Cluster create failed" 295 } 296 } 297 } 298 } 299 300 stage("Create image pull secrets") { 301 steps { 302 script { 303 createImagePullSecrets() 304 } 305 } 306 } 307 308 stage("Install platform operator") { 309 environment { 310 OCI_CLI_AUTH="instance_principal" 311 } 312 steps { 313 sh """ 314 echo "Install Platform Operator" 315 kubectl apply -f ${WORKSPACE}/acceptance-test-operator.yaml 316 317 # make sure ns exists 318 ${WORKSPACE}/tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 319 # create secret in verrazzano-install ns 320 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" "verrazzano-install" 321 """ 322 } 323 } 324 325 stage("Process install YAML") { 326 environment { 327 SYSTEM_LOG_ID="${OKE_SYSTEM_LOG_ID}" 328 APP_LOG_ID="${OKE_APP_LOG_ID}" 329 } 330 steps { 331 script { 332 sh """ 333 ${WORKSPACE}/tests/e2e/config/scripts/process_nipio_install_yaml.sh $INSTALL_CONFIG_FILE_NIPIO 334 """ 335 } 336 } 337 } 338 339 stage("Install Verrazzano") { 340 steps { 341 sh """ 342 echo "Waiting for Operator to be ready" 343 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 344 """ 345 346 sh """ 347 echo "Installing Verrazzano on ${TEST_ENV}" 348 # apply config to create cluster 349 350 kubectl apply -f ${INSTALL_CONFIG_FILE_NIPIO} 351 352 # wait for Verrazzano install to complete 353 ${WORKSPACE}/tests/e2e/config/scripts/wait-for-verrazzano-install.sh 354 # Create acceptance test configuration file 355 ${WORKSPACE}/tests/e2e/config/scripts/common-test-setup-script.sh "${WORKSPACE}" "${TEST_CONFIG_FILE}" "${env.DOCKER_REPO}" "${KUBECONFIG}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" "${VZ_ENVIRONMENT_NAME}" 356 357 # edit DNS info in the test config file 358 ${WORKSPACE}/tests/e2e/config/scripts/get_ingress_ip.sh ${TEST_CONFIG_FILE} 359 echo "----------Test config file:-------------" 360 cat ${TEST_CONFIG_FILE} 361 echo "----------------------------------------" 362 """ 363 } 364 post { 365 always { 366 script { 367 sh """ 368 # dump out install logs 369 mkdir -p ${WORKSPACE}/platform-operator/scripts/install/build/logs/oke 370 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/platform-operator/scripts/install/build/logs/oke/verrazzano-platform-operator.log --tail -1 371 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/platform-operator/scripts/install/build/logs/oke/verrazzano-platform-operator-pod.out 372 echo "Verrazzano platform operator logs dumped to verrazzano-platform-operator.log" 373 echo "Verrazzano platform operator pod description dumped to verrazzano-platform-operator-pod.out" 374 echo "------------------------------------------" 375 """ 376 } 377 } 378 } 379 } 380 381 stage('verify-install verrazzano') { 382 steps { 383 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 384 sh """ 385 cd ${WORKSPACE}/tests/e2e 386 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/verrazzano/ 387 """ 388 } 389 } 390 } 391 392 stage('OCI logging') { 393 environment { 394 LOG_GROUP_ID="${OKE_LOG_GROUP_ID}" 395 NS_LOG_ID="${OKE_NS_LOG_ID}" 396 397 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/oci-logging-oke" 398 } 399 steps { 400 script { 401 runGinkgo('oci/logging') 402 } 403 } 404 } 405 } 406 post { 407 always { 408 sh ''' 409 # Clean up the OCI Logging resources we created 410 ${WORKSPACE}/tests/e2e/config/scripts/delete_oci_logging_resources.sh "${OKE_LOG_GROUP_ID}" "${OKE_SYSTEM_LOG_ID}" "${OKE_APP_LOG_ID}" "${OKE_NS_LOG_ID}" || true 411 ''' 412 413 script { 414 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true || currentBuild.currentResult == 'FAILURE') { 415 dumpK8sCluster('oci-integration-tests-oke-cluster-snapshot') 416 } 417 } 418 419 sh """ 420 # Destroy the OKE cluster 421 if [ "${keepOKEClusterOnFailure}" == "false" ]; then 422 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=${env.BUILD_NUMBER}-${env.TIMESTAMP}-${env.BRANCH_NAME} ${WORKSPACE}/tests/e2e/config/scripts/delete_oke_cluster.sh || true 423 fi 424 """ 425 } 426 } 427 } 428 429 stage("KinD") { 430 environment { 431 KUBECONFIG="${KIND_KUBECONFIG}" 432 DUMP_KUBECONFIG="${KIND_KUBECONFIG}" 433 OCI_OS_LOCATION="${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 434 CLUSTER_NAME="verrazzano" 435 } 436 stages { 437 stage("Create OCI test resources") { 438 steps { 439 script { 440 env.KIND_LOG_IDS = sh(script:"${WORKSPACE}/tests/e2e/config/scripts/create_oci_logging_resources.sh ${COMPARTMENT_ID}", returnStdout: true).trim() 441 442 env.KIND_LOG_GROUP_ID = sh(script:"echo \'${KIND_LOG_IDS}\' | jq -r '.logGroupId'", returnStdout: true).trim() 443 env.KIND_SYSTEM_LOG_ID = sh(script:"echo \'${KIND_LOG_IDS}\' | jq -r '.systemLogId'", returnStdout: true).trim() 444 env.KIND_APP_LOG_ID = sh(script:"echo \'${KIND_LOG_IDS}\' | jq -r '.appLogId'", returnStdout: true).trim() 445 env.KIND_NS_LOG_ID = sh(script:"echo \'${KIND_LOG_IDS}\' | jq -r '.nsLogId'", returnStdout: true).trim() 446 } 447 } 448 } 449 450 stage("Create KinD cluster") { 451 environment { 452 KIND_KUBERNETES_CLUSTER_VERSION="${params.KIND_CLUSTER_VERSION}" 453 OCI_CLI_AUTH="instance_principal" 454 } 455 steps { 456 sh """ 457 echo "Using KUBECONFIG: ${KUBECONFIG}" 458 459 echo "Create Kind cluster" 460 cd ${TEST_SCRIPTS_DIR} 461 ./create_kind_cluster.sh ${CLUSTER_NAME} ${GO_REPO_PATH}/verrazzano/platform-operator ${KUBECONFIG} ${KIND_KUBERNETES_CLUSTER_VERSION} true true true false "NONE" 1 462 if [ \$? -ne 0 ]; then 463 mkdir ${WORKSPACE}/kind-logs 464 kind export logs ${WORKSPACE}/kind-logs 465 echo "Kind cluster creation failed" 466 exit 1 467 fi 468 469 kubectl wait --for=condition=ready nodes/${CLUSTER_NAME}-control-plane --timeout=5m --all 470 kubectl wait --for=condition=ready pods/kube-controller-manager-${CLUSTER_NAME}-control-plane -n kube-system --timeout=5m 471 472 echo "Install metallb" 473 cd ${GO_REPO_PATH}/verrazzano 474 ./tests/e2e/config/scripts/install-metallb.sh 475 """ 476 } 477 } 478 479 stage("Create image pull secrets") { 480 steps { 481 script { 482 createImagePullSecrets() 483 } 484 } 485 } 486 487 stage("Install platform operator") { 488 environment { 489 OCI_CLI_AUTH="instance_principal" 490 } 491 steps { 492 sh """ 493 echo "Install Platform Operator" 494 kubectl apply -f ${WORKSPACE}/acceptance-test-operator.yaml 495 496 # make sure ns exists 497 ${WORKSPACE}/tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 498 # create secret in verrazzano-install ns 499 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" "verrazzano-install" 500 """ 501 } 502 } 503 504 stage("Process install YAML") { 505 environment { 506 SYSTEM_LOG_ID="${KIND_SYSTEM_LOG_ID}" 507 APP_LOG_ID="${KIND_APP_LOG_ID}" 508 } 509 steps { 510 script { 511 sh """ 512 ${WORKSPACE}/tests/e2e/config/scripts/process_kind_install_yaml.sh "${INSTALL_CONFIG_FILE_KIND}" "${params.WILDCARD_DNS_DOMAIN}" 513 """ 514 } 515 } 516 } 517 518 stage("Create OCI API secret") { 519 environment { 520 OUTPUT_FILE="${WORKSPACE}/oci_config" 521 KEY_FILE="${WORKSPACE}/oci_key" 522 } 523 steps { 524 script { 525 sh """ 526 echo "Creating OCI config files" 527 echo "[DEFAULT]" > $OUTPUT_FILE 528 echo "region=${OCI_CLI_REGION}" >> $OUTPUT_FILE 529 echo "tenancy=${OCI_CLI_TENANCY}" >> $OUTPUT_FILE 530 echo "user=${OCI_CLI_USER}" >> $OUTPUT_FILE 531 echo "fingerprint=${OCI_CLI_FINGERPRINT}" >> $OUTPUT_FILE 532 echo "key_file=$KEY_FILE" >> $OUTPUT_FILE 533 cat ${OCI_CLI_KEY_FILE} > $KEY_FILE 534 535 echo "Creating OCI API secret from OCI config" 536 ${WORKSPACE}/platform-operator/scripts/install/create_oci_fluentd_secret.sh -o ${OUTPUT_FILE} 537 """ 538 } 539 } 540 post { 541 always { 542 sh """ 543 rm -f $OUTPUT_FILE $KEY_FILE 544 """ 545 } 546 } 547 } 548 549 stage("Install Verrazzano") { 550 steps { 551 sh """ 552 echo "Waiting for Operator to be ready" 553 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 554 """ 555 556 sh """ 557 echo "Installing Verrazzano on KinD" 558 kubectl apply -f ${INSTALL_CONFIG_FILE_KIND} 559 560 # wait for Verrazzano install to complete 561 ${WORKSPACE}/tests/e2e/config/scripts/wait-for-verrazzano-install.sh 562 """ 563 } 564 post { 565 always { 566 script { 567 sh """ 568 # dump out install logs 569 mkdir -p ${WORKSPACE}/platform-operator/scripts/install/build/logs/kind 570 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/platform-operator/scripts/install/build/logs/kind/verrazzano-platform-operator.log --tail -1 571 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/platform-operator/scripts/install/build/logs/kind/verrazzano-platform-operator-pod.out 572 echo "Verrazzano platform operator logs dumped to verrazzano-platform-operator.log" 573 echo "Verrazzano platform operator pod description dumped to verrazzano-platform-operator-pod.out" 574 echo "------------------------------------------" 575 """ 576 } 577 } 578 } 579 } 580 581 stage('verify-install verrazzano') { 582 steps { 583 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 584 sh """ 585 cd ${WORKSPACE}/tests/e2e 586 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/verrazzano/ 587 """ 588 } 589 } 590 } 591 592 stage('OCI logging') { 593 environment { 594 LOG_GROUP_ID="${KIND_LOG_GROUP_ID}" 595 NS_LOG_ID="${KIND_NS_LOG_ID}" 596 597 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/oci-logging-kind" 598 } 599 steps { 600 script { 601 runGinkgo('oci/logging') 602 } 603 } 604 } 605 } 606 post { 607 always { 608 sh ''' 609 # Clean up the OCI Logging resources we created 610 ${WORKSPACE}/tests/e2e/config/scripts/delete_oci_logging_resources.sh "${KIND_LOG_GROUP_ID}" "${KIND_SYSTEM_LOG_ID}" "${KIND_APP_LOG_ID}" "${KIND_NS_LOG_ID}" || true 611 ''' 612 613 script { 614 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true || currentBuild.currentResult == 'FAILURE') { 615 dumpK8sCluster('oci-integration-tests-kind-cluster-snapshot') 616 } 617 } 618 619 sh """ 620 # Delete the KinD cluster 621 kind delete cluster --name "${CLUSTER_NAME}" 622 """ 623 } 624 } 625 } 626 } 627 } 628 } 629 post { 630 always { 631 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/test-cluster-snapshots/**,**/${TEST_REPORT}", allowEmptyArchive: true 632 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 633 } 634 failure { 635 sh """ 636 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 637 """ 638 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 639 sh """ 640 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 641 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 642 rm archive.zip 643 """ 644 } 645 cleanup { 646 deleteDir() 647 } 648 } 649 } 650 651 def createImagePullSecrets() { 652 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 653 sh """ 654 # Create image pull secret for Verrazzano docker images 655 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 656 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 657 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 658 """ 659 } 660 } 661 662 def runGinkgo(testSuitePath) { 663 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 664 sh """ 665 cd ${WORKSPACE}/tests/e2e 666 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 667 """ 668 } 669 } 670 671 def dumpK8sCluster(dumpDirectory) { 672 sh """ 673 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 674 """ 675 } 676 677 def getEffectiveDumpOnSuccess() { 678 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 679 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 680 effectiveValue = true 681 echo "Forcing dump on success based on global override setting" 682 } 683 return effectiveValue 684 } 685 686 def setDisplayName() { 687 echo "Start setDisplayName" 688 def causes = currentBuild.getBuildCauses() 689 echo "causes: " + causes.toString() 690 for (cause in causes) { 691 def causeString = cause.toString() 692 echo "current cause: " + causeString 693 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 694 echo "This job was caused by " + causeString 695 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 696 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 697 } else if (causeString.contains("verrazzano-flaky-tests")) { 698 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 699 } 700 } 701 } 702 echo "End setDisplayName" 703 }