github.com/verrazzano/verrazzano@v1.7.1/ci/oci-integration/Jenkinsfile (about) 1 // Copyright (c) 2023, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 // This runs tests for OCI service integrations 5 6 def DEFAULT_REPO_URL 7 def testEnvironments = [ "magicdns_oke" ] 8 def installProfiles = [ "dev", "prod", "managed-cluster" ] 9 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 10 11 // pulling "ap-*" from the test regions given discovery of image pull issues 12 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ] 13 Collections.shuffle(availableRegions) 14 def keepOKEClusterOnFailure = "false" 15 def OKE_CLUSTER_PREFIX = "" 16 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 17 18 pipeline { 19 options { 20 skipDefaultCheckout true 21 copyArtifactPermission('*'); 22 timestamps () 23 } 24 25 agent { 26 docker { 27 image "${RUNNER_DOCKER_IMAGE}" 28 args "${RUNNER_DOCKER_ARGS} --cap-add=NET_ADMIN" 29 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 30 label "${agentLabel}" 31 } 32 } 33 34 parameters { 35 choice (description: 'OCI region to launch OKE clusters in', name: 'OKE_CLUSTER_REGION', 36 // 1st choice is the default value 37 choices: availableRegions ) 38 choice (description: 'OKE node pool configuration', name: 'OKE_NODE_POOL', 39 // 1st choice is the default value 40 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2", "VM.Standard.E2.2" ]) 41 choice (description: 'Kubernetes Version for OKE Cluster', name: 'OKE_CLUSTER_VERSION', 42 // 1st choice is the default value 43 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 44 choice (description: 'Kubernetes Version for KinD Cluster', 45 name: 'KIND_CLUSTER_VERSION', 46 // 1st choice is the default value 47 choices: ["1.27", "1.26", "1.25", "1.24" ]) 48 choice(description: 'Verrazzano Install Profile', name: "INSTALL_PROFILE", choices: installProfiles ) 49 string defaultValue: 'NONE', description: 'Verrazzano platform operator image name (within ghcr.io/verrazzano repo)', name: 'VERRAZZANO_OPERATOR_IMAGE', trim: true 50 choice (name: 'WILDCARD_DNS_DOMAIN', 51 description: 'This is the wildcard DNS domain', 52 // 1st choice is the default value 53 choices: [ "nip.io", "sslip.io"]) 54 choice (name: 'CRD_API_VERSION', 55 description: 'This is the API crd version.', 56 // 1st choice is the default value 57 choices: [ "v1beta1", "v1alpha1"]) 58 string (name: 'GIT_COMMIT_TO_USE', 59 defaultValue: 'NONE', 60 description: 'This is the full git commit hash from the source build to be used for all jobs', 61 trim: true) 62 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 63 string (name: 'TAGGED_TESTS', 64 defaultValue: '', 65 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 66 trim: true) 67 string (name: 'INCLUDED_TESTS', 68 defaultValue: '.*', 69 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 70 trim: true) 71 string (name: 'EXCLUDED_TESTS', 72 defaultValue: '_excluded_test', 73 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 74 trim: true) 75 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 76 } 77 78 environment { 79 OCR_CREDS = credentials('ocr-pull-and-push-account') 80 NETRC_FILE = credentials('netrc') 81 OCR_REPO = 'container-registry.oracle.com' 82 GHCR_REPO = 'ghcr.io' 83 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 84 TEST_ENV = "magicdns_oke" 85 INSTALL_PROFILE = "${params.INSTALL_PROFILE}" 86 GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw') 87 OCIR_CREDS = credentials('ocir-pull-and-push-account') 88 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 89 OCIR_PHX_REPO = 'phx.ocir.io' 90 POST_DUMP_FAILED = 'false' 91 GOPATH = '/home/opc/go' 92 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 93 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 94 95 DOCKER_CREDS = credentials('github-packages-credentials-rw') 96 DOCKER_EMAIL = credentials('github-packages-email') 97 DOCKER_REPO = 'ghcr.io' 98 DOCKER_NAMESPACE = 'verrazzano' 99 100 TF_VAR_tenancy_id = credentials('oci-tenancy') 101 TF_VAR_user_id = credentials('oci-user-ocid') 102 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 103 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 104 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 105 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 106 TF_VAR_api_private_key_path = credentials('oci-api-key') 107 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 108 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 109 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 110 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 111 112 OCI_CLI_TENANCY = credentials('oci-tenancy') 113 OCI_CLI_USER = credentials('oci-user-ocid') 114 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 115 OCI_CLI_KEY_FILE = credentials('oci-api-key') 116 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 117 118 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 119 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 120 OKE_KUBECONFIG = "${WORKSPACE}/test_oke_kubeconfig" 121 KIND_KUBECONFIG = "${WORKSPACE}/test_kind_kubeconfig" 122 123 DISABLE_SPINNER=1 124 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 125 126 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 127 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 128 129 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 130 131 INSTALL_CONFIG_FILE_NIPIO = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-nipio.yaml" 132 INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind-no-persistence.yaml" 133 134 VZ_ENVIRONMENT_NAME = "default" 135 136 // Environment variables required to capture cluster snapshot and bug report on test failure 137 DUMP_COMMAND="${WORKSPACE}/tools/scripts/k8s-dump-cluster.sh" 138 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 139 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 140 141 // Environment variable for Verrazzano CLI executable 142 VZ_COMMAND="${GO_REPO_PATH}/vz" 143 144 // used for console artifact capture on failure and for downloading the operator.yaml file 145 JENKINS_READ = credentials('jenkins-auditor') 146 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 147 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 148 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 149 150 COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid') 151 152 // used to emit metrics 153 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 154 TEST_ENV_LABEL = "${TEST_ENV}" 155 156 // used to generate Ginkgo test reports 157 TEST_REPORT = "test-report.xml" 158 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 159 } 160 161 stages { 162 stage('Initialize') { 163 steps { 164 script { 165 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 166 if (params.GIT_COMMIT_TO_USE == "NONE") { 167 echo "Specific GIT commit was not specified, use current head" 168 def scmInfo = checkout scm 169 env.GIT_COMMIT = scmInfo.GIT_COMMIT 170 env.GIT_BRANCH = scmInfo.GIT_BRANCH 171 } else { 172 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 173 def scmInfo = checkout([ 174 $class: 'GitSCM', 175 branches: [[name: params.GIT_COMMIT_TO_USE]], 176 doGenerateSubmoduleConfigurations: false, 177 extensions: [], 178 submoduleCfg: [], 179 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 180 env.GIT_COMMIT = scmInfo.GIT_COMMIT 181 env.GIT_BRANCH = scmInfo.GIT_BRANCH 182 // If the commit we were handed is not what the SCM says we are using, fail 183 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 184 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 185 exit 1 186 } 187 } 188 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 189 } 190 191 sh """ 192 cp -f "${NETRC_FILE}" $HOME/.netrc 193 chmod 600 $HOME/.netrc 194 """ 195 println("${params.OKE_CLUSTER_REGION}") 196 println("agentlabel: ${agentLabel}") 197 sh """ 198 echo "${NODE_LABELS}" 199 """ 200 201 sh """ 202 rm -rf ${GO_REPO_PATH}/verrazzano 203 mkdir -p ${GO_REPO_PATH}/verrazzano 204 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 205 """ 206 207 script { 208 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 209 // update the description with some meaningful info 210 setDisplayName() 211 currentBuild.description = SHORT_COMMIT_HASH + " : " + params.OKE_CLUSTER_REGION + " : " + params.OKE_CLUSTER_VERSION 212 213 // derive the prefix for the OKE cluster 214 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 215 } 216 script { 217 sh """ 218 echo "Downloading VZ CLI from object storage" 219 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 220 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 221 ${GO_REPO_PATH}/vz version 222 """ 223 } 224 } 225 } 226 227 stage("Download platform operator") { 228 environment { 229 OCI_CLI_AUTH="instance_principal" 230 } 231 steps { 232 sh """ 233 echo "Download Platform Operator" 234 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 235 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 236 """ 237 } 238 post { 239 always { 240 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 241 } 242 } 243 } 244 245 stage("Create clusters and run tests") { 246 parallel { 247 stage("OKE") { 248 environment { 249 KUBECONFIG="${OKE_KUBECONFIG}" 250 DUMP_KUBECONFIG="${OKE_KUBECONFIG}" 251 } 252 stages { 253 stage("Create OCI test resources") { 254 steps { 255 script { 256 env.OKE_LOG_IDS = sh(script:"${WORKSPACE}/tests/e2e/config/scripts/create_oci_logging_resources.sh ${COMPARTMENT_ID}", returnStdout: true).trim() 257 258 env.OKE_LOG_GROUP_ID = sh(script:"echo \'${OKE_LOG_IDS}\' | jq -r '.logGroupId'", returnStdout: true).trim() 259 env.OKE_SYSTEM_LOG_ID = sh(script:"echo \'${OKE_LOG_IDS}\' | jq -r '.systemLogId'", returnStdout: true).trim() 260 env.OKE_APP_LOG_ID = sh(script:"echo \'${OKE_LOG_IDS}\' | jq -r '.appLogId'", returnStdout: true).trim() 261 env.OKE_NS_LOG_ID = sh(script:"echo \'${OKE_LOG_IDS}\' | jq -r '.nsLogId'", returnStdout: true).trim() 262 } 263 } 264 } 265 266 stage("Create OKE cluster") { 267 environment { 268 TF_VAR_label_prefix="${OKE_CLUSTER_PREFIX}" 269 } 270 steps { 271 script { 272 withCredentials([sshUserPrivateKey(credentialsId: '5fcc03de-31ce-4566-b11f-9de38e5d98fd', keyFileVariable: 'OPC_USER_KEY_FILE', passphraseVariable: 'OPC_USER_PASSPHRASE', usernameVariable: 'OPC_USERNAME')]) { 273 sh """ 274 # get the ssh public key 275 ssh-keygen -y -e -f ${OPC_USER_KEY_FILE} > /tmp/opc_ssh2.pub 276 # convert SSH2 public key into an OpenSSH format 277 ssh-keygen -i -f /tmp/opc_ssh2.pub > /tmp/opc_ssh.pub 278 # set the ssh public key value for terraform 279 export TF_VAR_ssh_public_key_path=/tmp/opc_ssh.pub 280 export TF_VAR_state_name=${env.BUILD_NUMBER}-${env.TIMESTAMP}-${env.BRANCH_NAME} 281 # call create_oke_cluster with cluster access private 282 ${WORKSPACE}/tests/e2e/config/scripts/create_oke_cluster.sh true false 283 """ 284 } 285 } 286 } 287 post { 288 failure { 289 script { 290 echo "Cluster create failed" 291 } 292 } 293 } 294 } 295 296 stage("Create image pull secrets") { 297 steps { 298 script { 299 createImagePullSecrets() 300 } 301 } 302 } 303 304 stage("Install platform operator") { 305 environment { 306 OCI_CLI_AUTH="instance_principal" 307 } 308 steps { 309 sh """ 310 echo "Install Platform Operator" 311 kubectl apply -f ${WORKSPACE}/acceptance-test-operator.yaml 312 313 # make sure ns exists 314 ${WORKSPACE}/tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 315 # create secret in verrazzano-install ns 316 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" "verrazzano-install" 317 """ 318 } 319 } 320 321 stage("Process install YAML") { 322 environment { 323 SYSTEM_LOG_ID="${OKE_SYSTEM_LOG_ID}" 324 APP_LOG_ID="${OKE_APP_LOG_ID}" 325 } 326 steps { 327 script { 328 sh """ 329 ${WORKSPACE}/tests/e2e/config/scripts/process_nipio_install_yaml.sh $INSTALL_CONFIG_FILE_NIPIO 330 """ 331 } 332 } 333 } 334 335 stage("Install Verrazzano") { 336 steps { 337 sh """ 338 echo "Waiting for Operator to be ready" 339 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 340 """ 341 342 sh """ 343 echo "Installing Verrazzano on ${TEST_ENV}" 344 # apply config to create cluster 345 346 kubectl apply -f ${INSTALL_CONFIG_FILE_NIPIO} 347 348 # wait for Verrazzano install to complete 349 ${WORKSPACE}/tests/e2e/config/scripts/wait-for-verrazzano-install.sh 350 # Create acceptance test configuration file 351 ${WORKSPACE}/tests/e2e/config/scripts/common-test-setup-script.sh "${WORKSPACE}" "${TEST_CONFIG_FILE}" "${env.DOCKER_REPO}" "${KUBECONFIG}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" "${VZ_ENVIRONMENT_NAME}" 352 353 # edit DNS info in the test config file 354 ${WORKSPACE}/tests/e2e/config/scripts/get_ingress_ip.sh ${TEST_CONFIG_FILE} 355 echo "----------Test config file:-------------" 356 cat ${TEST_CONFIG_FILE} 357 echo "----------------------------------------" 358 """ 359 } 360 post { 361 always { 362 script { 363 sh """ 364 # dump out install logs 365 mkdir -p ${WORKSPACE}/platform-operator/scripts/install/build/logs/oke 366 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/platform-operator/scripts/install/build/logs/oke/verrazzano-platform-operator.log --tail -1 367 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/platform-operator/scripts/install/build/logs/oke/verrazzano-platform-operator-pod.out 368 echo "Verrazzano platform operator logs dumped to verrazzano-platform-operator.log" 369 echo "Verrazzano platform operator pod description dumped to verrazzano-platform-operator-pod.out" 370 echo "------------------------------------------" 371 """ 372 } 373 } 374 } 375 } 376 377 stage('verify-install verrazzano') { 378 steps { 379 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 380 sh """ 381 cd ${WORKSPACE}/tests/e2e 382 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/verrazzano/ 383 """ 384 } 385 } 386 } 387 388 stage('OCI logging') { 389 environment { 390 LOG_GROUP_ID="${OKE_LOG_GROUP_ID}" 391 NS_LOG_ID="${OKE_NS_LOG_ID}" 392 393 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/oci-logging-oke" 394 } 395 steps { 396 script { 397 runGinkgo('oci/logging') 398 } 399 } 400 } 401 } 402 post { 403 always { 404 sh ''' 405 # Clean up the OCI Logging resources we created 406 ${WORKSPACE}/tests/e2e/config/scripts/delete_oci_logging_resources.sh "${OKE_LOG_GROUP_ID}" "${OKE_SYSTEM_LOG_ID}" "${OKE_APP_LOG_ID}" "${OKE_NS_LOG_ID}" || true 407 ''' 408 409 script { 410 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true || currentBuild.currentResult == 'FAILURE') { 411 dumpK8sCluster('oci-integration-tests-oke-cluster-snapshot') 412 } 413 } 414 415 sh """ 416 # Destroy the OKE cluster 417 if [ "${keepOKEClusterOnFailure}" == "false" ]; then 418 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=${env.BUILD_NUMBER}-${env.TIMESTAMP}-${env.BRANCH_NAME} ${WORKSPACE}/tests/e2e/config/scripts/delete_oke_cluster.sh || true 419 fi 420 """ 421 } 422 } 423 } 424 425 stage("KinD") { 426 environment { 427 KUBECONFIG="${KIND_KUBECONFIG}" 428 DUMP_KUBECONFIG="${KIND_KUBECONFIG}" 429 OCI_OS_LOCATION="${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 430 CLUSTER_NAME="verrazzano" 431 } 432 stages { 433 stage("Create OCI test resources") { 434 steps { 435 script { 436 env.KIND_LOG_IDS = sh(script:"${WORKSPACE}/tests/e2e/config/scripts/create_oci_logging_resources.sh ${COMPARTMENT_ID}", returnStdout: true).trim() 437 438 env.KIND_LOG_GROUP_ID = sh(script:"echo \'${KIND_LOG_IDS}\' | jq -r '.logGroupId'", returnStdout: true).trim() 439 env.KIND_SYSTEM_LOG_ID = sh(script:"echo \'${KIND_LOG_IDS}\' | jq -r '.systemLogId'", returnStdout: true).trim() 440 env.KIND_APP_LOG_ID = sh(script:"echo \'${KIND_LOG_IDS}\' | jq -r '.appLogId'", returnStdout: true).trim() 441 env.KIND_NS_LOG_ID = sh(script:"echo \'${KIND_LOG_IDS}\' | jq -r '.nsLogId'", returnStdout: true).trim() 442 } 443 } 444 } 445 446 stage("Create KinD cluster") { 447 environment { 448 KIND_KUBERNETES_CLUSTER_VERSION="${params.KIND_CLUSTER_VERSION}" 449 OCI_CLI_AUTH="instance_principal" 450 } 451 steps { 452 sh """ 453 echo "Using KUBECONFIG: ${KUBECONFIG}" 454 455 echo "Create Kind cluster" 456 cd ${TEST_SCRIPTS_DIR} 457 ./create_kind_cluster.sh ${CLUSTER_NAME} ${GO_REPO_PATH}/verrazzano/platform-operator ${KUBECONFIG} ${KIND_KUBERNETES_CLUSTER_VERSION} true true true false "NONE" 1 458 if [ \$? -ne 0 ]; then 459 mkdir ${WORKSPACE}/kind-logs 460 kind export logs ${WORKSPACE}/kind-logs 461 echo "Kind cluster creation failed" 462 exit 1 463 fi 464 465 kubectl wait --for=condition=ready nodes/${CLUSTER_NAME}-control-plane --timeout=5m --all 466 kubectl wait --for=condition=ready pods/kube-controller-manager-${CLUSTER_NAME}-control-plane -n kube-system --timeout=5m 467 468 echo "Install metallb" 469 cd ${GO_REPO_PATH}/verrazzano 470 ./tests/e2e/config/scripts/install-metallb.sh 471 """ 472 } 473 } 474 475 stage("Create image pull secrets") { 476 steps { 477 script { 478 createImagePullSecrets() 479 } 480 } 481 } 482 483 stage("Install platform operator") { 484 environment { 485 OCI_CLI_AUTH="instance_principal" 486 } 487 steps { 488 sh """ 489 echo "Install Platform Operator" 490 kubectl apply -f ${WORKSPACE}/acceptance-test-operator.yaml 491 492 # make sure ns exists 493 ${WORKSPACE}/tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 494 # create secret in verrazzano-install ns 495 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" "verrazzano-install" 496 """ 497 } 498 } 499 500 stage("Process install YAML") { 501 environment { 502 SYSTEM_LOG_ID="${KIND_SYSTEM_LOG_ID}" 503 APP_LOG_ID="${KIND_APP_LOG_ID}" 504 } 505 steps { 506 script { 507 sh """ 508 ${WORKSPACE}/tests/e2e/config/scripts/process_kind_install_yaml.sh "${INSTALL_CONFIG_FILE_KIND}" "${params.WILDCARD_DNS_DOMAIN}" 509 """ 510 } 511 } 512 } 513 514 stage("Create OCI API secret") { 515 environment { 516 OUTPUT_FILE="${WORKSPACE}/oci_config" 517 KEY_FILE="${WORKSPACE}/oci_key" 518 } 519 steps { 520 script { 521 sh """ 522 echo "Creating OCI config files" 523 echo "[DEFAULT]" > $OUTPUT_FILE 524 echo "region=${OCI_CLI_REGION}" >> $OUTPUT_FILE 525 echo "tenancy=${OCI_CLI_TENANCY}" >> $OUTPUT_FILE 526 echo "user=${OCI_CLI_USER}" >> $OUTPUT_FILE 527 echo "fingerprint=${OCI_CLI_FINGERPRINT}" >> $OUTPUT_FILE 528 echo "key_file=$KEY_FILE" >> $OUTPUT_FILE 529 cat ${OCI_CLI_KEY_FILE} > $KEY_FILE 530 531 echo "Creating OCI API secret from OCI config" 532 ${WORKSPACE}/platform-operator/scripts/install/create_oci_fluentd_secret.sh -o ${OUTPUT_FILE} 533 """ 534 } 535 } 536 post { 537 always { 538 sh """ 539 rm -f $OUTPUT_FILE $KEY_FILE 540 """ 541 } 542 } 543 } 544 545 stage("Install Verrazzano") { 546 steps { 547 sh """ 548 echo "Waiting for Operator to be ready" 549 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 550 """ 551 552 sh """ 553 echo "Installing Verrazzano on KinD" 554 kubectl apply -f ${INSTALL_CONFIG_FILE_KIND} 555 556 # wait for Verrazzano install to complete 557 ${WORKSPACE}/tests/e2e/config/scripts/wait-for-verrazzano-install.sh 558 """ 559 } 560 post { 561 always { 562 script { 563 sh """ 564 # dump out install logs 565 mkdir -p ${WORKSPACE}/platform-operator/scripts/install/build/logs/kind 566 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/platform-operator/scripts/install/build/logs/kind/verrazzano-platform-operator.log --tail -1 567 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/platform-operator/scripts/install/build/logs/kind/verrazzano-platform-operator-pod.out 568 echo "Verrazzano platform operator logs dumped to verrazzano-platform-operator.log" 569 echo "Verrazzano platform operator pod description dumped to verrazzano-platform-operator-pod.out" 570 echo "------------------------------------------" 571 """ 572 } 573 } 574 } 575 } 576 577 stage('verify-install verrazzano') { 578 steps { 579 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 580 sh """ 581 cd ${WORKSPACE}/tests/e2e 582 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/verrazzano/ 583 """ 584 } 585 } 586 } 587 588 stage('OCI logging') { 589 environment { 590 LOG_GROUP_ID="${KIND_LOG_GROUP_ID}" 591 NS_LOG_ID="${KIND_NS_LOG_ID}" 592 593 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/oci-logging-kind" 594 } 595 steps { 596 script { 597 runGinkgo('oci/logging') 598 } 599 } 600 } 601 } 602 post { 603 always { 604 sh ''' 605 # Clean up the OCI Logging resources we created 606 ${WORKSPACE}/tests/e2e/config/scripts/delete_oci_logging_resources.sh "${KIND_LOG_GROUP_ID}" "${KIND_SYSTEM_LOG_ID}" "${KIND_APP_LOG_ID}" "${KIND_NS_LOG_ID}" || true 607 ''' 608 609 script { 610 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true || currentBuild.currentResult == 'FAILURE') { 611 dumpK8sCluster('oci-integration-tests-kind-cluster-snapshot') 612 } 613 } 614 615 sh """ 616 # Delete the KinD cluster 617 kind delete cluster --name "${CLUSTER_NAME}" 618 """ 619 } 620 } 621 } 622 } 623 } 624 } 625 post { 626 always { 627 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/test-cluster-snapshots/**,**/${TEST_REPORT}", allowEmptyArchive: true 628 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 629 } 630 failure { 631 sh """ 632 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 633 """ 634 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 635 sh """ 636 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 637 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 638 rm archive.zip 639 """ 640 } 641 cleanup { 642 deleteDir() 643 } 644 } 645 } 646 647 def createImagePullSecrets() { 648 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 649 sh """ 650 # Create image pull secret for Verrazzano docker images 651 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 652 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 653 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 654 """ 655 } 656 } 657 658 def runGinkgo(testSuitePath) { 659 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 660 sh """ 661 cd ${WORKSPACE}/tests/e2e 662 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 663 """ 664 } 665 } 666 667 def dumpK8sCluster(dumpDirectory) { 668 sh """ 669 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 670 """ 671 } 672 673 def getEffectiveDumpOnSuccess() { 674 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 675 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 676 effectiveValue = true 677 echo "Forcing dump on success based on global override setting" 678 } 679 return effectiveValue 680 } 681 682 def setDisplayName() { 683 echo "Start setDisplayName" 684 def causes = currentBuild.getBuildCauses() 685 echo "causes: " + causes.toString() 686 for (cause in causes) { 687 def causeString = cause.toString() 688 echo "current cause: " + causeString 689 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 690 echo "This job was caused by " + causeString 691 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 692 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 693 } else if (causeString.contains("verrazzano-flaky-tests")) { 694 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 695 } 696 } 697 } 698 echo "End setDisplayName" 699 }