github.com/verrazzano/verrazzano@v1.7.1/ci/ocne-cluster-driver/Jenkinsfile (about) 1 // Copyright (c) 2023, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DEFAULT_REPO_URL 5 def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','') 6 def agentLabel = env.JOB_NAME.contains('-dns-') ? "" : env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 7 8 // pulling "ap-*" from the test regions given discovery of image pull issues 9 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ] 10 Collections.shuffle(availableRegions) 11 def keepOKEClusterOnFailure = "false" 12 def OKE_CLUSTER_PREFIX = "" 13 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 14 def OCNEClusterNameSuffix = UUID.randomUUID().toString().substring(0,6).replace('-','') 15 16 pipeline { 17 options { 18 skipDefaultCheckout true 19 copyArtifactPermission('*'); 20 timestamps () 21 } 22 23 agent { 24 docker { 25 image "${RUNNER_DOCKER_IMAGE}" 26 args "${RUNNER_DOCKER_ARGS} --cap-add=NET_ADMIN" 27 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 28 label "${agentLabel}" 29 } 30 } 31 32 parameters { 33 // OKE_CLUSTER_REGION parameter will be ignored for private DNS tests. They get overwritten with runner region 34 choice (description: 'OCI region to launch OKE clusters in. This parameter will be ignored for private DNS tests', name: 'OKE_CLUSTER_REGION', 35 // 1st choice is the default value 36 choices: availableRegions ) 37 choice (description: 'OKE node pool configuration', name: 'OKE_NODE_POOL', 38 // 1st choice is the default value 39 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2", "VM.Standard.E2.2" ]) 40 choice (description: 'Specifies Nginx LoadBalancer scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'NGINX_LB_SCOPE', 41 // 1st choice is the default value 42 choices: [ "GLOBAL","PRIVATE" ]) 43 choice (description: 'Specifies Istio LoadBalancer scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'ISTIO_LB_SCOPE', 44 // 1st choice is the default value 45 choices: [ "GLOBAL","PRIVATE" ]) 46 choice (description: 'Kubernetes Version for OKE Cluster', name: 'OKE_CLUSTER_VERSION', 47 // 1st choice is the default value 48 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 49 choice (name: 'CRD_API_VERSION', 50 description: 'This is the API crd version.', 51 // 1st choice is the default value 52 choices: [ "v1beta1", "v1alpha1"]) 53 string defaultValue: 'prod', description: 'Verrazzano install profile name', name: "INSTALL_PROFILE", trim: true 54 string defaultValue: 'NONE', description: 'Verrazzano platform operator image name (within ghcr.io/verrazzano repo)', name: 'VERRAZZANO_OPERATOR_IMAGE', trim: true 55 string (name: 'GIT_COMMIT_TO_USE', 56 defaultValue: 'NONE', 57 description: 'This is the full git commit hash from the source build to be used for all jobs', 58 trim: true) 59 booleanParam (description: 'Whether to create the cluster with Calico for AT testing', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 60 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 61 string (name: 'TAGGED_TESTS', 62 defaultValue: '', 63 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 64 trim: true) 65 string (name: 'INCLUDED_TESTS', 66 defaultValue: '.*', 67 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 68 trim: true) 69 string (name: 'EXCLUDED_TESTS', 70 defaultValue: '_excluded_test', 71 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 72 trim: true) 73 booleanParam (description: 'Whether to run all the OCNE Cluster Driver tests (true), or just the 1 bare minimum test case (false).', name: 'RUN_ALL_TESTS', defaultValue: false) 74 75 // Optional parameters to the Ginkgo test suite 76 string (name: 'DOCKER_ROOT_DIR', 77 defaultValue: '', 78 description: 'An optional override to be passed into the OCNE cluster creation request.', 79 trim: true) 80 string (name: 'ENABLE_CLUSTER_ALERTING', 81 defaultValue: '', 82 description: 'An optional override to be passed into the OCNE cluster creation request.', 83 trim: true) 84 string (name: 'ENABLE_CLUSTER_MONITORING', 85 defaultValue: '', 86 description: 'An optional override to be passed into the OCNE cluster creation request.', 87 trim: true) 88 string (name: 'ENABLE_NETWORK_POLICY', 89 defaultValue: '', 90 description: 'An optional override to be passed into the OCNE cluster creation request.', 91 trim: true) 92 string (name: 'WINDOWS_PREFERRED_CLUSTER', 93 defaultValue: '', 94 description: 'An optional override to be passed into the OCNE cluster creation request.', 95 trim: true) 96 string (name: 'CALICO_IMAGE_PATH', 97 defaultValue: '', 98 description: 'An optional override to be passed into the OCNE cluster creation request.', 99 trim: true) 100 string (name: 'CLUSTER_CIDR', 101 defaultValue: '', 102 description: 'An optional override to be passed into the OCNE cluster creation request.', 103 trim: true) 104 string (name: 'CONTROL_PLANE_MEMORY_GBS', 105 defaultValue: '', 106 description: 'An optional override to be passed into the OCNE cluster creation request.', 107 trim: true) 108 string (name: 'CONTROL_PLANE_OCPUS', 109 defaultValue: '', 110 description: 'An optional override to be passed into the OCNE cluster creation request.', 111 trim: true) 112 string (name: 'CONTROL_PLANE_SHAPE', 113 defaultValue: '', 114 description: 'An optional override to be passed into the OCNE cluster creation request.', 115 trim: true) 116 string (name: 'CONTROL_PLANE_VOLUME_GBS', 117 defaultValue: '', 118 description: 'An optional override to be passed into the OCNE cluster creation request.', 119 trim: true) 120 string (name: 'CORE_DNS_IMAGE_TAG', 121 defaultValue: '', 122 description: 'An optional override to be passed into the OCNE cluster creation request.', 123 trim: true) 124 string (name: 'ETCD_IMAGE_TAG', 125 defaultValue: '', 126 description: 'An optional override to be passed into the OCNE cluster creation request.', 127 trim: true) 128 string (name: 'IMAGE_DISPLAY_NAME', 129 defaultValue: '', 130 description: 'An optional override to be passed into the OCNE cluster creation request.', 131 trim: true) 132 string (name: 'IMAGE_ID', 133 defaultValue: '', 134 description: 'An optional override to be passed into the OCNE cluster creation request.', 135 trim: true) 136 booleanParam (name: 'INSTALL_CALICO', 137 defaultValue: true, 138 description: 'An optional override to be passed into the OCNE cluster creation request.') 139 booleanParam (name: 'INSTALL_CCM', 140 defaultValue: true, 141 description: 'An optional override to be passed into the OCNE cluster creation request.') 142 booleanParam (name: 'INSTALL_VERRAZZANO', 143 defaultValue: false, 144 description: 'An optional override to be passed into the OCNE cluster creation request.') 145 string (name: 'KUBERNETES_VERSION', 146 defaultValue: '', 147 description: 'An optional override to be passed into the OCNE cluster creation request.', 148 trim: true) 149 string (name: 'NUM_CONTROL_PLANE_NODES', 150 defaultValue: '', 151 description: 'An optional override to be passed into the OCNE cluster creation request.', 152 trim: true) 153 string (name: 'OCNE_VERSION', 154 defaultValue: '', 155 description: 'An optional override to be passed into the OCNE cluster creation request.', 156 trim: true) 157 string (name: 'POD_CIDR', 158 defaultValue: '', 159 description: 'An optional override to be passed into the OCNE cluster creation request.', 160 trim: true) 161 string (name: 'PRIVATE_REGISTRY', 162 defaultValue: '', 163 description: 'An optional override to be passed into the OCNE cluster creation request.', 164 trim: true) 165 string (name: 'PROXY_ENDPOINT', 166 defaultValue: '', 167 description: 'An optional override to be passed into the OCNE cluster creation request.', 168 trim: true) 169 string (name: 'SKIP_OCNE_INSTALL', 170 defaultValue: '', 171 description: 'An optional override to be passed into the OCNE cluster creation request.', 172 trim: true) 173 string (name: 'TIGERA_IMAGE_TAG', 174 defaultValue: '', 175 description: 'An optional override to be passed into the OCNE cluster creation request.', 176 trim: true) 177 string (name: 'USE_NODE_PV_ENCRYPTION', 178 defaultValue: '', 179 description: 'An optional override to be passed into the OCNE cluster creation request.', 180 trim: true) 181 string (name: 'VERRAZZANO_RESOURCE', 182 defaultValue: '', 183 description: 'An optional override to be passed into the OCNE cluster creation request.', 184 trim: true) 185 string (name: 'VERRAZZANO_TAG', 186 defaultValue: '', 187 description: 'An optional override to be passed into the OCNE cluster creation request.', 188 trim: true) 189 string (name: 'VERRAZZANO_VERSION', 190 defaultValue: '', 191 description: 'An optional override to be passed into the OCNE cluster creation request.', 192 trim: true) 193 string (name: 'NODE_SHAPE', 194 defaultValue: '', 195 description: 'An optional override to be passed into the OCNE cluster creation request.', 196 trim: true) 197 string (name: 'NUM_WORKER_NODES', 198 defaultValue: '', 199 description: 'An optional override to be passed into the OCNE cluster creation request.', 200 trim: true) 201 string (name: 'APPLY_YAMLS', 202 defaultValue: '', 203 description: 'An optional override to be passed into the OCNE cluster creation request.', 204 trim: true) 205 } 206 207 environment { 208 DOCKER_CREDS = credentials('github-packages-credentials-rw') 209 DOCKER_EMAIL = credentials('github-packages-email') 210 DOCKER_REPO = 'ghcr.io' 211 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 212 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 213 DISABLE_SPINNER=1 214 VZ_ENVIRONMENT_NAME = "default" 215 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') 216 DATABASE_PSW = credentials('todo-mysql-password') 217 218 CLUSTER_NAME = 'byok8s-kind' 219 OCR_CREDS = credentials('ocr-pull-and-push-account') 220 NETRC_FILE = credentials('netrc') 221 OCR_REPO = 'container-registry.oracle.com' 222 GHCR_REPO = 'ghcr.io' 223 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 224 INSTALL_PROFILE = "${params.INSTALL_PROFILE}" 225 GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw') 226 OCIR_CREDS = credentials('ocir-pull-and-push-account') 227 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 228 OCIR_PHX_REPO = 'phx.ocir.io' 229 POST_DUMP_FAILED = 'false' 230 GOPATH = '/home/opc/go' 231 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 232 233 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 234 TF_VAR_tenancy_id = credentials('oci-tenancy') 235 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 236 TF_VAR_user_id = credentials('oci-user-ocid') 237 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 238 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 239 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 240 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 241 TF_VAR_api_private_key_path = credentials('oci-api-key') 242 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 243 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 244 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 245 246 OCI_CLI_TENANCY = credentials('oci-tenancy') 247 OCI_CLI_USER = credentials('oci-user-ocid') 248 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 249 OCI_CLI_KEY_FILE = credentials('oci-api-key') 250 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 251 252 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 253 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 254 255 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 256 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 257 258 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 259 260 INSTALL_CONFIG_FILE = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-nipio.yaml" 261 262 // Environment variables required to capture cluster snapshot and bug report on test failure 263 DUMP_KUBECONFIG="${KUBECONFIG}" 264 DUMP_COMMAND="${WORKSPACE}/tools/scripts/k8s-dump-cluster.sh" 265 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 266 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 267 LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test" 268 UNINSTALL_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/uninstall-test" 269 270 // Environment variable for Verrazzano CLI executable 271 VZ_COMMAND="${GO_REPO_PATH}/vz" 272 273 // used for console artifact capture on failure 274 JENKINS_READ = credentials('jenkins-auditor') 275 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 276 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 277 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 278 279 // used to emit metrics 280 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 281 TEST_ENV = "OKE" 282 TEST_ENV_LABEL = "magicdns_oke" 283 284 // used to generate Ginkgo test reports 285 TEST_REPORT = "test-report.xml" 286 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 287 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 288 289 OPERATOR_YAML_FILE = "${WORKSPACE}/acceptance-test-operator.yaml" 290 291 // For Ginkgo test 292 OCNE_CLUSTER_NAME_SUFFIX = "${OCNEClusterNameSuffix}" 293 } 294 295 stages { 296 stage('Clean workspace and checkout') { 297 steps { 298 sh """ 299 echo "${NODE_LABELS}" 300 """ 301 302 script { 303 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 304 if (params.GIT_COMMIT_TO_USE == "NONE") { 305 echo "Specific GIT commit was not specified, use current head" 306 def scmInfo = checkout scm 307 env.GIT_COMMIT = scmInfo.GIT_COMMIT 308 env.GIT_BRANCH = scmInfo.GIT_BRANCH 309 } else { 310 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 311 def scmInfo = checkout([ 312 $class: 'GitSCM', 313 branches: [[name: params.GIT_COMMIT_TO_USE]], 314 doGenerateSubmoduleConfigurations: false, 315 extensions: [], 316 submoduleCfg: [], 317 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 318 env.GIT_COMMIT = scmInfo.GIT_COMMIT 319 env.GIT_BRANCH = scmInfo.GIT_BRANCH 320 // If the commit we were handed is not what the SCM says we are using, fail 321 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 322 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 323 exit 1 324 } 325 } 326 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 327 } 328 329 sh """ 330 cp -f "${NETRC_FILE}" $HOME/.netrc 331 chmod 600 $HOME/.netrc 332 """ 333 334 script { 335 try { 336 sh """ 337 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 338 """ 339 } catch(error) { 340 echo "docker login failed, retrying after sleep" 341 retry(4) { 342 sleep(30) 343 sh """ 344 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 345 """ 346 } 347 } 348 } 349 sh """ 350 rm -rf ${GO_REPO_PATH}/verrazzano 351 mkdir -p ${GO_REPO_PATH}/verrazzano 352 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 353 """ 354 355 script { 356 def props = readProperties file: '.verrazzano-development-version' 357 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 358 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 359 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 360 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 361 // update the description with some meaningful info 362 setDisplayName() 363 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 364 365 // derive the prefix for the OKE cluster 366 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 367 } 368 script { 369 sh """ 370 echo "Downloading VZ CLI from object storage" 371 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 372 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 373 ${GO_REPO_PATH}/vz version 374 """ 375 } 376 } 377 } 378 379 stage("Create OKE Cluster") { 380 steps { 381 sh "TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=ocne-driver-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/create_oke_cluster.sh" 382 } 383 } 384 385 stage('Install Verrazzano') { 386 steps { 387 sh """ 388 # Create image pull secret for Verrazzano docker images 389 cd ${GO_REPO_PATH}/verrazzano 390 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 391 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 392 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 393 394 echo "Download Platform Operator" 395 if [ "NONE" = "${VERRAZZANO_OPERATOR_IMAGE}" ]; then 396 echo "Using operator.yaml from object storage" 397 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 398 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 399 else 400 echo "Generating operator.yaml based on image name provided: ${VERRAZZANO_OPERATOR_IMAGE}" 401 env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${WORKSPACE}/acceptance-test-operator.yaml 402 fi 403 404 echo "Create the verrazzano-install namespace" 405 kubectl create namespace verrazzano-install 406 407 # create secret in verrazzano-install ns 408 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 409 ./tests/e2e/config/scripts/process_nipio_install_yaml.sh ${INSTALL_CONFIG_FILE} 410 411 echo "Installing Verrazzano on OKE" 412 ${VZ_COMMAND} install --filename ${INSTALL_CONFIG_FILE} --manifests ${WORKSPACE}/acceptance-test-operator.yaml --timeout 45m 413 """ 414 } 415 post { 416 always { 417 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 418 } 419 success { 420 script { 421 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 422 dumpK8sCluster('verrazzano-install-cluster-dump') 423 } 424 } 425 } 426 failure { 427 script { 428 dumpK8sCluster('verrazzano-install-failure-cluster-dump') 429 sh """ 430 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs 431 ${LOOPING_TEST_SCRIPTS_DIR}/dump_resources.sh > ${WORKSPACE}/verrazzano-platform-operator/scripts/install/build/logs/resources.log 432 """ 433 } 434 } 435 } 436 } 437 438 stage("OCNE Cluster Driver Tests") { 439 steps { 440 script { 441 // Required environment variables for running the ocne-driver Ginkgo test suite 442 env.TF_VAR_label_prefix="${OKE_CLUSTER_PREFIX}" 443 setUpEnvVarsForOCNEClusterCreation() 444 445 runGinkgo('clusterapi/ocne-driver') 446 } 447 } 448 post { 449 always { 450 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 451 } 452 aborted { 453 script { 454 dumpK8sCluster('ocne-cluster-driver-cluster-dump') 455 } 456 } 457 failure { 458 script { 459 dumpK8sCluster('ocne-cluster-driver-cluster-dump') 460 } 461 } 462 success { 463 script { 464 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 465 dumpK8sCluster('ocne-cluster-driver-cluster-dump') 466 } 467 } 468 } 469 } 470 } 471 } 472 473 post { 474 always { 475 sh """ 476 # Copy the generated test reports to WORKSPACE to archive them 477 mkdir -p ${TEST_REPORT_DIR} 478 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 479 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 480 """ 481 archiveArtifacts artifacts: "**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/${TEST_REPORT}", allowEmptyArchive: true 482 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 483 } 484 failure { 485 script { 486 archiveArtifacts artifacts: '**/oke_kubeconfig,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**', allowEmptyArchive: true 487 } 488 } 489 cleanup { 490 sh "VERRAZZANO_KUBECONFIG=${env.KUBECONFIG} TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=ocne-driver-${env.BUILD_NUMBER}-${env.BRANCH_NAME}/${env.TIMESTAMP} ${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/delete_oke_cluster.sh || true" 491 deleteDir() 492 } 493 } 494 } 495 496 def dumpK8sCluster(dumpDirectory) { 497 sh """ 498 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 499 """ 500 } 501 502 def getEffectiveDumpOnSuccess() { 503 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 504 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 505 effectiveValue = true 506 echo "Forcing dump on success based on global override setting" 507 } 508 return effectiveValue 509 } 510 511 def setDisplayName() { 512 echo "Start setDisplayName" 513 def causes = currentBuild.getBuildCauses() 514 echo "causes: " + causes.toString() 515 for (cause in causes) { 516 def causeString = cause.toString() 517 echo "current cause: " + causeString 518 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 519 echo "This job was caused by " + causeString 520 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 521 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 522 } else if (causeString.contains("verrazzano-flaky-tests")) { 523 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 524 } 525 } 526 } 527 echo "End setDisplayName" 528 } 529 530 def createImagePullSecrets() { 531 sh """ 532 # Create image pull secrets for Verrazzano docker images 533 cd ${WORKSPACE} 534 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 535 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 536 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 537 """ 538 } 539 540 // Run the OCNE Cluster Driver Ginkgo test suite, passing in some parameters 541 def runGinkgo(testSuitePath) { 542 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 543 sh """ 544 echo Starting the OCNE cluster driver tests 545 cd ${WORKSPACE}/tests/e2e 546 ginkgo -p -v --timeout 2h --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --runAllTests=${params.RUN_ALL_TESTS} 547 """ 548 } 549 } 550 551 def setUpEnvVarsForOCNEClusterCreation() { 552 sh "echo Setting up inputs for the OCNE cluster driver tests" 553 env.VCN_ID=sh(returnStdout: true, script: """ oci network vcn list --compartment-id ${env.TF_VAR_compartment_id} --display-name ${env.TF_VAR_label_prefix}-oke-vcn | jq -r '.data[0].id' """).trim() 554 env.SUBNET_ID=sh(returnStdout: true, script: """ oci network subnet list --compartment-id ${env.TF_VAR_compartment_id} --vcn-id ${env.VCN_ID} --display-name ${env.TF_VAR_label_prefix}-workers | jq -r '.data[0].id' """).trim() 555 556 // required environment variables for the Ginkgo test suite 557 env.OCI_REGION="${env.TF_VAR_region}" 558 env.OCI_VCN_ID="${env.VCN_ID}" 559 env.OCI_USER_ID="${env.TF_VAR_user_id}" 560 env.OCI_TENANCY_ID="${env.TF_VAR_tenancy_id}" 561 env.OCI_CREDENTIALS_FINGERPRINT="${env.TF_VAR_api_fingerprint}" 562 env.OCI_PRIVATE_KEY_PATH="${TF_VAR_api_private_key_path}" 563 env.NODE_PUBLIC_KEY_PATH="${TF_VAR_ssh_public_key_path}" 564 env.OCI_COMPARTMENT_ID="${env.TF_VAR_compartment_id}" 565 env.WORKER_NODE_SUBNET="${env.SUBNET_ID}" 566 env.CONTROL_PLANE_SUBNET="${env.SUBNET_ID}" 567 env.LOAD_BALANCER_SUBNET="${env.SUBNET_ID}" 568 569 // optional parameters 570 env.DOCKER_ROOT_DIR="${params.DOCKER_ROOT_DIR}" 571 env.ENABLE_CLUSTER_ALERTING="${params.ENABLE_CLUSTER_ALERTING}" 572 env.ENABLE_CLUSTER_MONITORING="${params.ENABLE_CLUSTER_MONITORING}" 573 env.ENABLE_NETWORK_POLICY="${params.ENABLE_NETWORK_POLICY}" 574 env.WINDOWS_PREFERRED_CLUSTER="${params.WINDOWS_PREFERRED_CLUSTER}" 575 env.CALICO_IMAGE_PATH="${params.CALICO_IMAGE_PATH}" 576 env.CLUSTER_CIDR="${params.CLUSTER_CIDR}" 577 env.CONTROL_PLANE_MEMORY_GBS="${params.CONTROL_PLANE_MEMORY_GBS}" 578 env.CONTROL_PLANE_OCPUS="${params.CONTROL_PLANE_OCPUS}" 579 env.CONTROL_PLANE_SHAPE="${params.CONTROL_PLANE_SHAPE}" 580 env.CONTROL_PLANE_VOLUME_GBS="${params.CONTROL_PLANE_VOLUME_GBS}" 581 env.CORE_DNS_IMAGE_TAG="${params.CORE_DNS_IMAGE_TAG}" 582 env.ETCD_IMAGE_TAG="${params.ETCD_IMAGE_TAG}" 583 env.IMAGE_DISPLAY_NAME="${params.IMAGE_DISPLAY_NAME}" 584 env.IMAGE_ID="${params.IMAGE_ID}" 585 env.INSTALL_CALICO="${params.INSTALL_CALICO}" 586 env.INSTALL_CCM="${params.INSTALL_CCM}" 587 env.INSTALL_VERRAZZANO="${params.INSTALL_VERRAZZANO}" 588 env.KUBERNETES_VERSION="${params.KUBERNETES_VERSION}" 589 env.NUM_CONTROL_PLANE_NODES="${params.NUM_CONTROL_PLANE_NODES}" 590 env.OCNE_VERSION="${params.OCNE_VERSION}" 591 env.POD_CIDR="${params.POD_CIDR}" 592 env.PRIVATE_REGISTRY="${params.PRIVATE_REGISTRY}" 593 env.PROXY_ENDPOINT="${params.PROXY_ENDPOINT}" 594 env.SKIP_OCNE_INSTALL="${params.SKIP_OCNE_INSTALL}" 595 env.TIGERA_IMAGE_TAG="${params.TIGERA_IMAGE_TAG}" 596 env.USE_NODE_PV_ENCRYPTION="${params.USE_NODE_PV_ENCRYPTION}" 597 env.VERRAZZANO_RESOURCE="${params.VERRAZZANO_RESOURCE}" 598 env.VERRAZZANO_TAG="${params.VERRAZZANO_TAG}" 599 env.VERRAZZANO_VERSION="${params.VERRAZZANO_VERSION}" 600 env.NODE_SHAPE="${params.NODE_SHAPE}" 601 env.NUM_WORKER_NODES="${params.NUM_WORKER_NODES}" 602 env.APPLY_YAMLS="${params.APPLY_YAMLS}" 603 } 604 605 def downloadCLI() { 606 script { 607 sh "echo Downloading VZ CLI from object storage" 608 sh """ 609 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 610 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 611 ${GO_REPO_PATH}/vz version 612 """ 613 } 614 } 615 616 // Either download the specified release of the platform operator YAML, or create one 617 // using the specific operator image provided by the user. 618 def getVerrazzanoOperatorYaml() { 619 script { 620 sh """ 621 echo "Platform Operator Configuration" 622 cd ${GO_REPO_PATH}/verrazzano 623 if [ "NONE" == "${params.VERRAZZANO_OPERATOR_IMAGE}" ]; then 624 echo "Downloading operator.yaml from branch ${env.BRANCH_NAME} for commit ${SHORT_COMMIT_HASH}" 625 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${OPERATOR_YAML_FILE} 626 else 627 echo "Generating operator.yaml based on image name provided: ${params.VERRAZZANO_OPERATOR_IMAGE}" 628 env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${params.VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${OPERATOR_YAML_FILE} 629 fi 630 """ 631 } 632 } 633 634 def performVerrazzanoDirectoryOperations() { 635 script { 636 sh """ 637 rm -rf ${GO_REPO_PATH}/verrazzano 638 mkdir -p ${GO_REPO_PATH}/verrazzano 639 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 640 """ 641 } 642 }