github.com/verrazzano/verrazzano@v1.7.1/ci/oke-ocidns/Jenkinsfile (about) 1 // Copyright (c) 2020, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 // This runs the acceptance tests on an OKE cluster with OCI DNS 5 // This is used during the release process 6 // This will eventually be replaced by the new multi-cluster job! 7 8 def DEFAULT_REPO_URL 9 def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','') 10 def dns_zone_ocid = 'dummy' 11 // for different Jenkins jobs sharing this Jenkins file, the default TEST_ENV (the first in testEnvironments) is different. 12 def testEnvironments = env.JOB_NAME.contains('oci-dns-acceptance') 13 ? ["ocidns_oke", "magicdns_oke", "kind"] 14 : env.JOB_NAME.contains('magic-dns') 15 ? ["magicdns_oke", "ocidns_oke", "kind"] 16 : ["kind", "magicdns_oke", "ocidns_oke"] 17 def acmeEnvironments = [ "staging", "production" ] 18 def certIssuers = [ "self-signed", "acme" ] 19 def agentLabel = env.JOB_NAME.contains('-dns-') ? "" : env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 20 def runExamples = env.JOB_NAME.contains('-examples') 21 22 // pulling "ap-*" from the test regions given discovery of image pull issues 23 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ] 24 Collections.shuffle(availableRegions) 25 def keepOKEClusterOnFailure = "false" 26 def OKE_CLUSTER_PREFIX = "" 27 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 28 29 pipeline { 30 options { 31 skipDefaultCheckout true 32 copyArtifactPermission('*'); 33 timestamps () 34 } 35 36 agent { 37 docker { 38 image "${RUNNER_DOCKER_IMAGE}" 39 args "${RUNNER_DOCKER_ARGS} --cap-add=NET_ADMIN" 40 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 41 label "${agentLabel}" 42 } 43 } 44 45 parameters { 46 // OKE_CLUSTER_REGION parameter will be ignored for private DNS tests. They get overwritten with runner region 47 choice (description: 'OCI region to launch OKE clusters in. This parameter will be ignored for private DNS tests', name: 'OKE_CLUSTER_REGION', 48 // 1st choice is the default value 49 choices: availableRegions ) 50 choice (description: 'OKE node pool configuration', name: 'OKE_NODE_POOL', 51 // 1st choice is the default value 52 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2", "VM.Standard.E2.2" ]) 53 choice (description: 'Use instance principal for oci dns tests', name: 'OCI_DNS_AUTH', 54 // 1st choice is the default value 55 choices: [ "user_principal", "instance_principal" ]) 56 choice (description: 'Specifies DNS scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'DNS_SCOPE', 57 // 1st choice is the default value 58 choices: [ "GLOBAL","PRIVATE" ]) 59 choice (description: 'Specifies Nginx LoadBalancer scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'NGINX_LB_SCOPE', 60 // 1st choice is the default value 61 choices: [ "GLOBAL","PRIVATE" ]) 62 choice (description: 'Specifies Istio LoadBalancer scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'ISTIO_LB_SCOPE', 63 // 1st choice is the default value 64 choices: [ "GLOBAL","PRIVATE" ]) 65 choice (description: 'Kubernetes Version for OKE Cluster', name: 'OKE_CLUSTER_VERSION', 66 // 1st choice is the default value 67 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 68 choice (name: 'CRD_API_VERSION', 69 description: 'This is the API crd version.', 70 // 1st choice is the default value 71 choices: [ "v1beta1", "v1alpha1"]) 72 choice (description: 'Certificate Issuer', name: 'CERT_ISSUER', 73 choices: certIssuers) 74 choice (description: 'ACME Certificate Environment (Staging or Production)', name: 'ACME_ENVIRONMENT', 75 choices: acmeEnvironments) 76 string defaultValue: 'dev', description: 'Verrazzano install profile name', name: "INSTALL_PROFILE", trim: true 77 string defaultValue: 'NONE', description: 'Verrazzano platform operator image name (within ghcr.io/verrazzano repo)', name: 'VERRAZZANO_OPERATOR_IMAGE', trim: true 78 choice (description: 'Verrazzano Test Environment', name: 'TEST_ENV', 79 choices: testEnvironments) 80 string (name: 'GIT_COMMIT_TO_USE', 81 defaultValue: 'NONE', 82 description: 'This is the full git commit hash from the source build to be used for all jobs', 83 trim: true) 84 booleanParam (description: 'Whether to create the cluster with Calico for AT testing', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 85 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 86 string (name: 'TAGGED_TESTS', 87 defaultValue: '', 88 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 89 trim: true) 90 string (name: 'INCLUDED_TESTS', 91 defaultValue: '.*', 92 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 93 trim: true) 94 string (name: 'EXCLUDED_TESTS', 95 defaultValue: '_excluded_test', 96 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 97 trim: true) 98 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 99 } 100 101 environment { 102 CLUSTER_NAME = 'byok8s-kind' 103 OCR_CREDS = credentials('ocr-pull-and-push-account') 104 NETRC_FILE = credentials('netrc') 105 OCR_REPO = 'container-registry.oracle.com' 106 GHCR_REPO = 'ghcr.io' 107 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 108 TEST_ENV = "${params.TEST_ENV}" 109 INSTALL_PROFILE = "${params.INSTALL_PROFILE}" 110 GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw') 111 OCIR_CREDS = credentials('ocir-pull-and-push-account') 112 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // needed by install_todo.sh OAM example test 113 DATABASE_PSW = credentials('todo-mysql-password') // needed by install_todo.sh OAM example test 114 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 115 OCIR_PHX_REPO = 'phx.ocir.io' 116 POST_DUMP_FAILED = 'false' 117 GOPATH = '/home/opc/go' 118 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 119 120 TF_VAR_tenancy_id = credentials('oci-tenancy') 121 TF_VAR_user_id = credentials('oci-user-ocid') 122 123 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 124 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 125 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 126 TF_VAR_api_private_key_path = credentials('oci-api-key') 127 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 128 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 129 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 130 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 131 132 OCI_CLI_TENANCY = credentials('oci-tenancy') 133 OCI_CLI_USER = credentials('oci-user-ocid') 134 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 135 OCI_CLI_KEY_FILE = credentials('oci-api-key') 136 137 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 138 CLUSTER_TYPE = getTestClusterType("${TEST_ENV}") 139 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 140 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 141 142 DISABLE_SPINNER=1 143 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 144 145 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 146 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 147 148 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 149 150 INSTALL_CONFIG_FILE_OCIDNS = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-ocidns.yaml" 151 INSTALL_CONFIG_FILE_NIPIO = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-nipio.yaml" 152 INSTALL_CONFIG_FILE_NODEPORT = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-nodeport.yaml" 153 154 //OCI_DNS_ZONE_NAME="z${zoneId}.v8o.io" 155 OCI_DNS_ZONE_NAME="${params.DNS_SCOPE == 'PRIVATE' ? "z${zoneId}-private.v8o.io" : "z${zoneId}.v8o.io"}" 156 CERT_ISSUER="${params.CERT_ISSUER}" 157 ACME_ENVIRONMENT="${params.ACME_ENVIRONMENT}" 158 159 VZ_ENVIRONMENT_NAME = "${params.TEST_ENV == 'ocidns_oke' ? 'b' + env.BUILD_NUMBER : 'default'}" 160 161 // Environment variables required to capture cluster snapshot and bug report on test failure 162 DUMP_KUBECONFIG="${KUBECONFIG}" 163 DUMP_COMMAND="${WORKSPACE}/tools/scripts/k8s-dump-cluster.sh" 164 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 165 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 166 167 // Environment variable for Verrazzano CLI executable 168 VZ_COMMAND="${GO_REPO_PATH}/vz" 169 170 // used for console artifact capture on failure 171 JENKINS_READ = credentials('jenkins-auditor') 172 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 173 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 174 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 175 176 //OCI_COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid') 177 //OC_TELEMETRY_URL = credentials('oci-telemetry-url') 178 179 // used to emit metrics 180 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 181 TEST_ENV_LABEL = "${params.TEST_ENV}" 182 183 // used to generate Ginkgo test reports 184 TEST_REPORT = "test-report.xml" 185 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 186 } 187 188 stages { 189 stage('Initialize') { 190 environment { 191 OCI_CLI_AUTH="instance_principal" 192 } 193 steps { 194 script { 195 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 196 if (params.GIT_COMMIT_TO_USE == "NONE") { 197 echo "Specific GIT commit was not specified, use current head" 198 def scmInfo = checkout scm 199 env.GIT_COMMIT = scmInfo.GIT_COMMIT 200 env.GIT_BRANCH = scmInfo.GIT_BRANCH 201 } else { 202 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 203 def scmInfo = checkout([ 204 $class: 'GitSCM', 205 branches: [[name: params.GIT_COMMIT_TO_USE]], 206 doGenerateSubmoduleConfigurations: false, 207 extensions: [], 208 submoduleCfg: [], 209 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 210 env.GIT_COMMIT = scmInfo.GIT_COMMIT 211 env.GIT_BRANCH = scmInfo.GIT_BRANCH 212 // If the commit we were handed is not what the SCM says we are using, fail 213 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 214 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 215 exit 1 216 } 217 } 218 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 219 } 220 221 sh """ 222 cp -f "${NETRC_FILE}" $HOME/.netrc 223 chmod 600 $HOME/.netrc 224 """ 225 println("${params.OKE_CLUSTER_REGION}") 226 println("agentlabel: ${agentLabel}") 227 sh """ 228 echo "${NODE_LABELS}" 229 rm -rf ${GO_REPO_PATH}/verrazzano 230 mkdir -p ${GO_REPO_PATH}/verrazzano 231 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 232 """ 233 234 script { 235 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 236 // update the description with some meaningful info 237 setDisplayName() 238 currentBuild.description = SHORT_COMMIT_HASH + " : " + params.OKE_CLUSTER_REGION + " : " + params.OKE_CLUSTER_VERSION 239 240 if (params.TEST_ENV != "kind") { 241 // derive the prefix for the OKE cluster 242 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 243 } 244 } 245 script { 246 sh """ 247 echo "Downloading VZ CLI from object storage" 248 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 249 mkdir -p ${GO_REPO_PATH} 250 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 251 ${GO_REPO_PATH}/vz version 252 """ 253 } 254 } 255 } 256 257 stage("install-oke") { 258 when { expression { return params.TEST_ENV != 'kind' } } 259 environment { 260 TF_VAR_label_prefix="${OKE_CLUSTER_PREFIX}" 261 } 262 steps { 263 script { 264 withCredentials([sshUserPrivateKey(credentialsId: '5fcc03de-31ce-4566-b11f-9de38e5d98fd', keyFileVariable: 'OPC_USER_KEY_FILE', passphraseVariable: 'OPC_USER_PASSPHRASE', usernameVariable: 'OPC_USERNAME')]) { 265 def RUNNER_REGION = sh(returnStdout: true, script: "curl -s -H \"Authorization: Bearer Oracle\" http://169.254.169.254/opc/v2/instance/canonicalRegionName").trim() 266 env.TF_VAR_region = "${params.DNS_SCOPE == 'PRIVATE' ? RUNNER_REGION : params.OKE_CLUSTER_REGION}" 267 env.OCI_CLI_REGION = "${params.DNS_SCOPE == 'PRIVATE' ? RUNNER_REGION : params.OKE_CLUSTER_REGION}" 268 if (params.NGINX_LB_SCOPE == "PRIVATE" || params.ISTIO_LB_SCOPE == "PRIVATE") { 269 env.TF_VAR_region = RUNNER_REGION 270 env.OCI_CLI_REGION = RUNNER_REGION 271 } 272 println("runner region is ${RUNNER_REGION}, tf var region is ${env.TF_VAR_REGION}") 273 sh """ 274 # get the ssh public key 275 ssh-keygen -y -e -f ${OPC_USER_KEY_FILE} > /tmp/opc_ssh2.pub 276 # convert SSH2 public key into an OpenSSH format 277 ssh-keygen -i -f /tmp/opc_ssh2.pub > /tmp/opc_ssh.pub 278 # set the ssh public key value for terraform 279 export TF_VAR_ssh_public_key_path=/tmp/opc_ssh.pub 280 export TF_VAR_state_name=${env.BUILD_NUMBER}-${env.TIMESTAMP}-${env.BRANCH_NAME} 281 # call create_oke_cluster with cluster access private 282 ${WORKSPACE}/tests/e2e/config/scripts/create_oke_cluster.sh true ${params.CREATE_CLUSTER_USE_CALICO} 283 """ 284 } 285 } 286 } 287 post { 288 failure { 289 script { 290 echo "Cluster create failed" 291 } 292 } 293 } 294 } 295 296 stage('install-kind') { 297 when { expression { return params.TEST_ENV == 'kind' } } 298 steps { 299 sh """ 300 cd ${WORKSPACE}/verrazzano-acceptance-test-suite 301 ${WORKSPACE}/verrazzano-acceptance-test-suite/scripts/install_kind.sh 302 """ 303 } 304 } 305 306 stage("create-image-pull-secrets") { 307 steps { 308 createImagePullSecrets() 309 } 310 } 311 312 stage("install-platform-operator") { 313 environment { 314 OCI_CLI_AUTH="instance_principal" 315 } 316 steps { 317 sh """ 318 echo "Install Platform Operator" 319 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 320 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 321 322 # Install the verrazzano-platform-operator 323 kubectl apply -f $WORKSPACE/acceptance-test-operator.yaml 324 325 # make sure ns exists 326 ${WORKSPACE}/tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 327 # create secret in verrazzano-install ns 328 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" "verrazzano-install" 329 """ 330 } 331 post { 332 always { 333 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 334 } 335 } 336 } 337 338 stage("create-dns-zone") { 339 when { expression { return params.TEST_ENV == 'ocidns_oke' } } 340 environment { 341 TF_VAR_label_prefix="${OKE_CLUSTER_PREFIX}" 342 343 } 344 steps { 345 script { 346 println("DNS_SCOPE = ${params.DNS_SCOPE}") 347 if (params.DNS_SCOPE == "PRIVATE") { 348 switch(TF_VAR_region) { 349 case "uk-london-1": 350 env.VCN_VIEW_ID = "${DNS_VIEW_FOR_PRIVATE_DNS_LHR}" 351 break 352 default: 353 env.VCN_VIEW_ID = "${DNS_VIEW_FOR_PRIVATE_DNS_PHX}" 354 break 355 } 356 println("view id is ${env.VCN_VIEW_ID}") 357 } 358 // VCN_VIEW_ID is used inside oci_dns_ops.sh to associate private zones to view 359 // this is only used when zone is PRIVATE 360 dns_zone_ocid = sh(script: "${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o create -c ${TF_VAR_compartment_id} -s z${zoneId} -k ${params.DNS_SCOPE}", returnStdout: true) 361 } 362 } 363 } 364 365 stage("setup-oci-dns-config") { 366 when { expression { return params.TEST_ENV == 'ocidns_oke' } } 367 environment { 368 OCI_DNS_COMPARTMENT_OCID = credentials('oci-dns-compartment') 369 OCI_PRIVATE_KEY_FILE = credentials('oci-api-key') 370 OCI_DNS_ZONE_OCID = "${dns_zone_ocid}" 371 } 372 steps { 373 script { 374 375 println("+++++++++++++++++++ RUN DETAILS +++++++++++++++++++++++++") 376 println("AUTH_TYPE = ${params.OCI_DNS_AUTH}") 377 println("DNS_SCOPE = ${params.DNS_SCOPE}") 378 println("NGINX_LB_SCOPE = ${params.NGINX_LB_SCOPE}") 379 println("ISTIO_LB_SCOPE = ${params.ISTIO_LB_SCOPE}") 380 println("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") 381 382 sh """ 383 ${WORKSPACE}/tests/e2e/config/scripts/process_oci_dns_install_yaml.sh $INSTALL_CONFIG_FILE_OCIDNS $CERT_ISSUER $ACME_ENVIRONMENT ${params.DNS_SCOPE} ${params.NGINX_LB_SCOPE} ${params.ISTIO_LB_SCOPE} 384 """ 385 } 386 } 387 } 388 389 stage("create-oci-config-secret") { 390 when { expression { return params.TEST_ENV == 'ocidns_oke' } } 391 steps { 392 script { 393 if (params.OCI_DNS_AUTH == "instance_principal") { 394 env.OCI_DNS_AUTH="instance_principal" 395 } 396 sh """ 397 ${WORKSPACE}/tests/e2e/config/scripts/create-test-oci-config-secret.sh 398 """ 399 } 400 } 401 } 402 403 stage("setup-nip-io-config") { 404 when { expression { return params.TEST_ENV == 'magicdns_oke' } } 405 steps { 406 script { 407 sh """ 408 ${WORKSPACE}/tests/e2e/config/scripts/process_nipio_install_yaml.sh $INSTALL_CONFIG_FILE_NIPIO 409 """ 410 } 411 } 412 } 413 414 stage("setup-nodeport-config") { 415 when { expression { return params.TEST_ENV == 'kind' } } 416 steps { 417 script { 418 sh """ 419 ${WORKSPACE}/tests/e2e/config/scripts/process_nodeport_install_yaml.sh $INSTALL_CONFIG_FILE_NODEPORT 420 """ 421 } 422 } 423 } 424 425 stage("install-verrazzano") { 426 steps { 427 sh """ 428 echo "Waiting for Operator to be ready" 429 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 430 echo "Installing Verrazzano on ${TEST_ENV}" 431 # apply config to create cluster 432 if [ "${TEST_ENV}" == "magicdns_oke" ]; then 433 kubectl apply -f ${INSTALL_CONFIG_FILE_NIPIO} 434 elif [ "${TEST_ENV}" == "ocidns_oke" ]; then 435 kubectl apply -f ${INSTALL_CONFIG_FILE_OCIDNS} 436 elif [ "${TEST_ENV}" == "kind" ]; then 437 kubectl apply -f ${INSTALL_CONFIG_FILE_NODEPORT} 438 fi 439 # wait for Verrazzano install to complete 440 ${WORKSPACE}/tests/e2e/config/scripts/wait-for-verrazzano-install.sh 441 # Create acceptance test configuration file 442 ${WORKSPACE}/tests/e2e/config/scripts/common-test-setup-script.sh "${WORKSPACE}" "${TEST_CONFIG_FILE}" "${env.DOCKER_REPO}" "${KUBECONFIG}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" "${VZ_ENVIRONMENT_NAME}" 443 444 # edit DNS info in the test config file 445 if [ "${TEST_ENV}" == "magicdns_oke" ]; then 446 ${WORKSPACE}/tests/e2e/config/scripts/get_ingress_ip.sh ${TEST_CONFIG_FILE} 447 elif [ "${TEST_ENV}" == "ocidns_oke" ]; then 448 ${WORKSPACE}/tests/e2e/config/scripts/get_oci_dns_zone.sh ${TEST_CONFIG_FILE} ${OCI_DNS_ZONE_NAME} 449 elif [ "${TEST_ENV}" == "kind" ]; then 450 ${WORKSPACE}/tests/e2e/config/scripts/get_node_ip.sh ${CLUSTER_NAME} ${TEST_CONFIG_FILE} 451 fi 452 echo "----------Test config file:-------------" 453 cat ${TEST_CONFIG_FILE} 454 echo "----------------------------------------" 455 """ 456 } 457 } 458 459 stage('verify-install') { 460 steps { 461 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 462 sh """ 463 cd ${WORKSPACE}/tests/e2e 464 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/... 465 """ 466 } 467 } 468 } 469 470 stage('acceptance-tests') { 471 when { expression { return !runExamples } } 472 parallel { 473 stage('system component metrics') { 474 steps { 475 script { 476 runGinkgo('metrics/syscomponents') 477 } 478 } 479 } 480 stage('verify-infra restapi') { 481 steps { 482 script { 483 runGinkgo('verify-infra/restapi') 484 } 485 } 486 } 487 stage('verify-infra vmi') { 488 steps { 489 script { 490 runGinkgo('verify-infra/vmi') 491 } 492 } 493 } 494 stage('verify-infra oam') { 495 steps { 496 script { 497 runGinkgo('verify-infra/oam') 498 } 499 } 500 } 501 stage('system logging') { 502 environment { 503 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/system-logging" 504 } 505 steps { 506 runGinkgo('logging/system') 507 } 508 } 509 stage('istio authorization policy') { 510 environment { 511 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-istio-auth-policy" 512 } 513 steps { 514 runGinkgo('istio/authz') 515 } 516 } 517 stage('security rbac') { 518 environment { 519 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-rbac" 520 } 521 steps { 522 runGinkgo('security/rbac') 523 } 524 } 525 stage('webLogic logging') { 526 environment { 527 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-logging" 528 } 529 steps { 530 script { 531 runGinkgoFailFast('logging/weblogic') 532 } 533 } 534 } 535 stage('examples helidon') { 536 environment { 537 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/helidon-workload" 538 } 539 steps { 540 script { 541 runGinkgoFailFast('examples/helidon') 542 } 543 } 544 } 545 stage('weblogic workload') { 546 environment { 547 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-workload" 548 } 549 steps { 550 runGinkgoFailFast('workloads/weblogic') 551 } 552 } 553 stage('coherence workload') { 554 environment { 555 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/coherence-workload" 556 } 557 steps { 558 runGinkgoFailFast('workloads/coherence') 559 } 560 } 561 } 562 } 563 564 stage('Run Example Acceptance Tests') { 565 when { expression { return runExamples } } 566 parallel { 567 stage('examples todo') { 568 environment { 569 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-todo" 570 } 571 steps { 572 runGinkgoEx('examples/todo') 573 } 574 } 575 /*stage('examples socks') { 576 environment { 577 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-socks" 578 } 579 steps { 580 runSocksVariant('helidon') 581 runSocksVariant('micronaut') 582 runSocksVariant('spring') 583 } 584 }*/ 585 stage('examples springboot') { 586 environment { 587 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-spring" 588 } 589 steps { 590 runGinkgoEx('examples/springboot') 591 } 592 } 593 stage('examples helidon 1') { 594 environment { 595 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-1" 596 } 597 steps { 598 runGinkgoEx('examples/helidon') 599 } 600 } 601 stage('examples helidon 2') { 602 environment { 603 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-2" 604 } 605 steps { 606 runGinkgoEx('examples/helidon') 607 } 608 } 609 stage('examples helidon-config') { 610 environment { 611 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-config" 612 } 613 steps { 614 runGinkgoEx('examples/helidonconfig') 615 } 616 } 617 stage('examples bobs books') { 618 environment { 619 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-bobs" 620 } 621 steps { 622 runGinkgoEx('examples/bobsbooks') 623 } 624 } 625 stage('examples weblogic cluster') { 626 environment { 627 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-cluster" 628 } 629 steps { 630 runGinkgoEx('workloads/weblogic-cluster') 631 } 632 } 633 } 634 } 635 } 636 post { 637 always { 638 script { 639 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true || currentBuild.currentResult == 'FAILURE') { 640 dumpK8sCluster('oke-acceptance-tests-cluster-snapshot') 641 } 642 } 643 644 dumpVerrazzanoSystemPods() 645 dumpCattleSystemPods() 646 dumpNginxIngressControllerLogs() 647 dumpVerrazzanoPlatformOperatorLogs() 648 649 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/*full-cluster*/**,**/*bug-report*/**,**/test-cluster-snapshots/**/,**/${TEST_REPORT}", allowEmptyArchive: true 650 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 651 sh """ 652 if [ "${TEST_ENV}" == "ocidns_oke" ]; then 653 ${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o delete -s z${zoneId} -k ${params.DNS_SCOPE} || echo "Failed to delete DNS zone z${zoneId}" 654 fi 655 if [ "${TEST_ENV}" == "kind" ]; then 656 ${WORKSPACE}/tests/e2e/config/scripts/delete-kind-cluster.sh 657 elif [ "${keepOKEClusterOnFailure}" == "false" ]; then 658 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=${env.BUILD_NUMBER}-${env.TIMESTAMP}-${env.BRANCH_NAME} ${WORKSPACE}/tests/e2e/config/scripts/delete_oke_cluster.sh || true 659 fi 660 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 661 echo "Failures seen during dumping of artifacts, treat post as failed" 662 exit 1 663 fi 664 """ 665 } 666 failure { 667 sh """ 668 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 669 """ 670 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 671 sh """ 672 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 673 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 674 rm archive.zip 675 """ 676 } 677 cleanup { 678 deleteDir() 679 } 680 } 681 } 682 683 def runSocksVariant(variant) { 684 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 685 sh """ 686 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 687 SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" examples/socks/... 688 """ 689 } 690 } 691 692 def getTestClusterType(testEnv) { 693 if("kind".equalsIgnoreCase(testEnv)) { 694 return "KIND" 695 } else { 696 return "OKE" 697 } 698 } 699 700 def runGinkgo(testSuitePath) { 701 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 702 sh """ 703 cd ${WORKSPACE}/tests/e2e 704 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 705 """ 706 } 707 } 708 709 def runGinkgoEx(testSuitePath) { 710 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 711 sh """ 712 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 713 ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 714 """ 715 } 716 } 717 718 def runGinkgoFailFast(testSuitePath) { 719 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 720 sh """ 721 cd ${WORKSPACE}/tests/e2e 722 ginkgo -v --fail-fast --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 723 """ 724 } 725 } 726 727 def dumpK8sCluster(dumpDirectory) { 728 sh """ 729 ${WORKSPACE}/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 730 """ 731 } 732 733 def dumpVerrazzanoSystemPods() { 734 sh """ 735 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log" 736 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 737 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log" 738 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 739 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log" 740 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 741 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log" 742 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 743 """ 744 } 745 746 def dumpCertManagerNamespaceLogs() { 747 sh """ 748 kubectl logs --selector=app=cert-manager -n cert-manager > ${WORKSPACE}/platform-operator/scripts/install/build/logs/cert-manager.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 749 kubectl logs --selector=app.kubernetes.io/name=external-dns -n cert-manager > ${WORKSPACE}/platform-operator/scripts/install/build/logs/external-dns.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 750 """ 751 } 752 753 def dumpCattleSystemPods() { 754 sh """ 755 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/cattle-system-pods.log" 756 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 757 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/rancher.log" 758 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 759 """ 760 } 761 762 def dumpNginxIngressControllerLogs() { 763 sh """ 764 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log" 765 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 766 """ 767 } 768 769 def dumpVerrazzanoPlatformOperatorLogs() { 770 sh """ 771 ## dump out verrazzano-platform-operator logs 772 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 773 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 774 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 775 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 776 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 777 echo "------------------------------------------" 778 """ 779 } 780 781 def dumpVerrazzanoApplicationOperatorLogs() { 782 sh """ 783 ## dump out verrazzano-application-operator logs 784 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 785 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 786 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 787 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 788 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 789 echo "------------------------------------------" 790 """ 791 } 792 793 def dumpVerrazzanoApiLogs() { 794 sh """ 795 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log" 796 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 797 """ 798 } 799 800 def getEffectiveDumpOnSuccess() { 801 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 802 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 803 effectiveValue = true 804 echo "Forcing dump on success based on global override setting" 805 } 806 return effectiveValue 807 } 808 809 def setDisplayName() { 810 echo "Start setDisplayName" 811 def causes = currentBuild.getBuildCauses() 812 echo "causes: " + causes.toString() 813 for (cause in causes) { 814 def causeString = cause.toString() 815 echo "current cause: " + causeString 816 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 817 echo "This job was caused by " + causeString 818 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 819 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 820 } else if (causeString.contains("verrazzano-flaky-tests")) { 821 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 822 } 823 } 824 } 825 echo "End setDisplayName" 826 } 827 828 def createImagePullSecrets() { 829 sh """ 830 # Create image pull secrets for Verrazzano docker images 831 cd ${WORKSPACE} 832 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 833 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 834 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 835 """ 836 }