github.com/verrazzano/verrazzano@v1.7.0/ci/oke-ocidns/Jenkinsfile (about) 1 // Copyright (c) 2020, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 // This runs the acceptance tests on an OKE cluster with OCI DNS 5 // This is used during the release process 6 // This will eventually be replaced by the new multi-cluster job! 7 8 def DEFAULT_REPO_URL 9 def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','') 10 def dns_zone_ocid = 'dummy' 11 // for different Jenkins jobs sharing this Jenkins file, the default TEST_ENV (the first in testEnvironments) is different. 12 def testEnvironments = env.JOB_NAME.contains('oci-dns-acceptance') 13 ? ["ocidns_oke", "magicdns_oke", "kind"] 14 : env.JOB_NAME.contains('magic-dns') 15 ? ["magicdns_oke", "ocidns_oke", "kind"] 16 : ["kind", "magicdns_oke", "ocidns_oke"] 17 def acmeEnvironments = [ "staging", "production" ] 18 def certIssuers = [ "self-signed", "acme" ] 19 def agentLabel = env.JOB_NAME.contains('-dns-') ? "" : env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 20 def runExamples = env.JOB_NAME.contains('-examples') 21 22 // pulling "ap-*" from the test regions given discovery of image pull issues 23 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-frankfurt-1", "eu-zurich-1", "uk-london-1" ] 24 Collections.shuffle(availableRegions) 25 def keepOKEClusterOnFailure = "false" 26 def OKE_CLUSTER_PREFIX = "" 27 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 28 29 pipeline { 30 options { 31 skipDefaultCheckout true 32 copyArtifactPermission('*'); 33 timestamps () 34 } 35 36 agent { 37 docker { 38 image "${RUNNER_DOCKER_IMAGE}" 39 args "${RUNNER_DOCKER_ARGS} --cap-add=NET_ADMIN" 40 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 41 label "${agentLabel}" 42 } 43 } 44 45 parameters { 46 // OKE_CLUSTER_REGION parameter will be ignored for private DNS tests. They get overwritten with runner region 47 choice (description: 'OCI region to launch OKE clusters in. This parameter will be ignored for private DNS tests', name: 'OKE_CLUSTER_REGION', 48 // 1st choice is the default value 49 choices: availableRegions ) 50 choice (description: 'OKE node pool configuration', name: 'OKE_NODE_POOL', 51 // 1st choice is the default value 52 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2", "VM.Standard.E2.2" ]) 53 choice (description: 'Use instance principal for oci dns tests', name: 'OCI_DNS_AUTH', 54 // 1st choice is the default value 55 choices: [ "user_principal", "instance_principal" ]) 56 choice (description: 'Specifies DNS scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'DNS_SCOPE', 57 // 1st choice is the default value 58 choices: [ "GLOBAL","PRIVATE" ]) 59 choice (description: 'Specifies Nginx LoadBalancer scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'NGINX_LB_SCOPE', 60 // 1st choice is the default value 61 choices: [ "GLOBAL","PRIVATE" ]) 62 choice (description: 'Specifies Istio LoadBalancer scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'ISTIO_LB_SCOPE', 63 // 1st choice is the default value 64 choices: [ "GLOBAL","PRIVATE" ]) 65 choice (description: 'Kubernetes Version for OKE Cluster', name: 'OKE_CLUSTER_VERSION', 66 // 1st choice is the default value 67 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1" ]) 68 choice (name: 'CRD_API_VERSION', 69 description: 'This is the API crd version.', 70 // 1st choice is the default value 71 choices: [ "v1beta1", "v1alpha1"]) 72 choice (description: 'Certificate Issuer', name: 'CERT_ISSUER', 73 choices: certIssuers) 74 choice (description: 'ACME Certificate Environment (Staging or Production)', name: 'ACME_ENVIRONMENT', 75 choices: acmeEnvironments) 76 string defaultValue: 'dev', description: 'Verrazzano install profile name', name: "INSTALL_PROFILE", trim: true 77 string defaultValue: 'NONE', description: 'Verrazzano platform operator image name (within ghcr.io/verrazzano repo)', name: 'VERRAZZANO_OPERATOR_IMAGE', trim: true 78 choice (description: 'Verrazzano Test Environment', name: 'TEST_ENV', 79 choices: testEnvironments) 80 string (name: 'GIT_COMMIT_TO_USE', 81 defaultValue: 'NONE', 82 description: 'This is the full git commit hash from the source build to be used for all jobs', 83 trim: true) 84 booleanParam (description: 'Whether to create the cluster with Calico for AT testing', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 85 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 86 string (name: 'TAGGED_TESTS', 87 defaultValue: '', 88 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 89 trim: true) 90 string (name: 'INCLUDED_TESTS', 91 defaultValue: '.*', 92 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 93 trim: true) 94 string (name: 'EXCLUDED_TESTS', 95 defaultValue: '_excluded_test', 96 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 97 trim: true) 98 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 99 } 100 101 environment { 102 CLUSTER_NAME = 'byok8s-kind' 103 OCR_CREDS = credentials('ocr-pull-and-push-account') 104 NETRC_FILE = credentials('netrc') 105 OCR_REPO = 'container-registry.oracle.com' 106 GHCR_REPO = 'ghcr.io' 107 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 108 TEST_ENV = "${params.TEST_ENV}" 109 INSTALL_PROFILE = "${params.INSTALL_PROFILE}" 110 GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw') 111 OCIR_CREDS = credentials('ocir-pull-and-push-account') 112 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // needed by install_todo.sh OAM example test 113 DATABASE_PSW = credentials('todo-mysql-password') // needed by install_todo.sh OAM example test 114 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 115 OCIR_PHX_REPO = 'phx.ocir.io' 116 POST_DUMP_FAILED = 'false' 117 GOPATH = '/home/opc/go' 118 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 119 120 TF_VAR_tenancy_id = credentials('oci-tenancy') 121 TF_VAR_user_id = credentials('oci-user-ocid') 122 123 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 124 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 125 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 126 TF_VAR_api_private_key_path = credentials('oci-api-key') 127 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 128 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 129 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 130 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 131 132 OCI_CLI_TENANCY = credentials('oci-tenancy') 133 OCI_CLI_USER = credentials('oci-user-ocid') 134 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 135 OCI_CLI_KEY_FILE = credentials('oci-api-key') 136 137 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 138 CLUSTER_TYPE = getTestClusterType("${TEST_ENV}") 139 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 140 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 141 142 DISABLE_SPINNER=1 143 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 144 145 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 146 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 147 148 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 149 150 INSTALL_CONFIG_FILE_OCIDNS = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-ocidns.yaml" 151 INSTALL_CONFIG_FILE_NIPIO = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-nipio.yaml" 152 INSTALL_CONFIG_FILE_NODEPORT = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-nodeport.yaml" 153 154 //OCI_DNS_ZONE_NAME="z${zoneId}.v8o.io" 155 OCI_DNS_ZONE_NAME="${params.DNS_SCOPE == 'PRIVATE' ? "z${zoneId}-private.v8o.io" : "z${zoneId}.v8o.io"}" 156 CERT_ISSUER="${params.CERT_ISSUER}" 157 ACME_ENVIRONMENT="${params.ACME_ENVIRONMENT}" 158 159 VZ_ENVIRONMENT_NAME = "${params.TEST_ENV == 'ocidns_oke' ? 'b' + env.BUILD_NUMBER : 'default'}" 160 161 // Environment variables required to capture cluster snapshot and bug report on test failure 162 DUMP_KUBECONFIG="${KUBECONFIG}" 163 DUMP_COMMAND="${WORKSPACE}/tools/scripts/k8s-dump-cluster.sh" 164 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 165 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 166 167 // Environment variable for Verrazzano CLI executable 168 VZ_COMMAND="${GO_REPO_PATH}/vz" 169 170 // used for console artifact capture on failure 171 JENKINS_READ = credentials('jenkins-auditor') 172 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 173 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 174 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 175 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 176 177 //OCI_COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid') 178 //OC_TELEMETRY_URL = credentials('oci-telemetry-url') 179 180 // used to emit metrics 181 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 182 TEST_ENV_LABEL = "${params.TEST_ENV}" 183 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 184 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 185 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 186 187 // used to generate Ginkgo test reports 188 TEST_REPORT = "test-report.xml" 189 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 190 } 191 192 stages { 193 stage('Initialize') { 194 environment { 195 OCI_CLI_AUTH="instance_principal" 196 } 197 steps { 198 script { 199 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 200 if (params.GIT_COMMIT_TO_USE == "NONE") { 201 echo "Specific GIT commit was not specified, use current head" 202 def scmInfo = checkout scm 203 env.GIT_COMMIT = scmInfo.GIT_COMMIT 204 env.GIT_BRANCH = scmInfo.GIT_BRANCH 205 } else { 206 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 207 def scmInfo = checkout([ 208 $class: 'GitSCM', 209 branches: [[name: params.GIT_COMMIT_TO_USE]], 210 doGenerateSubmoduleConfigurations: false, 211 extensions: [], 212 submoduleCfg: [], 213 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 214 env.GIT_COMMIT = scmInfo.GIT_COMMIT 215 env.GIT_BRANCH = scmInfo.GIT_BRANCH 216 // If the commit we were handed is not what the SCM says we are using, fail 217 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 218 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 219 exit 1 220 } 221 } 222 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 223 } 224 225 sh """ 226 cp -f "${NETRC_FILE}" $HOME/.netrc 227 chmod 600 $HOME/.netrc 228 """ 229 println("${params.OKE_CLUSTER_REGION}") 230 println("agentlabel: ${agentLabel}") 231 sh """ 232 echo "${NODE_LABELS}" 233 rm -rf ${GO_REPO_PATH}/verrazzano 234 mkdir -p ${GO_REPO_PATH}/verrazzano 235 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 236 """ 237 238 script { 239 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 240 // update the description with some meaningful info 241 setDisplayName() 242 currentBuild.description = SHORT_COMMIT_HASH + " : " + params.OKE_CLUSTER_REGION + " : " + params.OKE_CLUSTER_VERSION 243 244 if (params.TEST_ENV != "kind") { 245 // derive the prefix for the OKE cluster 246 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 247 } 248 } 249 script { 250 sh """ 251 echo "Downloading VZ CLI from object storage" 252 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 253 mkdir -p ${GO_REPO_PATH} 254 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 255 ${GO_REPO_PATH}/vz version 256 """ 257 } 258 } 259 } 260 261 stage("install-oke") { 262 when { expression { return params.TEST_ENV != 'kind' } } 263 environment { 264 TF_VAR_label_prefix="${OKE_CLUSTER_PREFIX}" 265 } 266 steps { 267 script { 268 withCredentials([sshUserPrivateKey(credentialsId: '5fcc03de-31ce-4566-b11f-9de38e5d98fd', keyFileVariable: 'OPC_USER_KEY_FILE', passphraseVariable: 'OPC_USER_PASSPHRASE', usernameVariable: 'OPC_USERNAME')]) { 269 def RUNNER_REGION = sh(returnStdout: true, script: "curl -s -H \"Authorization: Bearer Oracle\" http://169.254.169.254/opc/v2/instance/canonicalRegionName").trim() 270 env.TF_VAR_region = "${params.DNS_SCOPE == 'PRIVATE' ? RUNNER_REGION : params.OKE_CLUSTER_REGION}" 271 env.OCI_CLI_REGION = "${params.DNS_SCOPE == 'PRIVATE' ? RUNNER_REGION : params.OKE_CLUSTER_REGION}" 272 if (params.NGINX_LB_SCOPE == "PRIVATE" || params.ISTIO_LB_SCOPE == "PRIVATE") { 273 env.TF_VAR_region = RUNNER_REGION 274 env.OCI_CLI_REGION = RUNNER_REGION 275 } 276 println("runner region is ${RUNNER_REGION}, tf var region is ${env.TF_VAR_REGION}") 277 sh """ 278 # get the ssh public key 279 ssh-keygen -y -e -f ${OPC_USER_KEY_FILE} > /tmp/opc_ssh2.pub 280 # convert SSH2 public key into an OpenSSH format 281 ssh-keygen -i -f /tmp/opc_ssh2.pub > /tmp/opc_ssh.pub 282 # set the ssh public key value for terraform 283 export TF_VAR_ssh_public_key_path=/tmp/opc_ssh.pub 284 export TF_VAR_state_name=${env.BUILD_NUMBER}-${env.TIMESTAMP}-${env.BRANCH_NAME} 285 # call create_oke_cluster with cluster access private 286 ${WORKSPACE}/tests/e2e/config/scripts/create_oke_cluster.sh true ${params.CREATE_CLUSTER_USE_CALICO} 287 """ 288 } 289 } 290 } 291 post { 292 failure { 293 script { 294 echo "Cluster create failed" 295 } 296 } 297 } 298 } 299 300 stage('install-kind') { 301 when { expression { return params.TEST_ENV == 'kind' } } 302 steps { 303 sh """ 304 cd ${WORKSPACE}/verrazzano-acceptance-test-suite 305 ${WORKSPACE}/verrazzano-acceptance-test-suite/scripts/install_kind.sh 306 """ 307 } 308 } 309 310 stage("create-image-pull-secrets") { 311 steps { 312 createImagePullSecrets() 313 } 314 } 315 316 stage("install-platform-operator") { 317 environment { 318 OCI_CLI_AUTH="instance_principal" 319 } 320 steps { 321 sh """ 322 echo "Install Platform Operator" 323 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${WORKSPACE}/downloaded-operator.yaml 324 cp ${WORKSPACE}/downloaded-operator.yaml ${WORKSPACE}/acceptance-test-operator.yaml 325 326 # Install the verrazzano-platform-operator 327 kubectl apply -f $WORKSPACE/acceptance-test-operator.yaml 328 329 # make sure ns exists 330 ${WORKSPACE}/tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 331 # create secret in verrazzano-install ns 332 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" "verrazzano-install" 333 """ 334 } 335 post { 336 always { 337 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml", allowEmptyArchive: true 338 } 339 } 340 } 341 342 stage("create-dns-zone") { 343 when { expression { return params.TEST_ENV == 'ocidns_oke' } } 344 environment { 345 TF_VAR_label_prefix="${OKE_CLUSTER_PREFIX}" 346 347 } 348 steps { 349 script { 350 println("DNS_SCOPE = ${params.DNS_SCOPE}") 351 if (params.DNS_SCOPE == "PRIVATE") { 352 switch(TF_VAR_region) { 353 case "uk-london-1": 354 env.VCN_VIEW_ID = "${DNS_VIEW_FOR_PRIVATE_DNS_LHR}" 355 break 356 default: 357 env.VCN_VIEW_ID = "${DNS_VIEW_FOR_PRIVATE_DNS_PHX}" 358 break 359 } 360 println("view id is ${env.VCN_VIEW_ID}") 361 } 362 // VCN_VIEW_ID is used inside oci_dns_ops.sh to associate private zones to view 363 // this is only used when zone is PRIVATE 364 dns_zone_ocid = sh(script: "${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o create -c ${TF_VAR_compartment_id} -s z${zoneId} -k ${params.DNS_SCOPE}", returnStdout: true) 365 } 366 } 367 } 368 369 stage("setup-oci-dns-config") { 370 when { expression { return params.TEST_ENV == 'ocidns_oke' } } 371 environment { 372 OCI_DNS_COMPARTMENT_OCID = credentials('oci-dns-compartment') 373 OCI_PRIVATE_KEY_FILE = credentials('oci-api-key') 374 OCI_DNS_ZONE_OCID = "${dns_zone_ocid}" 375 } 376 steps { 377 script { 378 379 println("+++++++++++++++++++ RUN DETAILS +++++++++++++++++++++++++") 380 println("AUTH_TYPE = ${params.OCI_DNS_AUTH}") 381 println("DNS_SCOPE = ${params.DNS_SCOPE}") 382 println("NGINX_LB_SCOPE = ${params.NGINX_LB_SCOPE}") 383 println("ISTIO_LB_SCOPE = ${params.ISTIO_LB_SCOPE}") 384 println("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") 385 386 sh """ 387 ${WORKSPACE}/tests/e2e/config/scripts/process_oci_dns_install_yaml.sh $INSTALL_CONFIG_FILE_OCIDNS $CERT_ISSUER $ACME_ENVIRONMENT ${params.DNS_SCOPE} ${params.NGINX_LB_SCOPE} ${params.ISTIO_LB_SCOPE} 388 """ 389 } 390 } 391 } 392 393 stage("create-oci-config-secret") { 394 when { expression { return params.TEST_ENV == 'ocidns_oke' } } 395 steps { 396 script { 397 if (params.OCI_DNS_AUTH == "instance_principal") { 398 env.OCI_DNS_AUTH="instance_principal" 399 } 400 sh """ 401 ${WORKSPACE}/tests/e2e/config/scripts/create-test-oci-config-secret.sh 402 """ 403 } 404 } 405 } 406 407 stage("setup-nip-io-config") { 408 when { expression { return params.TEST_ENV == 'magicdns_oke' } } 409 steps { 410 script { 411 sh """ 412 ${WORKSPACE}/tests/e2e/config/scripts/process_nipio_install_yaml.sh $INSTALL_CONFIG_FILE_NIPIO 413 """ 414 } 415 } 416 } 417 418 stage("setup-nodeport-config") { 419 when { expression { return params.TEST_ENV == 'kind' } } 420 steps { 421 script { 422 sh """ 423 ${WORKSPACE}/tests/e2e/config/scripts/process_nodeport_install_yaml.sh $INSTALL_CONFIG_FILE_NODEPORT 424 """ 425 } 426 } 427 } 428 429 stage("install-verrazzano") { 430 steps { 431 sh """ 432 echo "Waiting for Operator to be ready" 433 kubectl -n verrazzano-install rollout status deployment/verrazzano-platform-operator 434 echo "Installing Verrazzano on ${TEST_ENV}" 435 # apply config to create cluster 436 if [ "${TEST_ENV}" == "magicdns_oke" ]; then 437 kubectl apply -f ${INSTALL_CONFIG_FILE_NIPIO} 438 elif [ "${TEST_ENV}" == "ocidns_oke" ]; then 439 kubectl apply -f ${INSTALL_CONFIG_FILE_OCIDNS} 440 elif [ "${TEST_ENV}" == "kind" ]; then 441 kubectl apply -f ${INSTALL_CONFIG_FILE_NODEPORT} 442 fi 443 # wait for Verrazzano install to complete 444 ${WORKSPACE}/tests/e2e/config/scripts/wait-for-verrazzano-install.sh 445 # Create acceptance test configuration file 446 ${WORKSPACE}/tests/e2e/config/scripts/common-test-setup-script.sh "${WORKSPACE}" "${TEST_CONFIG_FILE}" "${env.DOCKER_REPO}" "${KUBECONFIG}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" "${VZ_ENVIRONMENT_NAME}" 447 448 # edit DNS info in the test config file 449 if [ "${TEST_ENV}" == "magicdns_oke" ]; then 450 ${WORKSPACE}/tests/e2e/config/scripts/get_ingress_ip.sh ${TEST_CONFIG_FILE} 451 elif [ "${TEST_ENV}" == "ocidns_oke" ]; then 452 ${WORKSPACE}/tests/e2e/config/scripts/get_oci_dns_zone.sh ${TEST_CONFIG_FILE} ${OCI_DNS_ZONE_NAME} 453 elif [ "${TEST_ENV}" == "kind" ]; then 454 ${WORKSPACE}/tests/e2e/config/scripts/get_node_ip.sh ${CLUSTER_NAME} ${TEST_CONFIG_FILE} 455 fi 456 echo "----------Test config file:-------------" 457 cat ${TEST_CONFIG_FILE} 458 echo "----------------------------------------" 459 """ 460 } 461 } 462 463 stage('verify-install') { 464 steps { 465 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 466 sh """ 467 cd ${WORKSPACE}/tests/e2e 468 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/... 469 """ 470 } 471 } 472 } 473 474 stage('acceptance-tests') { 475 when { expression { return !runExamples } } 476 parallel { 477 stage('system component metrics') { 478 steps { 479 script { 480 runGinkgo('metrics/syscomponents') 481 } 482 } 483 } 484 stage('verify-infra restapi') { 485 steps { 486 script { 487 runGinkgo('verify-infra/restapi') 488 } 489 } 490 } 491 stage('verify-infra vmi') { 492 steps { 493 script { 494 runGinkgo('verify-infra/vmi') 495 } 496 } 497 } 498 stage('verify-infra oam') { 499 steps { 500 script { 501 runGinkgo('verify-infra/oam') 502 } 503 } 504 } 505 stage('system logging') { 506 environment { 507 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/system-logging" 508 } 509 steps { 510 runGinkgo('logging/system') 511 } 512 } 513 stage('istio authorization policy') { 514 environment { 515 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-istio-auth-policy" 516 } 517 steps { 518 runGinkgo('istio/authz') 519 } 520 } 521 stage('security rbac') { 522 environment { 523 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-rbac" 524 } 525 steps { 526 runGinkgo('security/rbac') 527 } 528 } 529 stage('webLogic logging') { 530 environment { 531 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-logging" 532 } 533 steps { 534 script { 535 runGinkgoFailFast('logging/weblogic') 536 } 537 } 538 } 539 stage('examples helidon') { 540 environment { 541 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/helidon-workload" 542 } 543 steps { 544 script { 545 runGinkgoFailFast('examples/helidon') 546 } 547 } 548 } 549 stage('weblogic workload') { 550 environment { 551 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-workload" 552 } 553 steps { 554 runGinkgoFailFast('workloads/weblogic') 555 } 556 } 557 stage('coherence workload') { 558 environment { 559 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/coherence-workload" 560 } 561 steps { 562 runGinkgoFailFast('workloads/coherence') 563 } 564 } 565 } 566 } 567 568 stage('Run Example Acceptance Tests') { 569 when { expression { return runExamples } } 570 parallel { 571 stage('examples todo') { 572 environment { 573 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-todo" 574 } 575 steps { 576 runGinkgoEx('examples/todo') 577 } 578 } 579 /*stage('examples socks') { 580 environment { 581 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-socks" 582 } 583 steps { 584 runSocksVariant('helidon') 585 runSocksVariant('micronaut') 586 runSocksVariant('spring') 587 } 588 }*/ 589 stage('examples springboot') { 590 environment { 591 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-spring" 592 } 593 steps { 594 runGinkgoEx('examples/springboot') 595 } 596 } 597 stage('examples helidon 1') { 598 environment { 599 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-1" 600 } 601 steps { 602 runGinkgoEx('examples/helidon') 603 } 604 } 605 stage('examples helidon 2') { 606 environment { 607 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-2" 608 } 609 steps { 610 runGinkgoEx('examples/helidon') 611 } 612 } 613 stage('examples helidon-config') { 614 environment { 615 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-config" 616 } 617 steps { 618 runGinkgoEx('examples/helidonconfig') 619 } 620 } 621 stage('examples bobs books') { 622 environment { 623 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-bobs" 624 } 625 steps { 626 runGinkgoEx('examples/bobsbooks') 627 } 628 } 629 stage('examples weblogic cluster') { 630 environment { 631 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-cluster" 632 } 633 steps { 634 runGinkgoEx('workloads/weblogic-cluster') 635 } 636 } 637 } 638 } 639 } 640 post { 641 always { 642 script { 643 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true || currentBuild.currentResult == 'FAILURE') { 644 dumpK8sCluster('oke-acceptance-tests-cluster-snapshot') 645 } 646 } 647 648 dumpVerrazzanoSystemPods() 649 dumpCattleSystemPods() 650 dumpNginxIngressControllerLogs() 651 dumpVerrazzanoPlatformOperatorLogs() 652 653 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/*full-cluster*/**,**/*bug-report*/**,**/test-cluster-snapshots/**/,**/${TEST_REPORT}", allowEmptyArchive: true 654 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 655 sh """ 656 if [ "${TEST_ENV}" == "ocidns_oke" ]; then 657 ${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o delete -s z${zoneId} -k ${params.DNS_SCOPE} || echo "Failed to delete DNS zone z${zoneId}" 658 fi 659 if [ "${TEST_ENV}" == "kind" ]; then 660 ${WORKSPACE}/tests/e2e/config/scripts/delete-kind-cluster.sh 661 elif [ "${keepOKEClusterOnFailure}" == "false" ]; then 662 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=${env.BUILD_NUMBER}-${env.TIMESTAMP}-${env.BRANCH_NAME} ${WORKSPACE}/tests/e2e/config/scripts/delete_oke_cluster.sh || true 663 fi 664 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 665 echo "Failures seen during dumping of artifacts, treat post as failed" 666 exit 1 667 fi 668 """ 669 } 670 failure { 671 sh """ 672 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 673 """ 674 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 675 sh """ 676 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 677 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 678 rm archive.zip 679 """ 680 } 681 cleanup { 682 deleteDir() 683 } 684 } 685 } 686 687 def runSocksVariant(variant) { 688 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 689 sh """ 690 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 691 SOCKS_SHOP_VARIANT=${variant} ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" examples/socks/... 692 """ 693 } 694 } 695 696 def getTestClusterType(testEnv) { 697 if("kind".equalsIgnoreCase(testEnv)) { 698 return "KIND" 699 } else { 700 return "OKE" 701 } 702 } 703 704 def runGinkgo(testSuitePath) { 705 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 706 sh """ 707 cd ${WORKSPACE}/tests/e2e 708 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 709 """ 710 } 711 } 712 713 def runGinkgoEx(testSuitePath) { 714 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 715 sh """ 716 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 717 ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 718 """ 719 } 720 } 721 722 def runGinkgoFailFast(testSuitePath) { 723 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 724 sh """ 725 cd ${WORKSPACE}/tests/e2e 726 ginkgo -v --fail-fast --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 727 """ 728 } 729 } 730 731 def dumpK8sCluster(dumpDirectory) { 732 sh """ 733 ${WORKSPACE}/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 734 """ 735 } 736 737 def dumpVerrazzanoSystemPods() { 738 sh """ 739 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log" 740 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 741 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log" 742 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 743 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log" 744 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 745 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log" 746 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 747 """ 748 } 749 750 def dumpCertManagerNamespaceLogs() { 751 sh """ 752 kubectl logs --selector=app=cert-manager -n cert-manager > ${WORKSPACE}/platform-operator/scripts/install/build/logs/cert-manager.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 753 kubectl logs --selector=app.kubernetes.io/name=external-dns -n cert-manager > ${WORKSPACE}/platform-operator/scripts/install/build/logs/external-dns.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 754 """ 755 } 756 757 def dumpCattleSystemPods() { 758 sh """ 759 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/cattle-system-pods.log" 760 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 761 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/rancher.log" 762 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 763 """ 764 } 765 766 def dumpNginxIngressControllerLogs() { 767 sh """ 768 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log" 769 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 770 """ 771 } 772 773 def dumpVerrazzanoPlatformOperatorLogs() { 774 sh """ 775 ## dump out verrazzano-platform-operator logs 776 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 777 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 778 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 779 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 780 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 781 echo "------------------------------------------" 782 """ 783 } 784 785 def dumpVerrazzanoApplicationOperatorLogs() { 786 sh """ 787 ## dump out verrazzano-application-operator logs 788 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 789 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 790 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 791 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 792 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 793 echo "------------------------------------------" 794 """ 795 } 796 797 def dumpVerrazzanoApiLogs() { 798 sh """ 799 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log" 800 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 801 """ 802 } 803 804 def getEffectiveDumpOnSuccess() { 805 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 806 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 807 effectiveValue = true 808 echo "Forcing dump on success based on global override setting" 809 } 810 return effectiveValue 811 } 812 813 def setDisplayName() { 814 echo "Start setDisplayName" 815 def causes = currentBuild.getBuildCauses() 816 echo "causes: " + causes.toString() 817 for (cause in causes) { 818 def causeString = cause.toString() 819 echo "current cause: " + causeString 820 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 821 echo "This job was caused by " + causeString 822 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 823 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 824 } else if (causeString.contains("verrazzano-flaky-tests")) { 825 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 826 } 827 } 828 } 829 echo "End setDisplayName" 830 } 831 832 def createImagePullSecrets() { 833 sh """ 834 # Create image pull secrets for Verrazzano docker images 835 cd ${WORKSPACE} 836 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 837 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 838 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 839 """ 840 }