github.com/verrazzano/verrazzano@v1.7.1/ci/oke-ocidns/JenknsfileDnsKind (about) 1 // Copyright (c) 2020, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 // This runs the acceptance tests on an OKE cluster with OCI DNS 5 // This is used during the release process 6 // This will eventually be replaced by the new multi-cluster job! 7 8 def DEFAULT_REPO_URL 9 def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','') 10 def dns_zone_ocid = 'dummy' 11 // for different Jenkins jobs sharing this Jenkins file, the default TEST_ENV (the first in testEnvironments) is different. 12 def testEnvironments = env.JOB_NAME.contains('oci-dns-acceptance') 13 ? ["ocidns_oke", "magicdns_oke", "kind"] 14 : env.JOB_NAME.contains('magic-dns') 15 ? ["magicdns_oke", "ocidns_oke", "kind"] 16 : ["kind_oci_dns", "magicdns_oke", "ocidns_oke"] 17 def acmeEnvironments = [ "staging", "production" ] 18 def certIssuers = [ "self-signed", "acme" ] 19 def agentLabel = env.JOB_NAME.contains('oci-dns-kind-tests') ? "2.0-large-phx" : "2.0-large" 20 21 // pulling "ap-*" from the test regions given discovery of image pull issues 22 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", eu-zurich-1", "me-jeddah-1", 23 "sa-saopaulo-1", "uk-london-1" ] 24 Collections.shuffle(availableRegions) 25 def keepOKEClusterOnFailure = "false" 26 def OKE_CLUSTER_PREFIX = "" 27 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 28 29 pipeline { 30 options { 31 timeout(time: 1, unit: 'HOURS') 32 skipDefaultCheckout true 33 copyArtifactPermission('*'); 34 timestamps () 35 } 36 37 agent { 38 docker { 39 image "${RUNNER_DOCKER_IMAGE}" 40 args "${RUNNER_DOCKER_ARGS}" 41 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 42 registryCredentialsId 'ocir-pull-and-push-account' 43 label "${agentLabel}" 44 } 45 } 46 47 parameters { 48 choice (name: 'KUBERNETES_CLUSTER_VERSION', 49 description: 'Kubernetes Version for KinD Cluster', 50 // 1st choice is the default value 51 choices: [ "1.27", "1.26", "1.25", "1.24" ]) 52 choice (description: 'Use instance principal for oci dns tests', name: 'OCI_DNS_AUTH', 53 // 1st choice is the default value 54 choices: [ "user_principal", "instance_principal" ]) 55 choice (description: 'Specifies DNS scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'DNS_SCOPE', 56 // 1st choice is the default value 57 choices: [ "GLOBAL","PRIVATE" ]) 58 choice (description: 'Specifies Nginx LoadBalancer scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'NGINX_LB_SCOPE', 59 // 1st choice is the default value 60 choices: [ "GLOBAL","PRIVATE" ]) 61 choice (description: 'Specifies Istio LoadBalancer scope. Values: GLOBAL, PRIVATE. Default: GLOBAL',name: 'ISTIO_LB_SCOPE', 62 // 1st choice is the default value 63 choices: [ "GLOBAL","PRIVATE" ]) 64 choice (name: 'CRD_API_VERSION', 65 description: 'This is the API crd version.', 66 // 1st choice is the default value 67 choices: [ "v1beta1", "v1alpha1"]) 68 choice (description: 'Certificate Issuer', name: 'CERT_ISSUER', 69 choices: certIssuers) 70 choice (description: 'ACME Certificate Environment (Staging or Production)', name: 'ACME_ENVIRONMENT', 71 choices: acmeEnvironments) 72 string defaultValue: 'dev', description: 'Verrazzano install profile name', name: "INSTALL_PROFILE", trim: true 73 string defaultValue: 'NONE', description: 'Verrazzano platform operator image name (within ghcr.io/verrazzano repo)', name: 'VERRAZZANO_OPERATOR_IMAGE', trim: true 74 choice (description: 'Verrazzano Test Environment', name: 'TEST_ENV', 75 choices: testEnvironments) 76 string (name: 'GIT_COMMIT_TO_USE', 77 defaultValue: 'NONE', 78 description: 'This is the full git commit hash from the source build to be used for all jobs', 79 trim: true) 80 booleanParam (description: 'Whether to create the cluster with Calico for AT testing', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 81 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 82 string (name: 'TAGGED_TESTS', 83 defaultValue: '', 84 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 85 trim: true) 86 string (name: 'INCLUDED_TESTS', 87 defaultValue: '.*', 88 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 89 trim: true) 90 string (name: 'EXCLUDED_TESTS', 91 defaultValue: '_excluded_test', 92 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 93 trim: true) 94 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 95 } 96 97 environment { 98 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 99 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 100 DOCKER_CREDS = credentials('github-packages-credentials-rw') 101 DOCKER_EMAIL = credentials('github-packages-email') 102 DOCKER_REPO = 'ghcr.io' 103 DOCKER_NAMESPACE = 'verrazzano' 104 CLUSTER_NAME = 'verrazzano' 105 JENKINS_VCN = credentials('runner-vcn-name') 106 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 107 V80_COMPARTMENT_OCID = credentials('v8o-build-compartment') 108 109 110 OCR_CREDS = credentials('ocr-pull-and-push-account') 111 NETRC_FILE = credentials('netrc') 112 OCR_REPO = 'container-registry.oracle.com' 113 GHCR_REPO = 'ghcr.io' 114 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 115 TEST_ENV = "${params.TEST_ENV}" 116 INSTALL_PROFILE = "${params.INSTALL_PROFILE}" 117 GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw') 118 OCIR_CREDS = credentials('ocir-pull-and-push-account') 119 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // needed by install_todo.sh OAM example test 120 DATABASE_PSW = credentials('todo-mysql-password') // needed by install_todo.sh OAM example test 121 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 122 OCIR_PHX_REPO = 'phx.ocir.io' 123 POST_DUMP_FAILED = 'false' 124 GOPATH = '/home/opc/go' 125 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 126 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 127 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 128 129 OCI_CLI_TENANCY = credentials('oci-tenancy') 130 OCI_CLI_USER = credentials('oci-user-ocid') 131 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 132 OCI_CLI_KEY_FILE = credentials('oci-api-key') 133 134 TEST_CONFIG_FILE = "${HOME}/testConfigOke.yaml" 135 CLUSTER_TYPE = getTestClusterType("${TEST_ENV}") 136 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 137 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 138 139 DISABLE_SPINNER=1 140 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 141 142 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 143 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 144 145 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 146 INSTALL_CONFIG_FILE_OCIDNS = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-ocidns.yaml" 147 INSTALL_CONFIG_FILE_KIND = "${WORKSPACE}/tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-ocidns.yaml" 148 KIND_NODE_COUNT = "3" 149 150 //OCI_DNS_ZONE_NAME="z${zoneId}.v8o.io" 151 OCI_DNS_ZONE_NAME="${params.DNS_SCOPE == 'PRIVATE' ? "z${zoneId}-private.v8o.io" : "z${zoneId}.v8o.io"}" 152 CERT_ISSUER="${params.CERT_ISSUER}" 153 ACME_ENVIRONMENT="${params.ACME_ENVIRONMENT}" 154 155 VZ_ENVIRONMENT_NAME = "${'b' + env.BUILD_NUMBER}" 156 157 // Environment variables required to capture cluster snapshot and bug report on test failure 158 DUMP_KUBECONFIG="${KUBECONFIG}" 159 DUMP_COMMAND="${WORKSPACE}/tools/scripts/k8s-dump-cluster.sh" 160 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 161 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 162 163 // Environment variable for Verrazzano CLI executable 164 VZ_COMMAND="${GO_REPO_PATH}/vz" 165 166 // used for console artifact capture on failure 167 JENKINS_READ = credentials('jenkins-auditor') 168 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 169 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 170 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 171 172 //OCI_COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid') 173 //OC_TELEMETRY_URL = credentials('oci-telemetry-url') 174 175 // used to emit metrics 176 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 177 TEST_ENV_LABEL = "${params.TEST_ENV}" 178 179 // used to generate Ginkgo test reports 180 TEST_REPORT = "test-report.xml" 181 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 182 } 183 184 185 186 stages { 187 stage('Clean workspace and checkout') { 188 steps { 189 sh """ 190 echo "${NODE_LABELS}" 191 """ 192 193 script { 194 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 195 if (params.GIT_COMMIT_TO_USE == "NONE") { 196 echo "Specific GIT commit was not specified, use current head" 197 def scmInfo = checkout scm 198 env.GIT_COMMIT = scmInfo.GIT_COMMIT 199 env.GIT_BRANCH = scmInfo.GIT_BRANCH 200 } else { 201 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 202 def scmInfo = checkout([ 203 $class: 'GitSCM', 204 branches: [[name: params.GIT_COMMIT_TO_USE]], 205 doGenerateSubmoduleConfigurations: false, 206 extensions: [], 207 submoduleCfg: [], 208 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 209 env.GIT_COMMIT = scmInfo.GIT_COMMIT 210 env.GIT_BRANCH = scmInfo.GIT_BRANCH 211 // If the commit we were handed is not what the SCM says we are using, fail 212 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 213 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 214 exit 1 215 } 216 } 217 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 218 } 219 220 sh """ 221 cp -f "${NETRC_FILE}" $HOME/.netrc 222 chmod 600 $HOME/.netrc 223 """ 224 225 script { 226 try { 227 sh """ 228 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 229 """ 230 } catch(error) { 231 echo "docker login failed, retrying after sleep" 232 retry(4) { 233 sleep(30) 234 sh """ 235 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 236 """ 237 } 238 } 239 } 240 sh """ 241 rm -rf ${GO_REPO_PATH}/verrazzano 242 mkdir -p ${GO_REPO_PATH}/verrazzano 243 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 244 """ 245 246 script { 247 def props = readProperties file: '.verrazzano-development-version' 248 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 249 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 250 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 251 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 252 // update the description with some meaningful info 253 setDisplayName() 254 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 255 } 256 script { 257 sh """ 258 echo "Downloading VZ CLI from object storage" 259 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 260 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 261 ${GO_REPO_PATH}/vz version 262 """ 263 } 264 } 265 } 266 267 stage("create-dns-zone") { 268 steps { 269 script { 270 def RUNNER_REGION = sh(returnStdout: true, script: "curl -s -H \"Authorization: Bearer Oracle\" http://169.254.169.254/opc/v2/instance/canonicalRegionName").trim() 271 println("runner region is ${RUNNER_REGION}") 272 println("DNS_SCOPE = ${params.DNS_SCOPE}") 273 env.OCI_CLI_REGION = "${RUNNER_REGION}" 274 if (params.DNS_SCOPE == "PRIVATE") { 275 switch(RUNNER_REGION) { 276 case "uk-london-1": 277 env.VCN_VIEW_ID = "${DNS_VIEW_FOR_PRIVATE_DNS_LHR}" 278 break 279 default: 280 env.VCN_VIEW_ID = "${DNS_VIEW_FOR_PRIVATE_DNS_PHX}" 281 break 282 } 283 println("view id is ${env.VCN_VIEW_ID}") 284 //OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${WORKSPACE}/ci/scripts/derive_oke_cluster_name.sh").trim() 285 //env.TF_VAR_label_prefix="${OKE_CLUSTER_PREFIX}" 286 } 287 // VCN_VIEW_ID is used inside oci_dns_ops.sh to associate private zones to view 288 // this is only used when zone is PRIVATE 289 dns_zone_ocid = sh(script: "${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o create -c ${TF_VAR_compartment_id} -s z${zoneId} -k ${params.DNS_SCOPE}", returnStdout: true) 290 } 291 } 292 } 293 294 stage('Prepare KinD environment') { 295 when { expression { return params.TEST_ENV == 'kind_oci_dns' } } 296 environment { 297 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 298 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 299 REALM_USER_PASSWORD = credentials('todo-mysql-password') 300 REALM_NAME = "test-realm" 301 OCI_DNS_COMPARTMENT_OCID = credentials('oci-dns-compartment') 302 OCI_PRIVATE_KEY_FILE = credentials('oci-api-key') 303 OCI_DNS_ZONE_OCID="${dns_zone_ocid}" 304 DNS_SCOPE="${params.DNS_SCOPE}" 305 NGINX_LB_SCOPE="${params.NGINX_LB_SCOPE}" 306 ISTIO_LB_SCOPE="${params.ISTIO_LB_SCOPE}" 307 308 // They will be reference directly in prepare_kind_cluster_dns_pipeline.sh 309 // while constructing the Verrazzano CR 310 CERT_ISSUER="$CERT_ISSUER" 311 ACME_ENVIRONMENT="$ACME_ENVIRONMENT" 312 } 313 steps { 314 script { 315 withCredentials([sshUserPrivateKey(credentialsId: '5fcc03de-31ce-4566-b11f-9de38e5d98fd', keyFileVariable: 'OPC_USER_KEY_FILE', passphraseVariable: 'OPC_USER_PASSPHRASE', usernameVariable: 'OPC_USERNAME')]) { 316 def RUNNER_REGION = sh(returnStdout: true, script: "curl -s -H \"Authorization: Bearer Oracle\" http://169.254.169.254/opc/v2/instance/canonicalRegionName").trim() 317 env.OCI_CLI_REGION = "${RUNNER_REGION}" 318 println("runner region is ${RUNNER_REGION}") 319 if (params.OCI_DNS_AUTH == "instance_principal") { 320 env.OCI_DNS_AUTH="instance_principal" 321 } 322 println("+++++++++++++++++++ RUN DETAILS +++++++++++++++++++++++++") 323 println("AUTH_TYPE = ${params.OCI_DNS_AUTH}") 324 println("DNS_SCOPE = ${params.DNS_SCOPE}") 325 println("NGINX_LB_SCOPE = ${params.NGINX_LB_SCOPE}") 326 println("ISTIO_LB_SCOPE = ${params.ISTIO_LB_SCOPE}") 327 println("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") 328 329 sh """ 330 # cd ${GO_REPO_PATH}/verrazzano 331 ${WORKSPACE}/ci/scripts/prepare_kind_cluster_dns_pipeline.sh ${params.CREATE_CLUSTER_USE_CALICO} 332 """ 333 } 334 } 335 } 336 post { 337 failure { 338 archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true 339 } 340 always { 341 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true 342 } 343 } 344 } 345 346 347 stage('verify-install') { 348 steps { 349 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 350 sh """ 351 cd ${WORKSPACE}/tests/e2e 352 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/... 353 """ 354 } 355 } 356 } 357 358 stage('acceptance-tests') { 359 parallel { 360 stage('metrics') { 361 steps { 362 script { 363 runGinkgo('metrics/syscomponents') 364 } 365 } 366 } 367 stage('restapi') { 368 steps { 369 script { 370 runGinkgo('verify-infra/restapi') 371 } 372 } 373 } 374 stage('vmi') { 375 steps { 376 script { 377 runGinkgo('verify-infra/vmi') 378 } 379 } 380 } 381 stage('oam') { 382 steps { 383 script { 384 runGinkgo('verify-infra/oam') 385 } 386 } 387 } 388 stage('system logging') { 389 environment { 390 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/system-logging" 391 } 392 steps { 393 runGinkgo('logging/system') 394 } 395 } 396 stage('istio authorization policy') { 397 environment { 398 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-istio-auth-policy" 399 } 400 steps { 401 runGinkgo('istio/authz') 402 } 403 } 404 stage('security role based access') { 405 environment { 406 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-rbac" 407 } 408 steps { 409 runGinkgo('security/rbac') 410 } 411 } 412 stage('WebLogic logging') { 413 environment { 414 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-logging" 415 } 416 steps { 417 script { 418 runGinkgoFailFast('logging/weblogic') 419 } 420 } 421 } 422 stage('helidon workload') { 423 environment { 424 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/helidon-workload" 425 } 426 steps { 427 script { 428 runGinkgoFailFast('examples/helidon') 429 } 430 } 431 } 432 stage('weblogic workload') { 433 environment { 434 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/weblogic-workload" 435 } 436 steps { 437 runGinkgoFailFast('workloads/weblogic') 438 } 439 } 440 stage('coherence workload') { 441 environment { 442 DUMP_DIRECTORY="${TEST_DUMP_ROOT}/coherence-workload" 443 } 444 steps { 445 runGinkgoFailFast('workloads/coherence') 446 } 447 } 448 } 449 } 450 } 451 post { 452 always { 453 script { 454 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true || currentBuild.currentResult == 'FAILURE') { 455 dumpK8sCluster('oke-acceptance-tests-cluster-snapshot') 456 } 457 } 458 459 dumpVerrazzanoSystemPods() 460 dumpCattleSystemPods() 461 dumpNginxIngressControllerLogs() 462 dumpVerrazzanoPlatformOperatorLogs() 463 464 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/*full-cluster*/**,**/*bug-report*/**,**/test-cluster-snapshots/**/,**/${TEST_REPORT}", allowEmptyArchive: true 465 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 466 467 sh """ 468 ${WORKSPACE}/tests/e2e/config/scripts/oci_dns_ops.sh -o delete -s z${zoneId} -k ${params.DNS_SCOPE} || echo "Failed to delete DNS zone z${zoneId}" 469 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 470 echo "Failures seen during dumping of artifacts, treat post as failed" 471 exit 1 472 fi 473 """ 474 } 475 failure { 476 sh """ 477 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 478 """ 479 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 480 sh """ 481 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 482 OCI_CLI_AUTH="instance_principal" oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 483 rm archive.zip 484 """ 485 script { 486 if (env.JOB_NAME == "verrazzano/master" || env.JOB_NAME ==~ "verrazzano/release-.*" || env.BRANCH_NAME ==~ "mark/*") { 487 slackSend ( message: "Job Failed - \"${env.JOB_NAME}\" build: ${env.BUILD_NUMBER}\n\nView the log at:\n ${env.BUILD_URL}\n\nBlue Ocean:\n${env.RUN_DISPLAY_URL}" ) 488 } 489 } 490 } 491 cleanup { 492 deleteDir() 493 } 494 } 495 } 496 497 def getTestClusterType(testEnv) { 498 if("kind_oci_dns".equalsIgnoreCase(testEnv)) { 499 return "KIND" 500 } else { 501 return "OKE" 502 } 503 } 504 505 def runGinkgo(testSuitePath) { 506 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 507 sh """ 508 cd ${WORKSPACE}/tests/e2e 509 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 510 """ 511 } 512 } 513 514 def runGinkgoFailFast(testSuitePath) { 515 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 516 sh """ 517 cd ${WORKSPACE}/tests/e2e 518 ginkgo -v --fail-fast --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 519 """ 520 } 521 } 522 523 def dumpK8sCluster(dumpDirectory) { 524 sh """ 525 ${WORKSPACE}/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 526 """ 527 } 528 529 def dumpVerrazzanoSystemPods() { 530 sh """ 531 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-pods.log" 532 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 533 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-certs.log" 534 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 535 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-osd.log" 536 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 537 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-system-es-master.log" 538 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 539 """ 540 } 541 542 def dumpCertManagerNamespaceLogs() { 543 sh """ 544 kubectl logs --selector=app=cert-manager -n cert-manager > ${WORKSPACE}/platform-operator/scripts/install/build/logs/cert-manager.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 545 kubectl logs --selector=app.kubernetes.io/name=external-dns -n cert-manager > ${WORKSPACE}/platform-operator/scripts/install/build/logs/external-dns.log || echo "failed" > ${POST_DUMP_FAILED_FILE} 546 """ 547 } 548 549 def dumpCattleSystemPods() { 550 sh """ 551 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/cattle-system-pods.log" 552 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 553 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/rancher.log" 554 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 555 """ 556 } 557 558 def dumpNginxIngressControllerLogs() { 559 sh """ 560 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/nginx-ingress-controller.log" 561 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 562 """ 563 } 564 565 def dumpVerrazzanoPlatformOperatorLogs() { 566 sh """ 567 ## dump out verrazzano-platform-operator logs 568 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 569 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 570 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 571 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 572 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 573 echo "------------------------------------------" 574 """ 575 } 576 577 def dumpVerrazzanoApplicationOperatorLogs() { 578 sh """ 579 ## dump out verrazzano-application-operator logs 580 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 581 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 582 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 583 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 584 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 585 echo "------------------------------------------" 586 """ 587 } 588 589 def dumpVerrazzanoApiLogs() { 590 sh """ 591 export DIAGNOSTIC_LOG="${WORKSPACE}/platform-operator/scripts/install/build/logs/verrazzano-authproxy.log" 592 ${WORKSPACE}/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 593 """ 594 } 595 596 def getEffectiveDumpOnSuccess() { 597 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 598 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 599 effectiveValue = true 600 echo "Forcing dump on success based on global override setting" 601 } 602 return effectiveValue 603 } 604 605 def setDisplayName() { 606 echo "Start setDisplayName" 607 def causes = currentBuild.getBuildCauses() 608 echo "causes: " + causes.toString() 609 for (cause in causes) { 610 def causeString = cause.toString() 611 echo "current cause: " + causeString 612 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 613 echo "This job was caused by " + causeString 614 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 615 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 616 } else if (causeString.contains("verrazzano-flaky-tests")) { 617 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 618 } 619 } 620 } 621 echo "End setDisplayName" 622 } 623 624 def createImagePullSecrets() { 625 sh """ 626 # Create image pull secrets for Verrazzano docker images 627 cd ${WORKSPACE} 628 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 629 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${GHCR_REPO}" "${GITHUB_PKGS_CREDS_USR}" "${GITHUB_PKGS_CREDS_PSW}" 630 ${WORKSPACE}/tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 631 """ 632 }