github.com/verrazzano/verrazzano@v1.7.1/ci/clusterAPI/JenkinsfileCAPI (about) 1 // Copyright (c) 2023, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 // def agentLabel = "largeexperimental" 7 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 8 def clusterSuffix = UUID.randomUUID().toString().substring(0,6).replace('-','') 9 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ] 10 Collections.shuffle(availableRegions) 11 def ocneK8sVersionSupport = [ "v1.25.7","v1.25.11","v1.26.6" ] 12 Collections.shuffle(ocneK8sVersionSupport) 13 14 pipeline { 15 options { 16 timeout(time: 2, unit: 'HOURS') 17 skipDefaultCheckout true 18 timestamps () 19 } 20 21 agent { 22 docker { 23 image "${RUNNER_DOCKER_IMAGE}" 24 args "${RUNNER_DOCKER_ARGS}" 25 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 26 registryCredentialsId 'ocir-pull-and-push-account' 27 label "${agentLabel}" 28 } 29 } 30 31 parameters { 32 choice (name: 'KUBERNETES_CLUSTER_VERSION', 33 description: 'Kubernetes Version for Admin KinD Cluster', 34 // 1st choice is the default value 35 choices: [ "1.27", "1.26", "1.25", "1.24" ]) 36 choice (name: 'OCNE_KUBERNETES_CLUSTER_VERSION', 37 description: 'Kubernetes Version for OCNE Cluster', 38 // 1st choice is the default value 39 choices: ocneK8sVersionSupport) 40 string (name: 'GIT_COMMIT_TO_USE', 41 defaultValue: 'NONE', 42 description: 'This is the full git commit hash from the source build to be used for all jobs', 43 trim: true) 44 choice (description: 'OCI region to launch CAPI clusters in', name: 'CAPI_CLUSTER_REGION', choices: availableRegions ) 45 string (name: 'POD_CIDR', 46 defaultValue: '192.168.0.0/16', 47 description: 'POD cidr to be used in workload/managed OCNE cluster', 48 trim: true) 49 string (name: 'CLUSTER_CIDR', 50 defaultValue: '10.128.0.0/12', 51 description: 'POD CLUSTER_CIDR to be used in workload/managed OCNE cluster', 52 trim: true) 53 string (name: 'OCNE_IMAGE_REPOSITORY', 54 defaultValue: 'container-registry.oracle.com', 55 description: 'OCNE registry value. Default container-registry.oracle.com', 56 trim: true) 57 string (name: 'OCNE_IMAGE_PATH', 58 defaultValue: 'olcne', 59 description: 'Path in OCNE registry value. Default olcne', 60 trim: true) 61 choice (description: 'Number of OCNE control plane nodes', name: 'CONTROL_PLANE_MACHINE_COUNT', choices: ["1", "3"]) 62 choice (description: 'Number of OCNE worker nodes', name: 'NODE_MACHINE_COUNT', choices: ["2","1","3"]) 63 string (name: 'NODE_OCPU',defaultValue: '4',description: 'Number of OCPUs for worker nodes with shape VM.Standard.E4.Flex',trim: true) 64 string (name: 'MEMORY_GBS',defaultValue: '80',description: 'Amount of memory assigned to nodes with shape VM.Standard.E4.Flex',trim: true) 65 string (name: 'ORACLE_LINUX_NAME',defaultValue: 'Oracle-Linux-8',description: 'Oracle Linux Display Name',trim: true) 66 string (name: 'OPERATING_SYSTEM',defaultValue: 'Oracle Linux',description: 'Operating system filter used to fetch stock images',trim: true) 67 string (name: 'OPERATING_SYSTEM_VERSION',defaultValue: '8',description: 'Version of Operating system to fetch',trim: true) 68 69 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 70 defaultValue: 'NONE', 71 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator', 72 trim: true) 73 choice (name: 'WILDCARD_DNS_DOMAIN',description: 'This is the wildcard DNS domain',choices: [ "nip.io", "sslip.io"]) 74 choice (name: 'CRD_API_VERSION', description: 'This is the API crd version.', choices: [ "v1beta1", "v1alpha1"]) 75 booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 76 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 77 booleanParam (description: 'Whether to use a database for Grafana persistence', name: 'USE_DB_FOR_GRAFANA', defaultValue: false) 78 string (name: 'CONSOLE_REPO_BRANCH', defaultValue: '',description: 'The branch to check out after cloning the console repository.',trim: true) 79 booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true) 80 string (name: 'TAGGED_TESTS',defaultValue: '',description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',trim: true) 81 string (name: 'INCLUDED_TESTS', defaultValue: '.*', description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',trim: true) 82 string (name: 'EXCLUDED_TESTS', defaultValue: '_excluded_test',description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',trim: true) 83 string (name: 'INSTALL_PROFILE',defaultValue: 'dev', description: 'Verrazzano install profile (prod/dev) on admin cluster. default: dev',trim: true) 84 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 85 booleanParam (description: 'Whether to run Post-install Verify Tests on workload cluster', name: 'RUN_POST_INSTALL_VERIFY_TESTS', defaultValue: true) 86 booleanParam (description: 'Whether to run Post-install Infra Tests on workload cluster.', name: 'RUN_POST_INSTALL_INFRA_TESTS', defaultValue: false) 87 booleanParam (description: 'Whether to run Post-install Acceptance Tests on workload cluster.', name: 'RUN_POST_INSTALL_ACCEPTANCE_TESTS', defaultValue: false) 88 } 89 90 environment { 91 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 92 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 93 GOPATH = '/home/opc/go' 94 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 95 DOCKER_CREDS = credentials('github-packages-credentials-rw') 96 DOCKER_EMAIL = credentials('github-packages-email') 97 DOCKER_REPO = 'ghcr.io' 98 DOCKER_NAMESPACE = 'verrazzano' 99 NETRC_FILE = credentials('netrc') 100 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 101 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 102 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 103 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 104 OCR_CREDS = credentials('ocr-pull-and-push-account') 105 OCR_REPO = 'container-registry.oracle.com' 106 GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw') 107 GHCR_REPO = 'ghcr.io' 108 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 109 INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml" 110 TEST_OVERRIDE_CONFIGMAP_FILE = "./tests/e2e/config/scripts/pre-install-overrides/test-overrides-configmap.yaml" 111 TEST_OVERRIDE_SECRET_FILE = "./tests/e2e/config/scripts/pre-install-overrides/test-overrides-secret.yaml" 112 VZ_CLUSTERRESOURCESET_FILE = "./tests/e2e/clusterapi/capi/templates/cluster-template-verrazzano-resource.yaml" 113 INSTALL_PROFILE = "${params.INSTALL_PROFILE}" 114 KIND_NODE_COUNT = "3" 115 VZ_ENVIRONMENT_NAME = "default" 116 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 117 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 118 119 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test 120 DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test 121 122 // Environment variables required to capture cluster snapshot and bug report on test failure 123 DUMP_KUBECONFIG="${KUBECONFIG}" 124 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 125 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 126 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 127 128 // Environment variable for Verrazzano CLI executable 129 VZ_COMMAND="${GO_REPO_PATH}/vz" 130 131 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 132 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 133 134 // used for console artifact capture on failure 135 JENKINS_READ = credentials('jenkins-auditor') 136 // OCI_CLI_AUTH="instance_principal" 137 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 138 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 139 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 140 141 // used to emit metrics 142 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 143 TEST_ENV_LABEL = "magicdns_ocne" 144 TEST_ENV = "OCNE" 145 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 146 147 // used to generate Ginkgo test reports 148 TEST_REPORT = "test-report.xml" 149 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 150 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 151 152 //OCI parameters 153 OCI_CLI_TENANCY = credentials('oci-tenancy') 154 OCI_CLI_USER = credentials('oci-user-ocid') 155 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 156 OCI_CLI_KEY_FILE = credentials('oci-api-key') 157 158 // TF parameters 159 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 160 TF_VAR_tenancy_id = credentials('oci-tenancy') 161 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 162 TF_VAR_user_id = credentials('oci-user-ocid') 163 TF_VAR_region = "${params.CAPI_CLUSTER_REGION}" 164 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 165 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 166 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 167 TF_VAR_api_private_key_path = credentials('oci-api-key') 168 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 169 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 170 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 171 172 // CAPI variables 173 OCI_USER_ID = credentials('oci-user-ocid') 174 OCI_CREDENTIALS_FINGERPRINT = credentials('oci-api-key-fingerprint') 175 OCI_TENANCY_ID = credentials('oci-tenancy') 176 OCI_REGION = "${params.CAPI_CLUSTER_REGION}" 177 OCI_COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid') 178 179 POD_CIDR = "${params.POD_CIDR}" 180 CLUSTER_CIDR = "${params.CLUSTER_CIDR}" 181 OCNE_IMAGE_REPOSITORY = "${params.OCNE_IMAGE_REPOSITORY}" 182 OCNE_IMAGE_PATH = "${params.OCNE_IMAGE_PATH}" 183 KUBERNETES_VERSION = "${params.OCNE_KUBERNETES_CLUSTER_VERSION}" 184 CLUSTER_SUFFIX_ID = "${clusterSuffix}" 185 CLUSTER_NAMESPACE = "ocnecapikluster-${CLUSTER_SUFFIX_ID}" 186 CLUSTER_NAME = "ocnecapikluster-${CLUSTER_SUFFIX_ID}" 187 CAPI_NODE_SSH_KEY_PATH = credentials('oci-tf-pub-ssh-key') 188 CAPI_OCI_PRIVATE_KEY_PATH = credentials('oci-api-key') 189 ORACLE_LINUX_NAME = "${params.ORACLE_LINUX_NAME}" 190 OPERATING_SYSTEM = "${params.OPERATING_SYSTEM}" 191 OPERATING_SYSTEM_VERSION = "${params.OPERATING_SYSTEM_VERSION}" 192 OCI_NODE_MACHINE_TYPE_OCPUS = "${params.NODE_OCPU}" 193 OCI_NODE_MACHINE_MEMORY_GBS = "${params.MEMORY_GBS}" 194 } 195 196 stages { 197 stage('Clean workspace and checkout') { 198 steps { 199 sh """ 200 echo "${NODE_LABELS}" 201 """ 202 203 script { 204 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 205 if (params.GIT_COMMIT_TO_USE == "NONE") { 206 echo "Specific GIT commit was not specified, use current head" 207 def scmInfo = checkout scm 208 env.GIT_COMMIT = scmInfo.GIT_COMMIT 209 env.GIT_BRANCH = scmInfo.GIT_BRANCH 210 } else { 211 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 212 def scmInfo = checkout([ 213 $class: 'GitSCM', 214 branches: [[name: params.GIT_COMMIT_TO_USE]], 215 doGenerateSubmoduleConfigurations: false, 216 extensions: [], 217 submoduleCfg: [], 218 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 219 env.GIT_COMMIT = scmInfo.GIT_COMMIT 220 env.GIT_BRANCH = scmInfo.GIT_BRANCH 221 // If the commit we were handed is not what the SCM says we are using, fail 222 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 223 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 224 exit 1 225 } 226 } 227 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 228 } 229 230 sh """ 231 cp -f "${NETRC_FILE}" $HOME/.netrc 232 chmod 600 $HOME/.netrc 233 """ 234 235 script { 236 try { 237 sh """ 238 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 239 """ 240 } catch(error) { 241 echo "docker login failed, retrying after sleep" 242 retry(4) { 243 sleep(30) 244 sh """ 245 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 246 """ 247 } 248 } 249 } 250 sh """ 251 rm -rf ${GO_REPO_PATH}/verrazzano 252 mkdir -p ${GO_REPO_PATH}/verrazzano 253 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 254 """ 255 256 script { 257 def props = readProperties file: '.verrazzano-development-version' 258 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 259 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 260 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 261 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 262 // update the description with some meaningful info 263 setDisplayName() 264 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 265 def RUNNER_REGION = sh(returnStdout: true, script: "curl -s -H \"Authorization: Bearer Oracle\" http://169.254.169.254/opc/v2/instance/canonicalRegionName").trim() 266 println("runner region is ${RUNNER_REGION}, OCI CAPI region is ${env.OCI_REGION}, CAPI Cluster Name is ${env.CLUSTER_NAME}") 267 } 268 script { 269 sh """ 270 echo "Downloading VZ CLI from object storage" 271 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 272 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 273 ${GO_REPO_PATH}/vz version 274 """ 275 } 276 } 277 } 278 279 stage('Acceptance Tests on OCNE cluster') { 280 stages { 281 282 stage('Prepare KinD environment as admin cluster') { 283 environment { 284 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 285 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 286 REALM_USER_PASSWORD = credentials('todo-mysql-password') 287 REALM_NAME = "test-realm" 288 } 289 steps { 290 script { 291 def RUNNER_REGION = sh(returnStdout: true, script: "curl -s -H \"Authorization: Bearer Oracle\" http://169.254.169.254/opc/v2/instance/canonicalRegionName").trim() 292 println("runner region is ${RUNNER_REGION}, OCI CAPI region is ${env.OCI_REGION}, CAPI Cluster Name is ${env.CLUSTER_NAME}") 293 } 294 sh """ 295 cd ${GO_REPO_PATH}/verrazzano 296 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA} 297 """ 298 } 299 post { 300 failure { 301 archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true 302 } 303 always { 304 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true 305 // enable debug logging of Verrazzano api istio proxy 306 script { 307 if (params.ENABLE_API_ENVOY_LOGGING) { 308 sh ''' 309 vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\") 310 if [ -z "\$vz_api_pod" ]; then 311 echo "Could not find verrazzano-authproxy pod, not enabling debug logging" 312 else 313 kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug 314 fi 315 nginx_ing_pod=\$(kubectl get pod -n verrazzano-ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\") 316 if [ -z "\$nginx_ing_pod" ]; then 317 echo "Could not find nginx ingress controller pod, not enabling debug logging" 318 else 319 kubectl exec \$nginx_ing_pod -c istio-proxy -n verrazzano-ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug 320 fi 321 ''' 322 } 323 } 324 } 325 } 326 } 327 328 stage('Verify-install on admin cluster') { 329 steps { 330 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 331 sh """ 332 cd ${WORKSPACE}/tests/e2e 333 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/... 334 """ 335 } 336 } 337 post { 338 always { 339 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 340 junit testResults: '**/*test-result.xml', allowEmptyResults: true 341 } 342 failure { 343 script { 344 dumpK8sCluster('verify-install-admin-failure-dump') 345 } 346 } 347 } 348 } 349 350 stage('Display Verrazzano config to be deployed on workload cluster') { 351 steps { 352 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 353 sh """ 354 yq eval '. | select (.apiVersion == "v1") | .data."verrazzano.yaml"' ${VZ_CLUSTERRESOURCESET_FILE} 355 """ 356 } 357 } 358 } 359 360 361 stage('Deploy OCNE workload cluster using CAPI and install Verrazzano') { 362 steps { 363 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 364 script { 365 // This will deploy an OCNE cluster 366 // Then proceed to create image pull secrets required for jenkins 367 // Also deploy Verrazzano 368 set_capi_variables() 369 runGinkgo('clusterapi/capi') 370 } 371 } 372 } 373 post { 374 always { 375 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 376 junit testResults: '**/*test-result.xml', allowEmptyResults: true 377 } 378 failure { 379 script { 380 dumpK8sCluster('ocne-deploy-failure-admin-cluster') 381 dumpOCNECluster('ocne-deploy-failure-workload-cluster') 382 } 383 } 384 } 385 } 386 387 stage('Display CAPI cluster conditions post VZ install on workload cluster') { 388 steps { 389 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 390 sh """ 391 clusterctl describe cluster -n ${CLUSTER_NAMESPACE} ${CLUSTER_NAMESPACE} --grouping=false --echo --show-conditions all --show-templates --show-resourcesets --show-machinesets 392 """ 393 } 394 } 395 } 396 397 stage('Post-install Verify Tests on workload cluster') { 398 when { 399 expression {params.RUN_POST_INSTALL_VERIFY_TESTS == true} 400 } 401 steps { 402 script { 403 parallel generateVerifyInstallStages("${TEST_DUMP_ROOT}/post-install-verify-tests") 404 } 405 } 406 post { 407 always { 408 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 409 junit testResults: '**/*test-result.xml', allowEmptyResults: true 410 } 411 } 412 } 413 414 stage('Post-install Infra Tests on workload cluster') { 415 when { 416 expression {params.RUN_POST_INSTALL_INFRA_TESTS == true} 417 } 418 steps { 419 script { 420 parallel generateVerifyInfraStages("${TEST_DUMP_ROOT}/post-install-infra-tests") 421 } 422 } 423 post { 424 always { 425 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 426 junit testResults: '**/*test-result.xml', allowEmptyResults: true 427 } 428 } 429 } 430 431 stage('Post-install Acceptance Tests on workload cluster') { 432 when { 433 expression {params.RUN_POST_INSTALL_ACCEPTANCE_TESTS == true} 434 } 435 steps { 436 script { 437 parallel generateAllAcceptanceTestStages("${TEST_DUMP_ROOT}/post-install-acceptance-tests", 'false', 'false') 438 } 439 } 440 post { 441 always { 442 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 443 junit testResults: '**/*test-result.xml', allowEmptyResults: true 444 } 445 } 446 } 447 448 stage('Uninstall vz from workload cluster') { 449 steps { 450 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 451 sh """ 452 TS=\$(date +%Y%m%d%H%M%S%N) 453 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 454 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 455 kubectl --kubeconfig \$TEMPKUBECONFIGPATH get nodes -o wide 456 kubectl --kubeconfig \$TEMPKUBECONFIGPATH get pods -A 457 ${GO_REPO_PATH}/vz --kubeconfig \$TEMPKUBECONFIGPATH uninstall -y --timeout 60m 458 rm -rf \$TEMPKUBECONFIGPATH 459 """ 460 } 461 } 462 463 } 464 465 } 466 467 } 468 469 } 470 471 post { 472 always { 473 script { 474 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 475 dumpVerrazzanoSystemPods() 476 dumpCattleSystemPods() 477 dumpNginxIngressControllerLogs() 478 dumpVerrazzanoPlatformOperatorLogs() 479 dumpVerrazzanoApplicationOperatorLogs() 480 dumpOamKubernetesRuntimeLogs() 481 dumpVerrazzanoApiLogs() 482 dumpCAPIPODLogs() 483 dumpCAPIClusterDetails() 484 } 485 } 486 487 sh """ 488 # Copy the generated test reports to WORKSPACE to archive them 489 mkdir -p ${TEST_REPORT_DIR} 490 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 491 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 492 """ 493 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true 494 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 495 ensureVZCleanupFromWorkloadCluster() 496 ensureCAPICleanup() 497 deleteCluster() 498 } 499 failure { 500 sh """ 501 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 502 """ 503 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 504 sh """ 505 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 506 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 507 rm archive.zip 508 """ 509 } 510 cleanup { 511 deleteDir() 512 } 513 } 514 } 515 516 def runGinkgo(testSuitePath, kubeConfig = '') { 517 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 518 sh """ 519 if [ ! -z "${kubeConfig}" ]; then 520 export KUBECONFIG="${kubeConfig}" 521 fi 522 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 523 if [ -d "${testSuitePath}" ]; then 524 ginkgo -vv --progress --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 525 fi 526 """ 527 } 528 } 529 530 def runGinkgoWorkloadCluster(testSuitePath,dumpDir='') { 531 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 532 sh """ 533 if [ ! -z "${dumpDir}" ]; then 534 export DUMP_DIRECTORY=${dumpDir} 535 fi 536 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 537 TS=\$(date +%Y%m%d%H%M%S%N) 538 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 539 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 540 export KUBECONFIG=\$TEMPKUBECONFIGPATH 541 ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 542 unset KUBECONFIG 543 rm -rf \$TEMPKUBECONFIGPATH 544 """ 545 } 546 } 547 548 def runGinkgoRandomizeWorkloadCluster(testSuitePath,dumpDir='') { 549 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 550 sh """ 551 if [ ! -z "${dumpDir}" ]; then 552 export DUMP_DIRECTORY=${dumpDir} 553 fi 554 TS=\$(date +%Y%m%d%H%M%S%N) 555 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 556 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 557 export KUBECONFIG=\$TEMPKUBECONFIGPATH 558 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 559 if [ -d "${testSuitePath}" ]; then 560 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 561 fi 562 unset KUBECONFIG 563 rm -rf \$TEMPKUBECONFIGPATH 564 """ 565 } 566 } 567 568 def runGinkgoAppTestWorkloadCluster(testSuitePath, namespace, dumpDir='', skipDeploy='false', skipUndeploy='false', component='', appConfig='') { 569 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 570 sh """ 571 if [ ! -z "${dumpDir}" ]; then 572 export DUMP_DIRECTORY=${dumpDir} 573 fi 574 TS=\$(date +%Y%m%d%H%M%S%N) 575 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 576 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 577 export KUBECONFIG=\$TEMPKUBECONFIGPATH 578 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 579 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace} 580 unset KUBECONFIG 581 rm -rf \$TEMPKUBECONFIGPATH 582 """ 583 } 584 } 585 586 587 def saveConsoleScreenShots() { 588 sh "$GO_REPO_PATH/verrazzano/ci/scripts/save_console_test_artifacts.sh" 589 } 590 591 592 // Called in parallel Stage console of Stage Run Acceptance Tests 593 def acceptanceTestsConsole(dumpRoot) { 594 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 595 try { 596 sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} $GO_REPO_PATH/verrazzano/ci/scripts/run_console_tests.sh" 597 } catch (err) { 598 saveConsoleScreenShots() 599 error "${err}" 600 } 601 } 602 } 603 604 605 def generateVerifyInstallStages(dumpRoot) { 606 return [ 607 "verify-install keycloak": { 608 runGinkgoRandomizeWorkloadCluster('verify-install/keycloak',"${dumpRoot}/verify-install-keycloak") 609 }, 610 "verify-install kubernetes": { 611 runGinkgoRandomizeWorkloadCluster('verify-install/kubernetes',"${dumpRoot}/verify-install-kubernetes") 612 }, 613 "verify-install istio": { 614 runGinkgoRandomizeWorkloadCluster('verify-install/istio',"${dumpRoot}/verify-install-istio") 615 }, 616 "verify-install kiali": { 617 runGinkgoRandomizeWorkloadCluster('verify-install/kiali',"${dumpRoot}/verify-install-kiali") 618 }, 619 "verify-install verrazzano": { 620 runGinkgoRandomizeWorkloadCluster('verify-install/verrazzano',"${dumpRoot}/verify-install-verrazzano") 621 }, 622 "verify-install web": { 623 runGinkgoRandomizeWorkloadCluster('verify-install/web',"${dumpRoot}/verify-install-web") 624 }, 625 "verify-install clusteragent": { 626 runGinkgoRandomizeWorkloadCluster('verify-install/clusteragent',"${dumpRoot}/verify-install-clusteragent") 627 }, 628 "verify-install clusterapi": { 629 runGinkgoRandomizeWorkloadCluster('verify-install/clusterapi',"${dumpRoot}/verify-install-clusterapi") 630 }, 631 "verify-install thanos": { 632 runGinkgoRandomizeWorkloadCluster('verify-install/thanos',"${dumpRoot}/verify-install-thanos") 633 }, 634 "verify-install security": { 635 runGinkgoRandomizeWorkloadCluster('verify-install/security',"${dumpRoot}/verify-install-security") 636 }, 637 "verify-install velero": { 638 runGinkgoRandomizeWorkloadCluster('verify-install/velero',"${dumpRoot}/verify-install-velero") 639 }, 640 "verify-install validators": { 641 runGinkgoRandomizeWorkloadCluster('verify-install/validators',"${dumpRoot}/verify-install-validators") 642 }, 643 ] 644 } 645 646 def generateVerifyInfraStages(dumpRoot) { 647 return [ 648 "verify-scripts": { 649 runGinkgoWorkloadCluster('scripts') 650 }, 651 "verify-infra oam": { 652 runGinkgoRandomizeWorkloadCluster('verify-infra/oam', "${dumpRoot}/verify-infra-oam") 653 }, 654 "verify-infra restapi": { 655 runGinkgoRandomizeWorkloadCluster('verify-infra/restapi', "${dumpRoot}/verify-infra-restapi") 656 }, 657 "verify-infra vmi": { 658 runGinkgoRandomizeWorkloadCluster('verify-infra/vmi', "${dumpRoot}/verify-infra-vmi") 659 }, 660 "system component metrics": { 661 runGinkgoRandomizeWorkloadCluster('metrics/syscomponents', "${dumpRoot}/system-component-metrics") 662 }, 663 "system logging": { 664 runGinkgoRandomizeWorkloadCluster('logging/system', "${dumpRoot}/system-logging") 665 }, 666 ] 667 } 668 669 def generateAllAcceptanceTestStages(dumpRoot, skipDeploy='false', skipUndeploy='false') { 670 return generateSecurityTests(dumpRoot) + 671 generateNonWLSTests(dumpRoot, skipDeploy, skipUndeploy) + 672 generateWLSTests(dumpRoot, skipDeploy, skipUndeploy) 673 } 674 675 def generateSecurityTests(dumpRoot) { 676 return [ 677 "istio authorization policy": { 678 runGinkgoWorkloadCluster('istio/authz', "${dumpRoot}/istio-authz-policy") 679 }, 680 "security rbac": { 681 runGinkgoWorkloadCluster('security/rbac', "${dumpRoot}/sec-role-based-access") 682 }, 683 "security network policies": { 684 if (params.CREATE_CLUSTER_USE_CALICO == true) { 685 runGinkgoWorkloadCluster('security/netpol', "${dumpRoot}/netpol") 686 } else { 687 echo "[INFO] Calico not enabled, skipping network policies tests" 688 } 689 }, 690 ] 691 } 692 693 def generateNonWLSTests(dumpRoot, skipDeploy='false', skipUndeploy='false') { 694 return [ 695 "deployment metrics": { 696 runGinkgoWorkloadCluster('metrics/deploymetrics', "${dumpRoot}/k8sdeploy-workload-metrics") 697 }, 698 "examples logging helidon": { 699 runGinkgoWorkloadCluster('logging/helidon', "${dumpRoot}/examples-logging-helidon") 700 }, 701 "examples springboot": { 702 runGinkgoAppTestWorkloadCluster('examples/springboot', "springboot", "${dumpRoot}/examples-spring", skipDeploy, skipUndeploy) 703 }, 704 "examples helidon": { 705 runGinkgoAppTestWorkloadCluster('examples/helidon', "hello-helidon", "${dumpRoot}/examples-helidon", skipDeploy, skipUndeploy) 706 }, 707 "examples helidon-config": { 708 runGinkgoAppTestWorkloadCluster('examples/helidonconfig', "helidon-config", "${dumpRoot}/examples-helidon-config", skipDeploy, skipUndeploy) 709 }, 710 711 "istio authorization policy": { 712 runGinkgoWorkloadCluster('istio/authz', "${dumpRoot}/istio-authz-policy") 713 }, 714 715 "security network policies": { 716 runGinkgoWorkloadCluster('security/netpol', "${dumpRoot}/netpol") 717 }, 718 719 "examples helidon service template": { 720 runGinkgoAppTestWorkloadCluster('examples/helidon', 'hello-helidon-service-template', "${dumpRoot}/examples-helidon-service-template", 'false', 'false', 'examples/hello-helidon/hello-helidon-comp-service-template.yaml', 'examples/hello-helidon/hello-helidon-app-custom-port.yaml') 721 }, 722 723 "examples helidon manualscalertait": { 724 runGinkgoAppTestWorkloadCluster('examples/helidon', 'examples-helidon-manualscalertait', "${dumpRoot}/examples-helidon-manualscalertait", 'false', 'false','examples/hello-helidon/hello-helidon-comp.yaml', 'examples/hello-helidon/hello-helidon-app-scaler-trait.yaml') 725 }, 726 727 "jaeger helidon": { 728 runGinkgoWorkloadCluster('jaeger/helidon', "${dumpRoot}/jaeger-helidon") 729 }, 730 "jaeger system": { 731 runGinkgoWorkloadCluster('jaeger/system', "${dumpRoot}/jaeger-system") 732 }, 733 ] 734 } 735 736 def generateWLSTests(dumpRoot, skipDeploy='false', skipUndeploy='false') { 737 return [ 738 "weblogic workload": { 739 runGinkgoAppTestWorkloadCluster('workloads/weblogic', "hello-wls", "${dumpRoot}/weblogic-workload", skipDeploy, skipUndeploy) 740 }, 741 "coherence workload": { 742 runGinkgoAppTestWorkloadCluster('workloads/coherence', "hello-coherence", "${dumpRoot}/coherence-workload", skipDeploy, skipUndeploy) 743 }, 744 "console ingress": { 745 // doesn't work with the deployment hooks 746 runGinkgoWorkloadCluster('ingress/console', "wls-console") 747 }, 748 ] 749 } 750 751 def set_capi_variables() { 752 int OCNE_CP_COUNT = params.CONTROL_PLANE_MACHINE_COUNT.toInteger() 753 env.CONTROL_PLANE_MACHINE_COUNT = OCNE_CP_COUNT 754 int OCNE_WORKERS_COUNT = params.NODE_MACHINE_COUNT.toInteger() 755 env.NODE_MACHINE_COUNT = OCNE_WORKERS_COUNT 756 757 env.VPO_IMAGE = sh(returnStdout: true, script: "cat \${WORKSPACE}/acceptance-test-operator.yaml | grep image: | grep ghcr | head -1 | awk '{print \$2}' | cut -d: -f1").trim() 758 env.VPO_TAG = sh(returnStdout: true, script: "cat \${WORKSPACE}/acceptance-test-operator.yaml | grep image: | grep ghcr | head -1 | awk '{print \$2}' | cut -d: -f2-").trim() 759 println("VPO Image is ${env.VPO_IMAGE}, VPO Tag is ${env.VPO_TAG}") 760 } 761 762 def ensureVZCleanupFromWorkloadCluster() { 763 sh """ 764 TS=\$(date +%Y%m%d%H%M%S%N) 765 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 766 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 767 kubectl --kubeconfig \$TEMPKUBECONFIGPATH delete vz --all --timeout=10m 768 rm -rf \$TEMPKUBECONFIGPATH 769 """ 770 } 771 772 def ensureCAPICleanup() { 773 sh """ 774 kubectl delete cl -n ${CLUSTER_NAME} ${CLUSTER_NAME} 775 kubectl delete ns ${CLUSTER_NAME} 776 """ 777 } 778 779 780 def dumpK8sCluster(dumpDirectory) { 781 sh """ 782 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 783 """ 784 } 785 786 def dumpOCNECluster(dumpDirectory) { 787 sh """ 788 TS=\$(date +%Y%m%d%H%M%S%N) 789 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 790 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 791 export KUBECONFIG=\$TEMPKUBECONFIGPATH 792 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 793 unset KUBECONFIG 794 rm -rf \$TEMPKUBECONFIGPATH 795 """ 796 } 797 798 799 def dumpVerrazzanoSystemPods() { 800 sh """ 801 cd ${GO_REPO_PATH}/verrazzano/platform-operator 802 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log" 803 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 804 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log" 805 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 806 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log" 807 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 808 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log" 809 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 810 """ 811 } 812 813 def dumpCattleSystemPods() { 814 sh """ 815 cd ${GO_REPO_PATH}/verrazzano/platform-operator 816 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log" 817 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 818 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log" 819 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 820 """ 821 } 822 823 def dumpNginxIngressControllerLogs() { 824 sh """ 825 cd ${GO_REPO_PATH}/verrazzano/platform-operator 826 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log" 827 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 828 """ 829 } 830 831 def dumpVerrazzanoPlatformOperatorLogs() { 832 sh """ 833 ## dump out verrazzano-platform-operator logs 834 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 835 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 836 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 837 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 838 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 839 echo "------------------------------------------" 840 """ 841 } 842 843 def dumpVerrazzanoApplicationOperatorLogs() { 844 sh """ 845 ## dump out verrazzano-application-operator logs 846 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 847 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 848 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 849 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 850 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 851 echo "------------------------------------------" 852 """ 853 } 854 855 def dumpOamKubernetesRuntimeLogs() { 856 sh """ 857 ## dump out oam-kubernetes-runtime logs 858 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 859 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 860 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 861 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 862 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 863 echo "------------------------------------------" 864 """ 865 } 866 867 def dumpVerrazzanoApiLogs() { 868 sh """ 869 cd ${GO_REPO_PATH}/verrazzano/platform-operator 870 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log" 871 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 872 """ 873 } 874 875 def getEffectiveDumpOnSuccess() { 876 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 877 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 878 effectiveValue = true 879 echo "Forcing dump on success based on global override setting" 880 } 881 return effectiveValue 882 } 883 884 def deleteCluster() { 885 sh """ 886 cd ${GO_REPO_PATH}/verrazzano/platform-operator 887 make delete-cluster 888 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 889 echo "Failures seen during dumping of artifacts, treat post as failed" 890 exit 1 891 fi 892 """ 893 } 894 895 def setDisplayName() { 896 echo "Start setDisplayName" 897 def causes = currentBuild.getBuildCauses() 898 echo "causes: " + causes.toString() 899 for (cause in causes) { 900 def causeString = cause.toString() 901 echo "current cause: " + causeString 902 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 903 echo "This job was caused by " + causeString 904 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 905 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 906 } else if (causeString.contains("verrazzano-flaky-tests")) { 907 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 908 } 909 } 910 } 911 echo "End setDisplayName" 912 } 913 914 def dumpCAPIPODLogs() { 915 sh """ 916 ## dump out capi pod logs 917 mkdir -p ${WORKSPACE}/capi-kubernetes-runtime/logs 918 kubectl -n verrazzano-capi logs -l cluster.x-k8s.io/provider=cluster-api > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-controller-manager-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 919 kubectl -n verrazzano-capi logs -l cluster.x-k8s.io/provider=bootstrap-ocne > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-ocne-bootstrap-controller-manager-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 920 kubectl -n verrazzano-capi logs -l cluster.x-k8s.io/provider=control-plane-ocne > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-ocne-control-plane-controller-manager-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 921 kubectl -n verrazzano-capi logs -l cluster.x-k8s.io/provider=infrastructure-oci > ${WORKSPACE}/capi-kubernetes-runtime/logs/capoci-controller-manager-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 922 kubectl -n verrazzano-system logs -l app=verrazzano-cluster-operator > ${WORKSPACE}/capi-kubernetes-runtime/logs/verrazzano-cluster-operator.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 923 kubectl -n verrazzano-capi describe pod -l cluster.x-k8s.io/provider=cluster-api > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-controller-manager-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 924 kubectl -n verrazzano-capi describe pod -l cluster.x-k8s.io/provider=bootstrap-ocne > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-ocne-bootstrap-controller-manager-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 925 kubectl -n verrazzano-capi describe pod -l cluster.x-k8s.io/provider=control-plane-ocne > ${WORKSPACE}/capi-kubernetes-runtime/logs/ccapi-ocne-control-plane-controller-manager-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 926 kubectl -n verrazzano-capi describe pod -l cluster.x-k8s.io/provider=infrastructure-oci > ${WORKSPACE}/capi-kubernetes-runtime/logs/capoci-controller-manager-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 927 echo "capi logs logs dumped successfully" 928 echo "capi pod description dumped successfully" 929 echo "------------------------------------------" 930 """ 931 } 932 933 def dumpCAPIClusterDetails() { 934 sh """ 935 ## dump out capi conditions 936 mkdir -p ${WORKSPACE}/capi-conditions/logs 937 clusterctl describe cluster -n ${CLUSTER_NAMESPACE} ${CLUSTER_NAMESPACE} --grouping=false --echo --show-conditions all --show-templates --show-resourcesets --show-machinesets > ${WORKSPACE}/capi-conditions/logs/capi-cluster-conditions.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 938 echo "capi cluster conditions dumped successfully to capi-cluster-conditions.out" 939 echo "------------------------------------------" 940 """ 941 }