github.com/verrazzano/verrazzano@v1.7.0/ci/clusterAPI/JenkinsfileCAPI (about) 1 // Copyright (c) 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 6 // def agentLabel = "largeexperimental" 7 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 8 def clusterSuffix = UUID.randomUUID().toString().substring(0,6).replace('-','') 9 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-frankfurt-1", "eu-zurich-1", "uk-london-1" ] 10 Collections.shuffle(availableRegions) 11 def ocneK8sVersionSupport = [ "v1.25.7","v1.25.11","v1.26.6" ] 12 Collections.shuffle(ocneK8sVersionSupport) 13 14 pipeline { 15 options { 16 timeout(time: 2, unit: 'HOURS') 17 skipDefaultCheckout true 18 timestamps () 19 } 20 21 agent { 22 docker { 23 image "${RUNNER_DOCKER_IMAGE}" 24 args "${RUNNER_DOCKER_ARGS}" 25 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 26 registryCredentialsId 'ocir-pull-and-push-account' 27 label "${agentLabel}" 28 } 29 } 30 31 parameters { 32 choice (name: 'KUBERNETES_CLUSTER_VERSION', 33 description: 'Kubernetes Version for Admin KinD Cluster', 34 // 1st choice is the default value 35 choices: [ "1.27", "1.26", "1.25", "1.24" ]) 36 choice (name: 'OCNE_KUBERNETES_CLUSTER_VERSION', 37 description: 'Kubernetes Version for OCNE Cluster', 38 // 1st choice is the default value 39 choices: ocneK8sVersionSupport) 40 string (name: 'GIT_COMMIT_TO_USE', 41 defaultValue: 'NONE', 42 description: 'This is the full git commit hash from the source build to be used for all jobs', 43 trim: true) 44 choice (description: 'OCI region to launch CAPI clusters in', name: 'CAPI_CLUSTER_REGION', choices: availableRegions ) 45 string (name: 'POD_CIDR', 46 defaultValue: '192.168.0.0/16', 47 description: 'POD cidr to be used in workload/managed OCNE cluster', 48 trim: true) 49 string (name: 'CLUSTER_CIDR', 50 defaultValue: '10.128.0.0/12', 51 description: 'POD CLUSTER_CIDR to be used in workload/managed OCNE cluster', 52 trim: true) 53 string (name: 'OCNE_IMAGE_REPOSITORY', 54 defaultValue: 'container-registry.oracle.com', 55 description: 'OCNE registry value. Default container-registry.oracle.com', 56 trim: true) 57 string (name: 'OCNE_IMAGE_PATH', 58 defaultValue: 'olcne', 59 description: 'Path in OCNE registry value. Default olcne', 60 trim: true) 61 choice (description: 'Number of OCNE control plane nodes', name: 'CONTROL_PLANE_MACHINE_COUNT', choices: ["1", "3"]) 62 choice (description: 'Number of OCNE worker nodes', name: 'NODE_MACHINE_COUNT', choices: ["2","1","3"]) 63 string (name: 'NODE_OCPU',defaultValue: '4',description: 'Number of OCPUs for worker nodes with shape VM.Standard.E4.Flex',trim: true) 64 string (name: 'MEMORY_GBS',defaultValue: '80',description: 'Amount of memory assigned to nodes with shape VM.Standard.E4.Flex',trim: true) 65 string (name: 'ORACLE_LINUX_NAME',defaultValue: 'Oracle-Linux-8',description: 'Oracle Linux Display Name',trim: true) 66 string (name: 'OPERATING_SYSTEM',defaultValue: 'Oracle Linux',description: 'Operating system filter used to fetch stock images',trim: true) 67 string (name: 'OPERATING_SYSTEM_VERSION',defaultValue: '8',description: 'Version of Operating system to fetch',trim: true) 68 69 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 70 defaultValue: 'NONE', 71 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the operator.yaml from Verrazzano repo will be used to create Verrazzano platform operator', 72 trim: true) 73 choice (name: 'WILDCARD_DNS_DOMAIN',description: 'This is the wildcard DNS domain',choices: [ "nip.io", "sslip.io"]) 74 choice (name: 'CRD_API_VERSION', description: 'This is the API crd version.', choices: [ "v1beta1", "v1alpha1"]) 75 booleanParam (description: 'Whether to create the cluster with Calico for AT testing (defaults to true)', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 76 booleanParam (description: 'Whether to dump k8s cluster on success (off by default can be useful to capture for comparing to failed cluster)', name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', defaultValue: false) 77 booleanParam (description: 'Whether to use a database for Grafana persistence', name: 'USE_DB_FOR_GRAFANA', defaultValue: false) 78 string (name: 'CONSOLE_REPO_BRANCH', defaultValue: '',description: 'The branch to check out after cloning the console repository.',trim: true) 79 booleanParam (description: 'Whether to enable debug logging of the istio envoy in the VZ API pod', name: 'ENABLE_API_ENVOY_LOGGING', defaultValue: true) 80 string (name: 'TAGGED_TESTS',defaultValue: '',description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:',trim: true) 81 string (name: 'INCLUDED_TESTS', defaultValue: '.*', description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*',trim: true) 82 string (name: 'EXCLUDED_TESTS', defaultValue: '_excluded_test',description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test',trim: true) 83 string (name: 'INSTALL_PROFILE',defaultValue: 'dev', description: 'Verrazzano install profile (prod/dev) on admin cluster. default: dev',trim: true) 84 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 85 booleanParam (description: 'Whether to run Post-install Verify Tests on workload cluster', name: 'RUN_POST_INSTALL_VERIFY_TESTS', defaultValue: true) 86 booleanParam (description: 'Whether to run Post-install Infra Tests on workload cluster.', name: 'RUN_POST_INSTALL_INFRA_TESTS', defaultValue: false) 87 booleanParam (description: 'Whether to run Post-install Acceptance Tests on workload cluster.', name: 'RUN_POST_INSTALL_ACCEPTANCE_TESTS', defaultValue: false) 88 } 89 90 environment { 91 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 92 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 93 GOPATH = '/home/opc/go' 94 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 95 DOCKER_CREDS = credentials('github-packages-credentials-rw') 96 DOCKER_EMAIL = credentials('github-packages-email') 97 DOCKER_REPO = 'ghcr.io' 98 DOCKER_NAMESPACE = 'verrazzano' 99 NETRC_FILE = credentials('netrc') 100 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 101 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 102 KUBECONFIG = "${WORKSPACE}/test_kubeconfig" 103 VERRAZZANO_KUBECONFIG = "${KUBECONFIG}" 104 OCR_CREDS = credentials('ocr-pull-and-push-account') 105 OCR_REPO = 'container-registry.oracle.com' 106 GITHUB_PKGS_CREDS = credentials('github-packages-credentials-rw') 107 GHCR_REPO = 'ghcr.io' 108 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 109 INSTALL_CONFIG_FILE_KIND = "./tests/e2e/config/scripts/${params.CRD_API_VERSION}/install-verrazzano-kind.yaml" 110 TEST_OVERRIDE_CONFIGMAP_FILE = "./tests/e2e/config/scripts/pre-install-overrides/test-overrides-configmap.yaml" 111 TEST_OVERRIDE_SECRET_FILE = "./tests/e2e/config/scripts/pre-install-overrides/test-overrides-secret.yaml" 112 VZ_CLUSTERRESOURCESET_FILE = "./tests/e2e/clusterapi/capi/templates/cluster-template-verrazzano-resource.yaml" 113 INSTALL_PROFILE = "${params.INSTALL_PROFILE}" 114 KIND_NODE_COUNT = "3" 115 VZ_ENVIRONMENT_NAME = "default" 116 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 117 VERRAZZANO_OPERATOR_IMAGE="${params.VERRAZZANO_OPERATOR_IMAGE}" 118 119 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') // required by WebLogic application and console ingress test 120 DATABASE_PSW = credentials('todo-mysql-password') // required by console ingress test 121 122 // Environment variables required to capture cluster snapshot and bug report on test failure 123 DUMP_KUBECONFIG="${KUBECONFIG}" 124 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 125 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 126 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 127 128 // Environment variable for Verrazzano CLI executable 129 VZ_COMMAND="${GO_REPO_PATH}/vz" 130 131 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 132 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 133 134 // used for console artifact capture on failure 135 JENKINS_READ = credentials('jenkins-auditor') 136 // OCI_CLI_AUTH="instance_principal" 137 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 138 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 139 OCI_OS_COMMIT_BUCKET="verrazzano-builds-by-commit" 140 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 141 142 // used to emit metrics 143 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 144 TEST_ENV_LABEL = "magicdns_ocne" 145 TEST_ENV = "OCNE" 146 K8S_VERSION_LABEL = "${params.KUBERNETES_CLUSTER_VERSION}" 147 SEARCH_HTTP_ENDPOINT = credentials('search-gw-url') 148 SEARCH_PASSWORD = "${PROMETHEUS_CREDENTIALS_PSW}" 149 SEARCH_USERNAME = "${PROMETHEUS_CREDENTIALS_USR}" 150 151 // used to generate Ginkgo test reports 152 TEST_REPORT = "test-report.xml" 153 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 154 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 155 156 //OCI parameters 157 OCI_CLI_TENANCY = credentials('oci-tenancy') 158 OCI_CLI_USER = credentials('oci-user-ocid') 159 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 160 OCI_CLI_KEY_FILE = credentials('oci-api-key') 161 162 // TF parameters 163 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 164 TF_VAR_tenancy_id = credentials('oci-tenancy') 165 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 166 TF_VAR_user_id = credentials('oci-user-ocid') 167 TF_VAR_region = "${params.CAPI_CLUSTER_REGION}" 168 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 169 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 170 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 171 TF_VAR_api_private_key_path = credentials('oci-api-key') 172 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 173 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 174 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 175 176 // CAPI variables 177 OCI_USER_ID = credentials('oci-user-ocid') 178 OCI_CREDENTIALS_FINGERPRINT = credentials('oci-api-key-fingerprint') 179 OCI_TENANCY_ID = credentials('oci-tenancy') 180 OCI_REGION = "${params.CAPI_CLUSTER_REGION}" 181 OCI_COMPARTMENT_ID = credentials('oci-tiburon-dev-compartment-ocid') 182 183 POD_CIDR = "${params.POD_CIDR}" 184 CLUSTER_CIDR = "${params.CLUSTER_CIDR}" 185 OCNE_IMAGE_REPOSITORY = "${params.OCNE_IMAGE_REPOSITORY}" 186 OCNE_IMAGE_PATH = "${params.OCNE_IMAGE_PATH}" 187 KUBERNETES_VERSION = "${params.OCNE_KUBERNETES_CLUSTER_VERSION}" 188 CLUSTER_SUFFIX_ID = "${clusterSuffix}" 189 CLUSTER_NAMESPACE = "ocnecapikluster-${CLUSTER_SUFFIX_ID}" 190 CLUSTER_NAME = "ocnecapikluster-${CLUSTER_SUFFIX_ID}" 191 CAPI_NODE_SSH_KEY_PATH = credentials('oci-tf-pub-ssh-key') 192 CAPI_OCI_PRIVATE_KEY_PATH = credentials('oci-api-key') 193 ORACLE_LINUX_NAME = "${params.ORACLE_LINUX_NAME}" 194 OPERATING_SYSTEM = "${params.OPERATING_SYSTEM}" 195 OPERATING_SYSTEM_VERSION = "${params.OPERATING_SYSTEM_VERSION}" 196 OCI_NODE_MACHINE_TYPE_OCPUS = "${params.NODE_OCPU}" 197 OCI_NODE_MACHINE_MEMORY_GBS = "${params.MEMORY_GBS}" 198 } 199 200 stages { 201 stage('Clean workspace and checkout') { 202 steps { 203 sh """ 204 echo "${NODE_LABELS}" 205 """ 206 207 script { 208 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 209 if (params.GIT_COMMIT_TO_USE == "NONE") { 210 echo "Specific GIT commit was not specified, use current head" 211 def scmInfo = checkout scm 212 env.GIT_COMMIT = scmInfo.GIT_COMMIT 213 env.GIT_BRANCH = scmInfo.GIT_BRANCH 214 } else { 215 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 216 def scmInfo = checkout([ 217 $class: 'GitSCM', 218 branches: [[name: params.GIT_COMMIT_TO_USE]], 219 doGenerateSubmoduleConfigurations: false, 220 extensions: [], 221 submoduleCfg: [], 222 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 223 env.GIT_COMMIT = scmInfo.GIT_COMMIT 224 env.GIT_BRANCH = scmInfo.GIT_BRANCH 225 // If the commit we were handed is not what the SCM says we are using, fail 226 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 227 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 228 exit 1 229 } 230 } 231 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 232 } 233 234 sh """ 235 cp -f "${NETRC_FILE}" $HOME/.netrc 236 chmod 600 $HOME/.netrc 237 """ 238 239 script { 240 try { 241 sh """ 242 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 243 """ 244 } catch(error) { 245 echo "docker login failed, retrying after sleep" 246 retry(4) { 247 sleep(30) 248 sh """ 249 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 250 """ 251 } 252 } 253 } 254 sh """ 255 rm -rf ${GO_REPO_PATH}/verrazzano 256 mkdir -p ${GO_REPO_PATH}/verrazzano 257 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 258 """ 259 260 script { 261 def props = readProperties file: '.verrazzano-development-version' 262 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 263 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 264 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 265 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 266 // update the description with some meaningful info 267 setDisplayName() 268 currentBuild.description = params.KUBERNETES_CLUSTER_VERSION + " : " + SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 269 def RUNNER_REGION = sh(returnStdout: true, script: "curl -s -H \"Authorization: Bearer Oracle\" http://169.254.169.254/opc/v2/instance/canonicalRegionName").trim() 270 println("runner region is ${RUNNER_REGION}, OCI CAPI region is ${env.OCI_REGION}, CAPI Cluster Name is ${env.CLUSTER_NAME}") 271 } 272 script { 273 sh """ 274 echo "Downloading VZ CLI from object storage" 275 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 276 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 277 ${GO_REPO_PATH}/vz version 278 """ 279 } 280 } 281 } 282 283 stage('Acceptance Tests on OCNE cluster') { 284 stages { 285 286 stage('Prepare KinD environment as admin cluster') { 287 environment { 288 KIND_KUBERNETES_CLUSTER_VERSION="${params.KUBERNETES_CLUSTER_VERSION}" 289 OCI_OS_LOCATION="ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}" 290 REALM_USER_PASSWORD = credentials('todo-mysql-password') 291 REALM_NAME = "test-realm" 292 } 293 steps { 294 script { 295 def RUNNER_REGION = sh(returnStdout: true, script: "curl -s -H \"Authorization: Bearer Oracle\" http://169.254.169.254/opc/v2/instance/canonicalRegionName").trim() 296 println("runner region is ${RUNNER_REGION}, OCI CAPI region is ${env.OCI_REGION}, CAPI Cluster Name is ${env.CLUSTER_NAME}") 297 } 298 sh """ 299 cd ${GO_REPO_PATH}/verrazzano 300 ci/scripts/prepare_jenkins_at_environment.sh ${params.CREATE_CLUSTER_USE_CALICO} ${params.WILDCARD_DNS_DOMAIN} ${params.USE_DB_FOR_GRAFANA} 301 """ 302 } 303 post { 304 failure { 305 archiveArtifacts artifacts: "**/kind-logs/**", allowEmptyArchive: true 306 } 307 always { 308 archiveArtifacts artifacts: "acceptance-test-operator.yaml,downloaded-operator.yaml,$INSTALL_CONFIG_FILE_KIND", allowEmptyArchive: true 309 // enable debug logging of Verrazzano api istio proxy 310 script { 311 if (params.ENABLE_API_ENVOY_LOGGING) { 312 sh ''' 313 vz_api_pod=\$(kubectl get pod -n verrazzano-system -l app=verrazzano-authproxy --no-headers -o custom-columns=\":metadata.name\") 314 if [ -z "\$vz_api_pod" ]; then 315 echo "Could not find verrazzano-authproxy pod, not enabling debug logging" 316 else 317 kubectl exec \$vz_api_pod -c istio-proxy -n verrazzano-system -- curl -X POST http://localhost:15000/logging?level=debug 318 fi 319 nginx_ing_pod=\$(kubectl get pod -n verrazzano-ingress-nginx -l app.kubernetes.io/component=controller --no-headers -o custom-columns=\":metadata.name\") 320 if [ -z "\$nginx_ing_pod" ]; then 321 echo "Could not find nginx ingress controller pod, not enabling debug logging" 322 else 323 kubectl exec \$nginx_ing_pod -c istio-proxy -n verrazzano-ingress-nginx -- curl -X POST http://localhost:15000/logging?level=debug 324 fi 325 ''' 326 } 327 } 328 } 329 } 330 } 331 332 stage('Verify-install on admin cluster') { 333 steps { 334 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 335 sh """ 336 cd ${WORKSPACE}/tests/e2e 337 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" verify-install/... 338 """ 339 } 340 } 341 post { 342 always { 343 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 344 junit testResults: '**/*test-result.xml', allowEmptyResults: true 345 } 346 failure { 347 script { 348 dumpK8sCluster('verify-install-admin-failure-dump') 349 } 350 } 351 } 352 } 353 354 stage('Display Verrazzano config to be deployed on workload cluster') { 355 steps { 356 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 357 sh """ 358 yq eval '. | select (.apiVersion == "v1") | .data."verrazzano.yaml"' ${VZ_CLUSTERRESOURCESET_FILE} 359 """ 360 } 361 } 362 } 363 364 365 stage('Deploy OCNE workload cluster using CAPI and install Verrazzano') { 366 steps { 367 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 368 script { 369 // This will deploy an OCNE cluster 370 // Then proceed to create image pull secrets required for jenkins 371 // Also deploy Verrazzano 372 set_capi_variables() 373 runGinkgo('clusterapi/capi') 374 } 375 } 376 } 377 post { 378 always { 379 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-dumps/**', allowEmptyArchive: true 380 junit testResults: '**/*test-result.xml', allowEmptyResults: true 381 } 382 failure { 383 script { 384 dumpK8sCluster('ocne-deploy-failure-admin-cluster') 385 dumpOCNECluster('ocne-deploy-failure-workload-cluster') 386 } 387 } 388 } 389 } 390 391 stage('Display CAPI cluster conditions post VZ install on workload cluster') { 392 steps { 393 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 394 sh """ 395 clusterctl describe cluster -n ${CLUSTER_NAMESPACE} ${CLUSTER_NAMESPACE} --grouping=false --echo --show-conditions all --show-templates --show-resourcesets --show-machinesets 396 """ 397 } 398 } 399 } 400 401 stage('Post-install Verify Tests on workload cluster') { 402 when { 403 expression {params.RUN_POST_INSTALL_VERIFY_TESTS == true} 404 } 405 steps { 406 script { 407 parallel generateVerifyInstallStages("${TEST_DUMP_ROOT}/post-install-verify-tests") 408 } 409 } 410 post { 411 always { 412 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 413 junit testResults: '**/*test-result.xml', allowEmptyResults: true 414 } 415 } 416 } 417 418 stage('Post-install Infra Tests on workload cluster') { 419 when { 420 expression {params.RUN_POST_INSTALL_INFRA_TESTS == true} 421 } 422 steps { 423 script { 424 parallel generateVerifyInfraStages("${TEST_DUMP_ROOT}/post-install-infra-tests") 425 } 426 } 427 post { 428 always { 429 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 430 junit testResults: '**/*test-result.xml', allowEmptyResults: true 431 } 432 } 433 } 434 435 stage('Post-install Acceptance Tests on workload cluster') { 436 when { 437 expression {params.RUN_POST_INSTALL_ACCEPTANCE_TESTS == true} 438 } 439 steps { 440 script { 441 parallel generateAllAcceptanceTestStages("${TEST_DUMP_ROOT}/post-install-acceptance-tests", 'false', 'false') 442 } 443 } 444 post { 445 always { 446 archiveArtifacts artifacts: '**/coverage.html,**/logs/*,**/test-cluster-snapshots/**', allowEmptyArchive: true 447 junit testResults: '**/*test-result.xml', allowEmptyResults: true 448 } 449 } 450 } 451 452 stage('Uninstall vz from workload cluster') { 453 steps { 454 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 455 sh """ 456 TS=\$(date +%Y%m%d%H%M%S%N) 457 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 458 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 459 kubectl --kubeconfig \$TEMPKUBECONFIGPATH get nodes -o wide 460 kubectl --kubeconfig \$TEMPKUBECONFIGPATH get pods -A 461 ${GO_REPO_PATH}/vz --kubeconfig \$TEMPKUBECONFIGPATH uninstall -y --timeout 60m 462 rm -rf \$TEMPKUBECONFIGPATH 463 """ 464 } 465 } 466 467 } 468 469 } 470 471 } 472 473 } 474 475 post { 476 always { 477 script { 478 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 479 dumpVerrazzanoSystemPods() 480 dumpCattleSystemPods() 481 dumpNginxIngressControllerLogs() 482 dumpVerrazzanoPlatformOperatorLogs() 483 dumpVerrazzanoApplicationOperatorLogs() 484 dumpOamKubernetesRuntimeLogs() 485 dumpVerrazzanoApiLogs() 486 dumpCAPIPODLogs() 487 dumpCAPIClusterDetails() 488 } 489 } 490 491 sh """ 492 # Copy the generated test reports to WORKSPACE to archive them 493 mkdir -p ${TEST_REPORT_DIR} 494 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 495 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 496 """ 497 archiveArtifacts artifacts: "**/coverage.html,**/logs/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/bug-report/**,**/Screenshot*.png,**/ConsoleLog*.log,**/${TEST_REPORT}", allowEmptyArchive: true 498 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 499 ensureVZCleanupFromWorkloadCluster() 500 ensureCAPICleanup() 501 deleteCluster() 502 } 503 failure { 504 sh """ 505 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 506 """ 507 archiveArtifacts artifacts: '**/build-console-output.log', allowEmptyArchive: true 508 sh """ 509 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip 510 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip 511 rm archive.zip 512 """ 513 } 514 cleanup { 515 deleteDir() 516 } 517 } 518 } 519 520 def runGinkgo(testSuitePath, kubeConfig = '') { 521 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 522 sh """ 523 if [ ! -z "${kubeConfig}" ]; then 524 export KUBECONFIG="${kubeConfig}" 525 fi 526 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 527 if [ -d "${testSuitePath}" ]; then 528 ginkgo -vv --progress --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 529 fi 530 """ 531 } 532 } 533 534 def runGinkgoWorkloadCluster(testSuitePath,dumpDir='') { 535 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 536 sh """ 537 if [ ! -z "${dumpDir}" ]; then 538 export DUMP_DIRECTORY=${dumpDir} 539 fi 540 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 541 TS=\$(date +%Y%m%d%H%M%S%N) 542 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 543 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 544 export KUBECONFIG=\$TEMPKUBECONFIGPATH 545 ginkgo -v -keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 546 unset KUBECONFIG 547 rm -rf \$TEMPKUBECONFIGPATH 548 """ 549 } 550 } 551 552 def runGinkgoRandomizeWorkloadCluster(testSuitePath,dumpDir='') { 553 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 554 sh """ 555 if [ ! -z "${dumpDir}" ]; then 556 export DUMP_DIRECTORY=${dumpDir} 557 fi 558 TS=\$(date +%Y%m%d%H%M%S%N) 559 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 560 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 561 export KUBECONFIG=\$TEMPKUBECONFIGPATH 562 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 563 if [ -d "${testSuitePath}" ]; then 564 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 565 fi 566 unset KUBECONFIG 567 rm -rf \$TEMPKUBECONFIGPATH 568 """ 569 } 570 } 571 572 def runGinkgoAppTestWorkloadCluster(testSuitePath, namespace, dumpDir='', skipDeploy='false', skipUndeploy='false', component='', appConfig='') { 573 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 574 sh """ 575 if [ ! -z "${dumpDir}" ]; then 576 export DUMP_DIRECTORY=${dumpDir} 577 fi 578 TS=\$(date +%Y%m%d%H%M%S%N) 579 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 580 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 581 export KUBECONFIG=\$TEMPKUBECONFIGPATH 582 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 583 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... -- --skipDeploy=${skipDeploy} --skipUndeploy=${skipUndeploy} --namespace=${namespace} 584 unset KUBECONFIG 585 rm -rf \$TEMPKUBECONFIGPATH 586 """ 587 } 588 } 589 590 591 def saveConsoleScreenShots() { 592 sh "$GO_REPO_PATH/verrazzano/ci/scripts/save_console_test_artifacts.sh" 593 } 594 595 596 // Called in parallel Stage console of Stage Run Acceptance Tests 597 def acceptanceTestsConsole(dumpRoot) { 598 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 599 try { 600 sh "CONSOLE_REPO_BRANCH=${params.CONSOLE_REPO_BRANCH} $GO_REPO_PATH/verrazzano/ci/scripts/run_console_tests.sh" 601 } catch (err) { 602 saveConsoleScreenShots() 603 error "${err}" 604 } 605 } 606 } 607 608 609 def generateVerifyInstallStages(dumpRoot) { 610 return [ 611 "verify-install keycloak": { 612 runGinkgoRandomizeWorkloadCluster('verify-install/keycloak',"${dumpRoot}/verify-install-keycloak") 613 }, 614 "verify-install kubernetes": { 615 runGinkgoRandomizeWorkloadCluster('verify-install/kubernetes',"${dumpRoot}/verify-install-kubernetes") 616 }, 617 "verify-install istio": { 618 runGinkgoRandomizeWorkloadCluster('verify-install/istio',"${dumpRoot}/verify-install-istio") 619 }, 620 "verify-install kiali": { 621 runGinkgoRandomizeWorkloadCluster('verify-install/kiali',"${dumpRoot}/verify-install-kiali") 622 }, 623 "verify-install verrazzano": { 624 runGinkgoRandomizeWorkloadCluster('verify-install/verrazzano',"${dumpRoot}/verify-install-verrazzano") 625 }, 626 "verify-install web": { 627 runGinkgoRandomizeWorkloadCluster('verify-install/web',"${dumpRoot}/verify-install-web") 628 }, 629 "verify-install clusteragent": { 630 runGinkgoRandomizeWorkloadCluster('verify-install/clusteragent',"${dumpRoot}/verify-install-clusteragent") 631 }, 632 "verify-install clusterapi": { 633 runGinkgoRandomizeWorkloadCluster('verify-install/clusterapi',"${dumpRoot}/verify-install-clusterapi") 634 }, 635 "verify-install thanos": { 636 runGinkgoRandomizeWorkloadCluster('verify-install/thanos',"${dumpRoot}/verify-install-thanos") 637 }, 638 "verify-install security": { 639 runGinkgoRandomizeWorkloadCluster('verify-install/security',"${dumpRoot}/verify-install-security") 640 }, 641 "verify-install velero": { 642 runGinkgoRandomizeWorkloadCluster('verify-install/velero',"${dumpRoot}/verify-install-velero") 643 }, 644 "verify-install validators": { 645 runGinkgoRandomizeWorkloadCluster('verify-install/validators',"${dumpRoot}/verify-install-validators") 646 }, 647 ] 648 } 649 650 def generateVerifyInfraStages(dumpRoot) { 651 return [ 652 "verify-scripts": { 653 runGinkgoWorkloadCluster('scripts') 654 }, 655 "verify-infra oam": { 656 runGinkgoRandomizeWorkloadCluster('verify-infra/oam', "${dumpRoot}/verify-infra-oam") 657 }, 658 "verify-infra restapi": { 659 runGinkgoRandomizeWorkloadCluster('verify-infra/restapi', "${dumpRoot}/verify-infra-restapi") 660 }, 661 "verify-infra vmi": { 662 runGinkgoRandomizeWorkloadCluster('verify-infra/vmi', "${dumpRoot}/verify-infra-vmi") 663 }, 664 "system component metrics": { 665 runGinkgoRandomizeWorkloadCluster('metrics/syscomponents', "${dumpRoot}/system-component-metrics") 666 }, 667 "system logging": { 668 runGinkgoRandomizeWorkloadCluster('logging/system', "${dumpRoot}/system-logging") 669 }, 670 ] 671 } 672 673 def generateAllAcceptanceTestStages(dumpRoot, skipDeploy='false', skipUndeploy='false') { 674 return generateSecurityTests(dumpRoot) + 675 generateNonWLSTests(dumpRoot, skipDeploy, skipUndeploy) + 676 generateWLSTests(dumpRoot, skipDeploy, skipUndeploy) 677 } 678 679 def generateSecurityTests(dumpRoot) { 680 return [ 681 "istio authorization policy": { 682 runGinkgoWorkloadCluster('istio/authz', "${dumpRoot}/istio-authz-policy") 683 }, 684 "security rbac": { 685 runGinkgoWorkloadCluster('security/rbac', "${dumpRoot}/sec-role-based-access") 686 }, 687 "security network policies": { 688 if (params.CREATE_CLUSTER_USE_CALICO == true) { 689 runGinkgoWorkloadCluster('security/netpol', "${dumpRoot}/netpol") 690 } else { 691 echo "[INFO] Calico not enabled, skipping network policies tests" 692 } 693 }, 694 ] 695 } 696 697 def generateNonWLSTests(dumpRoot, skipDeploy='false', skipUndeploy='false') { 698 return [ 699 "deployment metrics": { 700 runGinkgoWorkloadCluster('metrics/deploymetrics', "${dumpRoot}/k8sdeploy-workload-metrics") 701 }, 702 "examples logging helidon": { 703 runGinkgoWorkloadCluster('logging/helidon', "${dumpRoot}/examples-logging-helidon") 704 }, 705 "examples springboot": { 706 runGinkgoAppTestWorkloadCluster('examples/springboot', "springboot", "${dumpRoot}/examples-spring", skipDeploy, skipUndeploy) 707 }, 708 "examples helidon": { 709 runGinkgoAppTestWorkloadCluster('examples/helidon', "hello-helidon", "${dumpRoot}/examples-helidon", skipDeploy, skipUndeploy) 710 }, 711 "examples helidon-config": { 712 runGinkgoAppTestWorkloadCluster('examples/helidonconfig', "helidon-config", "${dumpRoot}/examples-helidon-config", skipDeploy, skipUndeploy) 713 }, 714 715 "istio authorization policy": { 716 runGinkgoWorkloadCluster('istio/authz', "${dumpRoot}/istio-authz-policy") 717 }, 718 719 "security network policies": { 720 runGinkgoWorkloadCluster('security/netpol', "${dumpRoot}/netpol") 721 }, 722 723 "examples helidon service template": { 724 runGinkgoAppTestWorkloadCluster('examples/helidon', 'hello-helidon-service-template', "${dumpRoot}/examples-helidon-service-template", 'false', 'false', 'examples/hello-helidon/hello-helidon-comp-service-template.yaml', 'examples/hello-helidon/hello-helidon-app-custom-port.yaml') 725 }, 726 727 "examples helidon manualscalertait": { 728 runGinkgoAppTestWorkloadCluster('examples/helidon', 'examples-helidon-manualscalertait', "${dumpRoot}/examples-helidon-manualscalertait", 'false', 'false','examples/hello-helidon/hello-helidon-comp.yaml', 'examples/hello-helidon/hello-helidon-app-scaler-trait.yaml') 729 }, 730 731 "jaeger helidon": { 732 runGinkgoWorkloadCluster('jaeger/helidon', "${dumpRoot}/jaeger-helidon") 733 }, 734 "jaeger system": { 735 runGinkgoWorkloadCluster('jaeger/system', "${dumpRoot}/jaeger-system") 736 }, 737 ] 738 } 739 740 def generateWLSTests(dumpRoot, skipDeploy='false', skipUndeploy='false') { 741 return [ 742 "weblogic workload": { 743 runGinkgoAppTestWorkloadCluster('workloads/weblogic', "hello-wls", "${dumpRoot}/weblogic-workload", skipDeploy, skipUndeploy) 744 }, 745 "coherence workload": { 746 runGinkgoAppTestWorkloadCluster('workloads/coherence', "hello-coherence", "${dumpRoot}/coherence-workload", skipDeploy, skipUndeploy) 747 }, 748 "console ingress": { 749 // doesn't work with the deployment hooks 750 runGinkgoWorkloadCluster('ingress/console', "wls-console") 751 }, 752 ] 753 } 754 755 def set_capi_variables() { 756 int OCNE_CP_COUNT = params.CONTROL_PLANE_MACHINE_COUNT.toInteger() 757 env.CONTROL_PLANE_MACHINE_COUNT = OCNE_CP_COUNT 758 int OCNE_WORKERS_COUNT = params.NODE_MACHINE_COUNT.toInteger() 759 env.NODE_MACHINE_COUNT = OCNE_WORKERS_COUNT 760 761 env.VPO_IMAGE = sh(returnStdout: true, script: "cat \${WORKSPACE}/acceptance-test-operator.yaml | grep image: | grep ghcr | head -1 | awk '{print \$2}' | cut -d: -f1").trim() 762 env.VPO_TAG = sh(returnStdout: true, script: "cat \${WORKSPACE}/acceptance-test-operator.yaml | grep image: | grep ghcr | head -1 | awk '{print \$2}' | cut -d: -f2-").trim() 763 println("VPO Image is ${env.VPO_IMAGE}, VPO Tag is ${env.VPO_TAG}") 764 } 765 766 def ensureVZCleanupFromWorkloadCluster() { 767 sh """ 768 TS=\$(date +%Y%m%d%H%M%S%N) 769 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 770 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 771 kubectl --kubeconfig \$TEMPKUBECONFIGPATH delete vz --all --timeout=10m 772 rm -rf \$TEMPKUBECONFIGPATH 773 """ 774 } 775 776 def ensureCAPICleanup() { 777 sh """ 778 kubectl delete cl -n ${CLUSTER_NAME} ${CLUSTER_NAME} 779 kubectl delete ns ${CLUSTER_NAME} 780 """ 781 } 782 783 784 def dumpK8sCluster(dumpDirectory) { 785 sh """ 786 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 787 """ 788 } 789 790 def dumpOCNECluster(dumpDirectory) { 791 sh """ 792 TS=\$(date +%Y%m%d%H%M%S%N) 793 TEMPKUBECONFIGPATH=/tmp/${CLUSTER_NAME}-kubeconfig-\$TS 794 kubectl get secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-kubeconfig -o json | jq -r '.data.value' | base64 -d > \$TEMPKUBECONFIGPATH 795 export KUBECONFIG=\$TEMPKUBECONFIGPATH 796 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory} 797 unset KUBECONFIG 798 rm -rf \$TEMPKUBECONFIGPATH 799 """ 800 } 801 802 803 def dumpVerrazzanoSystemPods() { 804 sh """ 805 cd ${GO_REPO_PATH}/verrazzano/platform-operator 806 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-pods.log" 807 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 808 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-certs.log" 809 ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 810 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-osd.log" 811 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 812 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-system-es-master.log" 813 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 814 """ 815 } 816 817 def dumpCattleSystemPods() { 818 sh """ 819 cd ${GO_REPO_PATH}/verrazzano/platform-operator 820 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/cattle-system-pods.log" 821 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 822 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/rancher.log" 823 ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 824 """ 825 } 826 827 def dumpNginxIngressControllerLogs() { 828 sh """ 829 cd ${GO_REPO_PATH}/verrazzano/platform-operator 830 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/nginx-ingress-controller.log" 831 ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 832 """ 833 } 834 835 def dumpVerrazzanoPlatformOperatorLogs() { 836 sh """ 837 ## dump out verrazzano-platform-operator logs 838 mkdir -p ${WORKSPACE}/verrazzano-platform-operator/logs 839 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 840 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${WORKSPACE}/verrazzano-platform-operator/logs/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 841 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 842 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 843 echo "------------------------------------------" 844 """ 845 } 846 847 def dumpVerrazzanoApplicationOperatorLogs() { 848 sh """ 849 ## dump out verrazzano-application-operator logs 850 mkdir -p ${WORKSPACE}/verrazzano-application-operator/logs 851 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 852 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${WORKSPACE}/verrazzano-application-operator/logs/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 853 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 854 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 855 echo "------------------------------------------" 856 """ 857 } 858 859 def dumpOamKubernetesRuntimeLogs() { 860 sh """ 861 ## dump out oam-kubernetes-runtime logs 862 mkdir -p ${WORKSPACE}/oam-kubernetes-runtime/logs 863 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/oam-kubernetes-runtime/logs/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 864 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${WORKSPACE}/verrazzano-application-operator/logs/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 865 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 866 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 867 echo "------------------------------------------" 868 """ 869 } 870 871 def dumpVerrazzanoApiLogs() { 872 sh """ 873 cd ${GO_REPO_PATH}/verrazzano/platform-operator 874 export DIAGNOSTIC_LOG="${VERRAZZANO_INSTALL_LOGS_DIR}/verrazzano-authproxy.log" 875 ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 876 """ 877 } 878 879 def getEffectiveDumpOnSuccess() { 880 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 881 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 882 effectiveValue = true 883 echo "Forcing dump on success based on global override setting" 884 } 885 return effectiveValue 886 } 887 888 def deleteCluster() { 889 sh """ 890 cd ${GO_REPO_PATH}/verrazzano/platform-operator 891 make delete-cluster 892 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 893 echo "Failures seen during dumping of artifacts, treat post as failed" 894 exit 1 895 fi 896 """ 897 } 898 899 def setDisplayName() { 900 echo "Start setDisplayName" 901 def causes = currentBuild.getBuildCauses() 902 echo "causes: " + causes.toString() 903 for (cause in causes) { 904 def causeString = cause.toString() 905 echo "current cause: " + causeString 906 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 907 echo "This job was caused by " + causeString 908 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 909 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 910 } else if (causeString.contains("verrazzano-flaky-tests")) { 911 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 912 } 913 } 914 } 915 echo "End setDisplayName" 916 } 917 918 def dumpCAPIPODLogs() { 919 sh """ 920 ## dump out capi pod logs 921 mkdir -p ${WORKSPACE}/capi-kubernetes-runtime/logs 922 kubectl -n verrazzano-capi logs -l cluster.x-k8s.io/provider=cluster-api > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-controller-manager-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 923 kubectl -n verrazzano-capi logs -l cluster.x-k8s.io/provider=bootstrap-ocne > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-ocne-bootstrap-controller-manager-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 924 kubectl -n verrazzano-capi logs -l cluster.x-k8s.io/provider=control-plane-ocne > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-ocne-control-plane-controller-manager-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 925 kubectl -n verrazzano-capi logs -l cluster.x-k8s.io/provider=infrastructure-oci > ${WORKSPACE}/capi-kubernetes-runtime/logs/capoci-controller-manager-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 926 kubectl -n verrazzano-system logs -l app=verrazzano-cluster-operator > ${WORKSPACE}/capi-kubernetes-runtime/logs/verrazzano-cluster-operator.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 927 kubectl -n verrazzano-capi describe pod -l cluster.x-k8s.io/provider=cluster-api > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-controller-manager-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 928 kubectl -n verrazzano-capi describe pod -l cluster.x-k8s.io/provider=bootstrap-ocne > ${WORKSPACE}/capi-kubernetes-runtime/logs/capi-ocne-bootstrap-controller-manager-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 929 kubectl -n verrazzano-capi describe pod -l cluster.x-k8s.io/provider=control-plane-ocne > ${WORKSPACE}/capi-kubernetes-runtime/logs/ccapi-ocne-control-plane-controller-manager-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 930 kubectl -n verrazzano-capi describe pod -l cluster.x-k8s.io/provider=infrastructure-oci > ${WORKSPACE}/capi-kubernetes-runtime/logs/capoci-controller-manager-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 931 echo "capi logs logs dumped successfully" 932 echo "capi pod description dumped successfully" 933 echo "------------------------------------------" 934 """ 935 } 936 937 def dumpCAPIClusterDetails() { 938 sh """ 939 ## dump out capi conditions 940 mkdir -p ${WORKSPACE}/capi-conditions/logs 941 clusterctl describe cluster -n ${CLUSTER_NAMESPACE} ${CLUSTER_NAMESPACE} --grouping=false --echo --show-conditions all --show-templates --show-resourcesets --show-machinesets > ${WORKSPACE}/capi-conditions/logs/capi-cluster-conditions.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 942 echo "capi cluster conditions dumped successfully to capi-cluster-conditions.out" 943 echo "------------------------------------------" 944 """ 945 }