github.com/verrazzano/verrazzano@v1.7.1/ci/multicluster/Jenkinsfile (about) 1 // Copyright (c) 2021, 2024, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 def DOCKER_IMAGE_TAG 5 def SKIP_ACCEPTANCE_TESTS = false 6 def EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = false 7 def availableRegions = [ "us-ashburn-1", "ca-montreal-1", "ca-toronto-1", "eu-amsterdam-1", "eu-zurich-1", "uk-london-1" ] 8 def acmeEnvironments = [ "staging", "production" ] 9 Collections.shuffle(availableRegions) 10 def zoneId = UUID.randomUUID().toString().substring(0,6).replace('-','') 11 def dns_zone_ocid = 'dummy' 12 def OKE_CLUSTER_PREFIX = "" 13 def agentLabel = env.JOB_NAME.contains('master') ? "2.0-large-phx" : "2.0-large" 14 15 installerFileName = "install-verrazzano.yaml" 16 minimalFileName = "minimal-verrazzano.yaml" 17 18 pipeline { 19 options { 20 timeout(time: 2, unit: 'HOURS') 21 skipDefaultCheckout true 22 timestamps () 23 } 24 25 agent { 26 docker { 27 image "${RUNNER_DOCKER_IMAGE}" 28 args "${RUNNER_DOCKER_ARGS}" 29 registryUrl "${RUNNER_DOCKER_REGISTRY_URL}" 30 registryCredentialsId 'ocir-pull-and-push-account' 31 label "${agentLabel}" 32 } 33 } 34 35 parameters { 36 booleanParam (description: 'Whether to use External Elasticsearch', name: 'EXTERNAL_ELASTICSEARCH', defaultValue: false) 37 choice (description: 'Number of Cluster', name: 'TOTAL_CLUSTERS', choices: ["2", "1", "3"]) 38 choice (description: 'Verrazzano Test Environment', name: 'TEST_ENV', 39 choices: ["KIND", "magicdns_oke", "ocidns_oke"]) 40 choice (description: 'ACME Certificate Environment (Staging or Production)', name: 'ACME_ENVIRONMENT', 41 choices: acmeEnvironments) 42 choice (description: 'OCI region to launch OKE clusters', name: 'OKE_CLUSTER_REGION', 43 // 1st choice is the default value 44 choices: availableRegions ) 45 choice (description: 'OKE node pool configuration', name: 'OKE_NODE_POOL', 46 // 1st choice is the default value 47 choices: [ "VM.Standard.E3.Flex-4-2", "VM.Standard2.4-2", "VM.Standard.E3.Flex-8-2" ]) 48 choice (name: 'OKE_CLUSTER_VERSION', 49 description: 'Kubernetes Version for OKE Cluster', 50 // 1st choice is the default value 51 choices: [ "v1.27.2", "v1.26.2", "v1.25.4", "v1.24.1"]) 52 choice (name: 'KIND_CLUSTER_VERSION', 53 description: 'Kubernetes Version for KIND Cluster', 54 // 1st choice is the default value 55 choices: [ "1.27", "1.26", "1.25", "1.24" ]) 56 string (name: 'GIT_COMMIT_TO_USE', 57 defaultValue: 'NONE', 58 description: 'This is the full git commit hash from the source build to be used for all jobs', 59 trim: true) 60 string (name: 'VERRAZZANO_OPERATOR_IMAGE', 61 defaultValue: 'NONE', 62 description: 'Verrazzano platform operator image name (in ghcr.io repo). If not specified, the latest operator.yaml published to the Verrazzano Object Store will be used', 63 trim: true) 64 choice (name: 'ADMIN_CLUSTER_PROFILE', 65 description: 'Verrazzano Admin Cluster install profile name', 66 // 1st choice is the default value 67 choices: [ "prod", "dev" ]) 68 choice (name: 'MANAGED_CLUSTER_PROFILE', 69 description: 'Verrazzano Managed Cluster install profile name', 70 // 1st choice is the default value 71 choices: [ "managed-cluster", "prod", "dev" ]) 72 choice (name: 'WILDCARD_DNS_DOMAIN', 73 description: 'This is the wildcard DNS domain', 74 // 1st choice is the default value 75 choices: [ "nip.io", "sslip.io"]) 76 choice (name: 'CRD_API_VERSION', 77 description: 'This is the API crd version.', 78 // 1st choice is the default value 79 choices: [ "v1beta1", "v1alpha1"]) 80 booleanParam (description: 'Whether to create the cluster with Calico for AT testing', name: 'CREATE_CLUSTER_USE_CALICO', defaultValue: true) 81 booleanParam (name: 'DUMP_K8S_CLUSTER_ON_SUCCESS', description: 'Whether to dump k8s cluster on success (off by default, can be useful to capture for comparing to failed cluster)', defaultValue: false) 82 string (name: 'CONSOLE_REPO_BRANCH', 83 defaultValue: '', 84 description: 'The branch to check out after cloning the console repository.', 85 trim: true) 86 string (name: 'TAGGED_TESTS', 87 defaultValue: '', 88 description: 'A comma separated list of build tags for tests that should be executed (e.g. unstable_test). Default:', 89 trim: true) 90 string (name: 'INCLUDED_TESTS', 91 defaultValue: '.*', 92 description: 'A regex matching any fully qualified test file that should be executed (e.g. examples/helidon/). Default: .*', 93 trim: true) 94 string (name: 'EXCLUDED_TESTS', 95 defaultValue: '_excluded_test', 96 description: 'A regex matching any fully qualified test file that should not be executed (e.g. multicluster/|_excluded_test). Default: _excluded_test', 97 trim: true) 98 booleanParam (description: 'Whether to capture full cluster snapshot on test failure', name: 'CAPTURE_FULL_CLUSTER', defaultValue: false) 99 } 100 101 environment { 102 DOCKER_PLATFORM_CI_IMAGE_NAME = 'verrazzano-platform-operator-jenkins' 103 DOCKER_PLATFORM_PUBLISH_IMAGE_NAME = 'verrazzano-platform-operator' 104 DOCKER_OAM_CI_IMAGE_NAME = 'verrazzano-application-operator-jenkins' 105 DOCKER_OAM_PUBLISH_IMAGE_NAME = 'verrazzano-application-operator' 106 GOPATH = '/home/opc/go' 107 GO_REPO_PATH = "${GOPATH}/src/github.com/verrazzano" 108 DOCKER_CREDS = credentials('github-packages-credentials-rw') 109 DOCKER_EMAIL = credentials('github-packages-email') 110 DOCKER_REPO = 'ghcr.io' 111 DOCKER_NAMESPACE = 'verrazzano' 112 NETRC_FILE = credentials('netrc') 113 CLUSTER_NAME_PREFIX = 'verrazzano' 114 TESTS_EXECUTED_FILE = "${WORKSPACE}/tests_executed_file.tmp" 115 POST_DUMP_FAILED_FILE = "${WORKSPACE}/post_dump_failed_file.tmp" 116 KUBECONFIG_DIR = "${WORKSPACE}/kubeconfig" 117 118 OCR_CREDS = credentials('ocr-pull-and-push-account') 119 OCR_REPO = 'container-registry.oracle.com' 120 IMAGE_PULL_SECRET = 'verrazzano-container-registry' 121 122 TEST_ENV = "${params.TEST_ENV}" 123 MANAGED_CLUSTER_PROFILE = "${params.MANAGED_CLUSTER_PROFILE}" 124 ADMIN_CLUSTER_PROFILE = "${params.ADMIN_CLUSTER_PROFILE}" 125 126 // Find a better way to handle this 127 // OKE_CLUSTER_VERSION = "${params.KUBERNETES_VERSION == '1.17' ? 'v1.17.13' : 'v1.18.10'}" 128 TF_VAR_compartment_id = credentials('oci-tiburon-dev-compartment-ocid') 129 TF_VAR_tenancy_id = credentials('oci-tenancy') 130 TF_VAR_tenancy_name = credentials('oci-tenancy-name') 131 TF_VAR_user_id = credentials('oci-user-ocid') 132 TF_VAR_region = "${params.OKE_CLUSTER_REGION}" 133 TF_VAR_kubernetes_version = "${params.OKE_CLUSTER_VERSION}" 134 TF_VAR_nodepool_config = "${params.OKE_NODE_POOL}" 135 TF_VAR_api_fingerprint = credentials('oci-api-key-fingerprint') 136 TF_VAR_api_private_key_path = credentials('oci-api-key') 137 TF_VAR_s3_bucket_access_key = credentials('oci-s3-bucket-access-key') 138 TF_VAR_s3_bucket_secret_key = credentials('oci-s3-bucket-secret-key') 139 TF_VAR_ssh_public_key_path = credentials('oci-tf-pub-ssh-key') 140 141 OCI_CLI_TENANCY = credentials('oci-tenancy') 142 OCI_CLI_USER = credentials('oci-user-ocid') 143 OCI_CLI_FINGERPRINT = credentials('oci-api-key-fingerprint') 144 OCI_CLI_KEY_FILE = credentials('oci-api-key') 145 OCI_CLI_REGION = "${params.OKE_CLUSTER_REGION}" 146 OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING = 'True' 147 148 OPERATOR_YAML_FILE = "${WORKSPACE}/acceptance-test-operator.yaml" 149 150 INSTALL_CONFIG_FILE_KIND = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1beta1/install-vz-prod-kind-multicluster.yaml" 151 INSTALL_CONFIG_FILE_OCIDNS = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1beta1/install-verrazzano-ocidns.yaml" 152 INSTALL_CONFIG_FILE_NIPIO = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1beta1/install-verrazzano-nipio.yaml" 153 INSTALL_CONFIG_FILE_MINIMAL = "${WORKSPACE}/install-verrazzano-minimal.yaml" 154 OCI_DNS_ZONE_NAME="z${zoneId}.v8o.io" 155 ACME_ENVIRONMENT="${params.ACME_ENVIRONMENT}" 156 157 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 158 SHORT_TIME_STAMP = sh(returnStdout: true, script: "date +%m%d%H%M%S").trim() 159 TEST_SCRIPTS_DIR = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts" 160 LOOPING_TEST_SCRIPTS_DIR = "${TEST_SCRIPTS_DIR}/looping-test" 161 162 ADMIN_KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 163 164 // Environment variables required to capture cluster snapshot and bug report on test failure 165 DUMP_COMMAND="${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh" 166 TEST_DUMP_ROOT="${WORKSPACE}/test-cluster-snapshots" 167 CAPTURE_FULL_CLUSTER="${params.CAPTURE_FULL_CLUSTER}" 168 169 // Environment variable for Verrazzano CLI executable 170 VZ_COMMAND="${GO_REPO_PATH}/vz" 171 172 VERRAZZANO_INSTALL_LOGS_DIR="${WORKSPACE}/verrazzano/platform-operator/scripts/install/build/logs" 173 VERRAZZANO_INSTALL_LOG="verrazzano-install.log" 174 175 EXTERNAL_ELASTICSEARCH = "${params.EXTERNAL_ELASTICSEARCH}" 176 177 // used for console artifact capture on failure 178 JENKINS_READ = credentials('jenkins-auditor') 179 OCI_CLI_AUTH="instance_principal" 180 OCI_OS_NAMESPACE = credentials('oci-os-namespace') 181 OCI_OS_ARTIFACT_BUCKET="build-failure-artifacts" 182 VZ_CLI_TARGZ="vz-linux-amd64.tar.gz" 183 184 // used to emit metrics 185 PROMETHEUS_CREDENTIALS = credentials('prometheus-credentials') 186 TEST_ENV_LABEL = "${params.TEST_ENV}" 187 188 // sample app deployed before upgrade and UI console tests 189 SAMPLE_APP_NAME="hello-helidon" 190 SAMPLE_APP_NAMESPACE="hello-helidon-sample" 191 SAMPLE_APP_PROJECT="hello-helidon-sample-proj" 192 SAMPLE_APP_COMPONENT="hello-helidon-component" 193 194 // used by ToDoList example test 195 WEBLOGIC_PSW = credentials('weblogic-example-domain-password') 196 DATABASE_PSW = credentials('todo-mysql-password') 197 198 // used to generate Ginkgo test reports 199 TEST_REPORT = "test-report.xml" 200 GINKGO_REPORT_ARGS = "--junit-report=${TEST_REPORT} --keep-separate-reports=true" 201 TEST_REPORT_DIR = "${WORKSPACE}/tests/e2e" 202 } 203 204 stages { 205 stage('Clean workspace and checkout') { 206 steps { 207 sh """ 208 echo "${NODE_LABELS}" 209 """ 210 211 script { 212 EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS = getEffectiveDumpOnSuccess() 213 if (params.GIT_COMMIT_TO_USE == "NONE") { 214 echo "Specific GIT commit was not specified, use current head" 215 def scmInfo = checkout scm 216 env.GIT_COMMIT = scmInfo.GIT_COMMIT 217 env.GIT_BRANCH = scmInfo.GIT_BRANCH 218 } else { 219 echo "SCM checkout of ${params.GIT_COMMIT_TO_USE}" 220 def scmInfo = checkout([ 221 $class: 'GitSCM', 222 branches: [[name: params.GIT_COMMIT_TO_USE]], 223 doGenerateSubmoduleConfigurations: false, 224 extensions: [], 225 submoduleCfg: [], 226 userRemoteConfigs: [[url: env.SCM_VERRAZZANO_GIT_URL]]]) 227 env.GIT_COMMIT = scmInfo.GIT_COMMIT 228 env.GIT_BRANCH = scmInfo.GIT_BRANCH 229 // If the commit we were handed is not what the SCM says we are using, fail 230 if (!env.GIT_COMMIT.equals(params.GIT_COMMIT_TO_USE)) { 231 echo "SCM didn't checkout the commit we expected. Expected: ${params.GIT_COMMIT_TO_USE}, Found: ${scmInfo.GIT_COMMIT}" 232 exit 1 233 } 234 } 235 echo "SCM checkout of ${env.GIT_BRANCH} at ${env.GIT_COMMIT}" 236 } 237 238 sh """ 239 cp -f "${NETRC_FILE}" $HOME/.netrc 240 chmod 600 $HOME/.netrc 241 """ 242 243 script { 244 try { 245 sh """ 246 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 247 """ 248 } catch(error) { 249 echo "docker login failed, retrying after sleep" 250 retry(4) { 251 sleep(30) 252 sh """ 253 echo "${DOCKER_CREDS_PSW}" | docker login ${env.DOCKER_REPO} -u ${DOCKER_CREDS_USR} --password-stdin 254 """ 255 } 256 } 257 } 258 sh """ 259 rm -rf ${GO_REPO_PATH}/verrazzano 260 mkdir -p ${GO_REPO_PATH}/verrazzano 261 tar cf - . | (cd ${GO_REPO_PATH}/verrazzano/ ; tar xf -) 262 """ 263 script { 264 setVZCRDVersionForInstallation() 265 def props = readProperties file: '.verrazzano-development-version' 266 VERRAZZANO_DEV_VERSION = props['verrazzano-development-version'] 267 TIMESTAMP = sh(returnStdout: true, script: "date +%Y%m%d%H%M%S").trim() 268 SHORT_COMMIT_HASH = sh(returnStdout: true, script: "git rev-parse --short=8 HEAD").trim() 269 DOCKER_IMAGE_TAG = "${VERRAZZANO_DEV_VERSION}-${TIMESTAMP}-${SHORT_COMMIT_HASH}" 270 // update the description with some meaningful info 271 setDisplayName() 272 currentBuild.description = SHORT_COMMIT_HASH + " : " + env.GIT_COMMIT + " : " + params.GIT_COMMIT_TO_USE 273 274 if (params.TEST_ENV != "KIND") { 275 // derive the prefix for the OKE cluster 276 OKE_CLUSTER_PREFIX = sh(returnStdout: true, script: "${GO_REPO_PATH}/verrazzano/ci/scripts/derive_oke_cluster_name.sh").trim() 277 } 278 } 279 script { 280 sh """ 281 echo "Downloading VZ CLI from object storage" 282 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/${VZ_CLI_TARGZ} --file ${VZ_CLI_TARGZ} 283 tar xzf ${VZ_CLI_TARGZ} -C ${GO_REPO_PATH} 284 ${VZ_COMMAND} version 285 """ 286 } 287 } 288 } 289 290 stage('Install and Configure') { 291 when { 292 allOf { 293 not { buildingTag() } 294 anyOf { 295 branch 'master'; 296 expression {SKIP_ACCEPTANCE_TESTS == false}; 297 } 298 } 299 } 300 stages { 301 stage('Prepare AT environment') { 302 parallel { 303 stage('Create Kind Clusters') { 304 when { expression { return params.TEST_ENV == 'KIND' } } 305 steps { 306 createKindClusters() 307 } 308 } 309 stage('Create OKE Clusters') { 310 when { expression { return params.TEST_ENV == 'ocidns_oke' || params.TEST_ENV == 'magicdns_oke'} } 311 steps { 312 echo "OKE Cluster Prefix: ${OKE_CLUSTER_PREFIX}" 313 createOKEClusters("${OKE_CLUSTER_PREFIX}") 314 } 315 } 316 } 317 } 318 stage("Configure Clusters") { 319 parallel { 320 stage("Configure OKE/OCI DNS") { 321 when { expression { return params.TEST_ENV == 'ocidns_oke' } } 322 stages { 323 stage('Create OCI DNS zone') { 324 steps { 325 script { 326 dns_zone_ocid = sh(script: "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/oci_dns_ops.sh -o create -c ${TF_VAR_compartment_id} -s z${zoneId}", returnStdout: true) 327 } 328 } 329 } 330 stage('Configure OCI DNS Resources') { 331 environment { 332 OCI_DNS_COMPARTMENT_OCID = credentials('oci-dns-compartment') 333 OCI_PRIVATE_KEY_FILE = credentials('oci-api-key') 334 OCI_DNS_ZONE_OCID = "${dns_zone_ocid}" 335 } 336 steps { 337 script { 338 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 339 for(int count=1; count<=clusterCount; count++) { 340 sh """ 341 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 342 cd ${GO_REPO_PATH}/verrazzano 343 ./tests/e2e/config/scripts/create-test-oci-config-secret.sh 344 """ 345 } 346 } 347 } 348 } 349 stage('Configure OCI DNS Installers') { 350 environment { 351 OCI_DNS_COMPARTMENT_OCID = credentials('oci-dns-compartment') 352 OCI_PRIVATE_KEY_FILE = credentials('oci-api-key') 353 OCI_DNS_ZONE_OCID = "${dns_zone_ocid}" 354 } 355 steps { 356 script { 357 configureVerrazzanoInstallers(env.INSTALL_CONFIG_FILE_OCIDNS, "./tests/e2e/config/scripts/process_oci_dns_install_yaml.sh", "acme", params.ACME_ENVIRONMENT) 358 downloadMinimalVerrazzanoYaml() 359 configureManagedInstallers(env.INSTALL_CONFIG_FILE_MINIMAL, "./tests/e2e/config/scripts/process_oci_dns_install_yaml.sh", minimalFileName, params.ACME_ENVIRONMENT) 360 } 361 } 362 } 363 } 364 } 365 stage("Configure KinD") { 366 when { expression { return params.TEST_ENV == 'KIND' } } 367 steps { 368 configureVerrazzanoInstallers(env.INSTALL_CONFIG_FILE_KIND,"./tests/e2e/config/scripts/process_kind_install_yaml.sh", params.WILDCARD_DNS_DOMAIN) 369 downloadMinimalVerrazzanoYaml() 370 configureManagedInstallers(env.INSTALL_CONFIG_FILE_MINIMAL, "./tests/e2e/config/scripts/process_kind_install_yaml.sh", minimalFileName, params.WILDCARD_DNS_DOMAIN) 371 } 372 } 373 stage("Configure OKE/MagicDNS") { 374 when { expression { return params.TEST_ENV == 'magicdns_oke' } } 375 steps { 376 configureVerrazzanoInstallers(env.INSTALL_CONFIG_FILE_NIPIO, "./tests/e2e/config/scripts/process_nipio_install_yaml.sh", params.WILDCARD_DNS_DOMAIN) 377 downloadMinimalVerrazzanoYaml() 378 configureManagedInstallers(env.INSTALL_CONFIG_FILE_MINIMAL, "./tests/e2e/config/scripts/process_nipio_install_yaml.sh", minimalFileName, params.WILDCARD_DNS_DOMAIN) 379 } 380 } 381 } 382 } 383 stage("Minimal Managed Cluster Tests") { 384 stages { 385 stage('Install Verrazzano') { 386 steps { 387 script { 388 getVerrazzanoOperatorYaml() 389 } 390 installMinimalVerrazzano() 391 } 392 post { 393 always { 394 script { 395 dumpInstallLogs() 396 } 397 } 398 failure { 399 script { 400 dumpK8sCluster("${WORKSPACE}/install-failure-cluster-snapshot") 401 } 402 } 403 } 404 } 405 stage('Register min managed cluster') { 406 steps { 407 registerManagedClusters() 408 } 409 } 410 stage('Verify min managed cluster') { 411 parallel { 412 stage('Verify Install Cluster Agent') { 413 steps { 414 runGinkgoRandomize('verify-install/clusteragent') 415 } 416 } 417 stage('Verify Register Min Managed Cluster') { 418 steps { 419 verifyRegisterManagedClusters(true) 420 } 421 } 422 } 423 } 424 stage('Verify deregister min managed cluster') { 425 steps { 426 verifyDeregisterManagedClusters() 427 } 428 } 429 stage('Delete min managed cluster') { 430 steps { 431 deleteClusters(2) 432 } 433 } 434 } 435 } 436 stage('Prepare full managed cluster(s)') { 437 stages { 438 stage('Create KinD managed cluster') { 439 steps { 440 // create managed clusters, pass in false for cleanup existing kind clusters, since we want 441 // the admin cluster to be left as-is 442 createKindClusters(2, false) 443 } 444 } 445 stage('Configure KinD managed cluster') { 446 steps { 447 configureManagedInstallers(env.INSTALL_CONFIG_FILE_KIND,"./tests/e2e/config/scripts/process_kind_install_yaml.sh", installerFileName, params.WILDCARD_DNS_DOMAIN) 448 } 449 } 450 stage ('Install managed clusters') { 451 steps { 452 installManagedClusters() 453 } 454 } 455 } 456 } 457 458 stage ('Register managed clusters') { 459 steps { 460 registerManagedClusters() 461 } 462 } 463 stage ('Deploy Sample Application') { 464 steps { 465 deploySampleApp() 466 } 467 } 468 stage ('verify-register') { 469 steps { 470 verifyRegisterManagedClusters(false) 471 } 472 } 473 stage ('system component metrics') { 474 steps { 475 runGinkgoRandomize('metrics/syscomponents') 476 } 477 } 478 } 479 post { 480 failure { 481 script { 482 dumpK8sCluster("${WORKSPACE}/multicluster-install-cluster-snapshot") 483 } 484 } 485 } 486 } 487 488 stage ('Verify Install') { 489 stages { 490 stage("verify-fluentd-update") { 491 steps { 492 runGinkgoAdmin('update/fluentdextes') 493 } 494 } 495 stage("verify-dns-update") { 496 steps { 497 script { 498 parallel verifyDNSUpdate() 499 } 500 } 501 } 502 // cert-manager update suites related to admin and managed clusters should be run sequentially to 503 // avoid intermittent test failures. 504 stage("verify-admin-cluster-cert-manager-update") { 505 steps { 506 script { 507 runGinkgoAdmin('update/certac') 508 } 509 } 510 } 511 stage("verify-managed-clusters-cert-manager-update") { 512 steps { 513 script { 514 runGinkgoAdmin('update/certmc') 515 } 516 } 517 } 518 stage('verify-install') { 519 steps { 520 runGinkgoRandomize('verify-install') 521 } 522 } 523 stage('verify-argocd') { 524 steps { 525 runGinkgo('multicluster/verify-argocd', '${TEST_DUMP_ROOT}/verify-argocd') 526 } 527 } 528 stage('verify-rancher') { 529 steps { 530 runGinkgo('multicluster/verify-rancher', '${TEST_DUMP_ROOT}/verify-rancher') 531 } 532 } 533 stage('verify-cluster-sync') { 534 steps { 535 runGinkgoAdmin('multicluster/verify-cluster-sync') 536 } 537 } 538 stage ('mc verify-install') { 539 steps { 540 runGinkgoRandomize('multicluster/verify-install') 541 } 542 } 543 } 544 post { 545 failure { 546 script { 547 dumpK8sCluster("${WORKSPACE}/multicluster-verify-cluster-snapshot") 548 } 549 } 550 aborted { 551 script { 552 dumpK8sCluster("${WORKSPACE}/multicluster-verify-cluster-snapshot") 553 } 554 } 555 success { 556 script { 557 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 558 dumpK8sCluster("${WORKSPACE}/multicluster-verify-cluster-snapshot") 559 } 560 } 561 } 562 } 563 } 564 565 stage ('Verify Infra') { 566 steps { 567 script { 568 parallel verifyInfra() 569 } 570 } 571 post { 572 always { 573 archiveArtifacts artifacts: '**/coverage.html,**/logs/*', allowEmptyArchive: true 574 junit testResults: '**/*test-result.xml', allowEmptyResults: true 575 } 576 } 577 } 578 579 stage('Acceptance Tests') { 580 stages { 581 stage ('Example apps') { 582 steps { 583 script { 584 parallel verifyExamples() 585 } 586 } 587 } 588 stage ('Console') { 589 steps { 590 runConsoleTests() 591 } 592 post { 593 always { 594 sh "${GO_REPO_PATH}/verrazzano/ci/scripts/save_console_test_artifacts.sh" 595 } 596 } 597 } 598 stage ('Undeploy Sample Application') { 599 steps { 600 undeploySampleApp() 601 } 602 } 603 stage ('mc verify-api') { 604 steps { 605 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 606 script { 607 runMulticlusterVerifyApi() 608 } 609 } 610 } 611 post { 612 failure { 613 script { 614 dumpK8sCluster("${WORKSPACE}/multicluster-acceptance-tests-cluster-snapshot-pre-uninstall") 615 } 616 } 617 } 618 } 619 } 620 post { 621 failure { 622 script { 623 dumpK8sCluster("${WORKSPACE}/multicluster-acceptance-tests-cluster-snapshot") 624 } 625 } 626 aborted { 627 script { 628 dumpK8sCluster("${WORKSPACE}/multicluster-acceptance-tests-cluster-snapshot") 629 } 630 } 631 success { 632 script { 633 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 634 dumpK8sCluster("${WORKSPACE}/multicluster-acceptance-tests-cluster-snapshot") 635 } 636 } 637 } 638 } 639 } 640 stage('Cleanup Tests') { 641 stages { 642 stage('verify deregister') { 643 steps { 644 verifyDeregisterManagedClusters() 645 } 646 } 647 } 648 post { 649 failure { 650 script { 651 dumpK8sCluster("${WORKSPACE}/multicluster-cleanup-tests-cluster-snapshot") 652 } 653 } 654 } 655 } 656 stage('Uninstall Verrazzano') { 657 stages { 658 stage('Uninstall') { 659 steps { 660 uninstallVerrazzano() 661 } 662 } 663 stage('Verify Uninstall') { 664 steps { 665 verifyUninstall() 666 } 667 } 668 } 669 post { 670 failure { 671 script { 672 dumpK8sCluster("${WORKSPACE}/multicluster-uninstall-dump") 673 } 674 } 675 aborted { 676 script { 677 dumpK8sCluster("${WORKSPACE}/multicluster-uninstall-dump") 678 } 679 } 680 success { 681 script { 682 if (EFFECTIVE_DUMP_K8S_CLUSTER_ON_SUCCESS == true) { 683 dumpK8sCluster("${WORKSPACE}/multicluster-uninstall-dump") 684 } 685 } 686 } 687 always { 688 archiveArtifacts artifacts: "**/multicluster-uninstall-dump/**,**/*full-cluster*/**,**/*bug-report*/**,**/test-cluster-snapshots/**", allowEmptyArchive: true 689 } 690 } 691 } 692 } 693 post { 694 failure { 695 sh """ 696 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o ${WORKSPACE}/build-console-output.log ${BUILD_URL}consoleText 697 """ 698 archiveArtifacts artifacts: '**/build-console-output.log,**/Screenshot*.png,**/ConsoleLog*.log', allowEmptyArchive: true 699 // Ignore failures in any of the following actions so that the "always" post step that cleans up clusters is executed 700 sh """ 701 curl -k -u ${JENKINS_READ_USR}:${JENKINS_READ_PSW} -o archive.zip ${BUILD_URL}artifact/*zip*/archive.zip || true 702 oci --region us-phoenix-1 os object put --force --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_ARTIFACT_BUCKET} --name ${env.JOB_NAME}/${env.BRANCH_NAME}/${env.BUILD_NUMBER}/archive.zip --file archive.zip || true 703 rm archive.zip || true 704 """ 705 } 706 always { 707 script { 708 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 709 dumpVerrazzanoSystemPods() 710 dumpCattleSystemPods() 711 dumpNginxIngressControllerLogs() 712 dumpVerrazzanoPlatformOperatorLogs() 713 dumpVerrazzanoApplicationOperatorLogs() 714 dumpOamKubernetesRuntimeLogs() 715 dumpVerrazzanoApiLogs() 716 } 717 } 718 sh """ 719 # Copy the generated test reports to WORKSPACE to archive them 720 mkdir -p ${TEST_REPORT_DIR} 721 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 722 find . -name "${TEST_REPORT}" | cpio -pdm ${TEST_REPORT_DIR} 723 """ 724 archiveArtifacts artifacts: "**/*-operator.yaml,**/install-verrazzano.yaml,**/kube_config,**/coverage.html,**/logs/**,**/build/resources/**,**/verrazzano_images.txt,**/*full-cluster*/**,**/*bug-report*/**,**/test-cluster-snapshots/**,**/${TEST_REPORT},**/Screenshot*.png", allowEmptyArchive: true 725 junit testResults: "**/${TEST_REPORT}", allowEmptyResults: true 726 727 script { 728 sh """ 729 if [ -f ${POST_DUMP_FAILED_FILE} ]; then 730 echo "Failures seen during dumping of artifacts, treat post as failed" 731 exit 1 732 fi 733 """ 734 } 735 } 736 cleanup { 737 // Delete clusters as the very first thing in cleanup to reclaim cluster resources especially OKE resources 738 deleteClusters() 739 deleteDir() 740 } 741 } 742 } 743 744 def deleteOkeClusters() { 745 script { 746 sh """ 747 mkdir -p ${KUBECONFIG_DIR} 748 if [ "${TEST_ENV}" == "ocidns_oke" ]; then 749 cd ${GO_REPO_PATH}/verrazzano 750 ./tests/e2e/config/scripts/oci_dns_ops.sh -o delete -s z${zoneId} || echo "Failed to delete DNS zone z${zoneId}" 751 fi 752 cd ${TEST_SCRIPTS_DIR} 753 TF_VAR_label_prefix=${OKE_CLUSTER_PREFIX} TF_VAR_state_name=multicluster-${env.BUILD_NUMBER}-${env.BRANCH_NAME} ./delete_oke_cluster.sh "$clusterCount" "${KUBECONFIG_DIR}" || true 754 """ 755 } 756 } 757 758 def deleteClusters(start = 1) { 759 script { 760 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 761 if (env.TEST_ENV == "KIND") { 762 for(int count=start; count<=clusterCount; count++) { 763 sh """ 764 if [ "${env.TEST_ENV}" == "KIND" ] 765 then 766 kind delete cluster --name ${CLUSTER_NAME_PREFIX}-$count 767 fi 768 """ 769 } 770 } else { 771 deleteOkeClusters() 772 } 773 } 774 } 775 776 // Create a KinD cluster instance 777 // - count - the cluster index into $KUBECONFIG_DIR 778 // - metallbAddressRange - the address range to provide the Metallb install within the KinD Docker bridge network address range 779 // - cleanupKindContainers - indicates to the script whether or not to remove any existing clusters with the same name before creating the new one 780 // - connectJenkinsRunnerToNetwork - indicates whether or not to connect the KinD Docker bridge network to the Jenkins local docker network 781 def installKindCluster(count, metallbAddressRange, cleanupKindContainers, connectJenkinsRunnerToNetwork) { 782 // For parallel execution, wrap this in a Groovy enclosure {} 783 return script { 784 sh """ 785 echo ${CLUSTER_NAME_PREFIX}-$count 786 echo ${KUBECONFIG_DIR}/$count/kube_config 787 mkdir -p ${KUBECONFIG_DIR}/$count 788 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 789 echo "Create Kind cluster \$1" 790 cd ${TEST_SCRIPTS_DIR} 791 # As a stop gap, for now we are using the api/vpo caches here to see if it helps with rate limiting issues, we will need to add specific caches so for now 792 # specify the cache name based on the count value, this is assuming 1 or 2 clusters 793 case "${count}" in 794 1) 795 ./create_kind_cluster.sh "${CLUSTER_NAME_PREFIX}-$count" "${GO_REPO_PATH}/verrazzano/platform-operator" "${KUBECONFIG_DIR}/$count/kube_config" "${params.KIND_CLUSTER_VERSION}" "$cleanupKindContainers" "$connectJenkinsRunnerToNetwork" true ${params.CREATE_CLUSTER_USE_CALICO} "vpo_integ" 796 ;; 797 2) 798 ./create_kind_cluster.sh "${CLUSTER_NAME_PREFIX}-$count" "${GO_REPO_PATH}/verrazzano/platform-operator" "${KUBECONFIG_DIR}/$count/kube_config" "${params.KIND_CLUSTER_VERSION}" "$cleanupKindContainers" "$connectJenkinsRunnerToNetwork" true ${params.CREATE_CLUSTER_USE_CALICO} "apo_integ" 799 ;; 800 *) 801 ./create_kind_cluster.sh "${CLUSTER_NAME_PREFIX}-$count" "${GO_REPO_PATH}/verrazzano/platform-operator" "${KUBECONFIG_DIR}/$count/kube_config" "${params.KIND_CLUSTER_VERSION}" "$cleanupKindContainers" "$connectJenkinsRunnerToNetwork" false ${params.CREATE_CLUSTER_USE_CALICO} "NONE" 802 ;; 803 esac 804 if [ ${params.CREATE_CLUSTER_USE_CALICO} == true ]; then 805 echo "Install Calico" 806 cd ${GO_REPO_PATH}/verrazzano 807 ./ci/scripts/install_calico.sh "${CLUSTER_NAME_PREFIX}-$count" 808 fi 809 kubectl wait --for=condition=ready nodes/${CLUSTER_NAME_PREFIX}-$count-control-plane --timeout=5m 810 kubectl wait --for=condition=ready pods/kube-controller-manager-${CLUSTER_NAME_PREFIX}-$count-control-plane -n kube-system --timeout=5m 811 echo "Listing pods in kube-system namespace ..." 812 kubectl get pods -n kube-system 813 echo "Install metallb" 814 cd ${GO_REPO_PATH}/verrazzano 815 ./tests/e2e/config/scripts/install-metallb.sh $metallbAddressRange 816 echo "Deploy external es and create its secret on the admin cluster if EXTERNAL_ELASTICSEARCH is true" 817 CLUSTER_NUMBER=${count} ./tests/e2e/config/scripts/create-external-os.sh 818 """ 819 } 820 } 821 822 // Either download the specified release of the platform operator YAML, or create one 823 // using the specific operator image provided by the user. 824 def getVerrazzanoOperatorYaml() { 825 script { 826 sh """ 827 echo "Platform Operator Configuration" 828 cd ${GO_REPO_PATH}/verrazzano 829 if [ "NONE" == "${params.VERRAZZANO_OPERATOR_IMAGE}" ]; then 830 echo "Downloading operator.yaml from branch ${env.BRANCH_NAME} for commit ${SHORT_COMMIT_HASH}" 831 oci --region us-phoenix-1 os object get --namespace ${OCI_OS_NAMESPACE} -bn ${OCI_OS_COMMIT_BUCKET} --name ephemeral/${env.BRANCH_NAME}/${SHORT_COMMIT_HASH}/operator.yaml --file ${OPERATOR_YAML_FILE} 832 else 833 echo "Generating operator.yaml based on image name provided: ${params.VERRAZZANO_OPERATOR_IMAGE}" 834 env IMAGE_PULL_SECRETS=verrazzano-container-registry DOCKER_IMAGE=${params.VERRAZZANO_OPERATOR_IMAGE} ./tools/scripts/generate_operator_yaml.sh > ${OPERATOR_YAML_FILE} 835 fi 836 """ 837 } 838 } 839 840 // download the file for the minimal verrazzano install 841 def downloadMinimalVerrazzanoYaml() { 842 script { 843 sh """ 844 echo "Downloading minimal-thanos.yaml from branch ${env.BRANCH_NAME} to ${env.INSTALL_CONFIG_FILE_MINIMAL}" 845 wget -O ${env.INSTALL_CONFIG_FILE_MINIMAL} https://raw.githubusercontent.com/verrazzano/verrazzano/${env.BRANCH_NAME}/examples/multicluster/managed-clusters/minimal-thanos.yaml 846 """ 847 } 848 } 849 850 // Update Verrazzano on each of the managed clusters 851 def installManagedClusters() { 852 script { 853 def verrazzanoInstallStages = [:] 854 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 855 for(int count=2; count<=clusterCount; count++) { 856 def key = "vz-mgd-${count-1}" 857 def installerPath="${KUBECONFIG_DIR}/${count}/${installerFileName}" 858 verrazzanoInstallStages["${key}"] = installVerrazzanoOnCluster(count, installerPath, false) 859 } 860 parallel verrazzanoInstallStages 861 } 862 } 863 864 def installMinimalVerrazzano() { 865 script { 866 // Create a dictionary of Verrazzano install steps to be executed in parallel 867 // - the first one will always be the Admin cluster 868 // - clusters 2-max are managed clusters 869 def verrazzanoInstallStages = [:] 870 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 871 for(int count=1; count<=clusterCount; count++) { 872 def installerPath="${KUBECONFIG_DIR}/${count}/${minimalFileName}" 873 def key = "vz-mgd-${count-1}" 874 def minimalInstall = true 875 if (count == 1) { 876 key = "vz-admin" 877 installerPath="${KUBECONFIG_DIR}/${count}/${installerFileName}" 878 minimalInstall = false 879 } 880 verrazzanoInstallStages["${key}"] = installVerrazzanoOnCluster(count, installerPath, minimalInstall) 881 } 882 parallel verrazzanoInstallStages 883 } 884 } 885 886 // Install Verrazzano on a target cluster 887 // - count is the cluster index into the $KUBECONFIG_DIR 888 // - verrazzanoConfig is the Verrazzano CR to use to install VZ on the cluster 889 def installVerrazzanoOnCluster(count, verrazzanoConfig, minimalInstall) { 890 // For parallel execution, wrap this in a Groovy enclosure {} 891 return { 892 script { 893 sh """ 894 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 895 cd ${GO_REPO_PATH}/verrazzano 896 897 # Display the kubectl and cluster versions 898 kubectl version 899 # Display the VZ CLI version 900 ${VZ_COMMAND} version 901 902 echo "Create Image Pull Secrets" 903 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 904 ./tests/e2e/config/scripts/create-image-pull-secret.sh github-packages "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" 905 ./tests/e2e/config/scripts/create-image-pull-secret.sh ocr "${OCR_REPO}" "${OCR_CREDS_USR}" "${OCR_CREDS_PSW}" 906 907 # make sure ns exists and create secret in verrazzano-install ns 908 kubectl create namespace verrazzano-install || true 909 ./tests/e2e/config/scripts/check_verrazzano_ns_exists.sh verrazzano-install 910 ./tests/e2e/config/scripts/create-image-pull-secret.sh "${IMAGE_PULL_SECRET}" "${DOCKER_REPO}" "${DOCKER_CREDS_USR}" "${DOCKER_CREDS_PSW}" "verrazzano-install" 911 912 ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/cluster${count}/pre-install-resources 913 914 # Update VZ CR to enable required components 915 ./tests/e2e/config/scripts/multicluster_edit_vz.sh ${count} ${verrazzanoConfig} ${minimalInstall} 916 917 echo "Installing the Verrazzano Platform Operator" 918 time ${VZ_COMMAND} install --manifests ${OPERATOR_YAML_FILE} -f ${verrazzanoConfig} 919 """ 920 } 921 } 922 } 923 924 def uninstallVerrazzano() { 925 script { 926 // Create a dictionary of Verrazzano uninstall steps to be executed in parallel 927 // - the first one will always be the Admin cluster 928 // - clusters 2-max are managed clusters 929 def verrazzanoUninstallStages = [:] 930 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 931 for (int count = 1; count <= clusterCount; count++) { 932 def installerPath = "${KUBECONFIG_DIR}/${count}/${installerFileName}" 933 def key = "vz-mgd-${count - 1}" 934 if (count == 1) { 935 key = "vz-admin" 936 } 937 verrazzanoUninstallStages["${key}"] = uninstallVerrazzanoOnCluster(count, installerPath) 938 } 939 parallel verrazzanoUninstallStages 940 } 941 } 942 943 // Uninstall Verrazzano 944 // - count is the cluster index into the $KUBECONFIG_DIR 945 // - verrazzanoConfig is the Verrazzano CR to use to install VZ on the cluster 946 def uninstallVerrazzanoOnCluster(count, verrazzanoConfig) { 947 // For parallel execution, wrap this in a Groovy enclosure {} 948 return { 949 script { 950 sh """ 951 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 952 echo "Deleting \\$verrazzanoConfig" 953 time ${VZ_COMMAND} uninstall -y --timeout 45m 954 """ 955 } 956 } 957 } 958 959 // Verify uninstall on all clusters 960 def verifyUninstall() { 961 script { 962 // Create a dictionary of Verrazzano verify uninstall steps to be executed in parallel 963 // - the first one will always be the Admin cluster 964 // - clusters 2-max are managed clusters 965 def verifyUninstallStages = [:] 966 // disabling for managed cluster since complete removal of resources on managed cluster interferes with cluster 967 // management facilities in Rancher. See VZ-10055 968 //int clusterCount = params.TOTAL_CLUSTERS.toInteger() 969 int clusterCount = 1 970 for (int count = 1; count <= clusterCount; count++) { 971 def key = "vz-mgd-${count - 1}" 972 if (count == 1) { 973 key = "vz-admin" 974 } 975 verifyUninstallStages["${key}"] = verifyUninstallOnCluster(count) 976 } 977 parallel verifyUninstallStages 978 } 979 } 980 981 // Verify uninstall Verrazzano on a single cluster 982 // - count is the cluster index into the $KUBECONFIG_DIR 983 def verifyUninstallOnCluster(count) { 984 // For parallel execution, wrap this in a Groovy enclosure {} 985 return { 986 script { 987 sh """ 988 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 989 ${LOOPING_TEST_SCRIPTS_DIR}/dump_cluster.sh ${WORKSPACE}/verrazzano/build/resources/cluster${count}/post-uninstall-resources false 990 ${LOOPING_TEST_SCRIPTS_DIR}/verify_uninstall.sh ${WORKSPACE}/verrazzano/build/resources/cluster${count} 991 """ 992 } 993 } 994 } 995 996 // register all managed clusters 997 def registerManagedClusters() { 998 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 999 script { 1000 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1001 for(int count=2; count<=clusterCount; count++) { 1002 sh """ 1003 export MANAGED_CLUSTER_DIR="${KUBECONFIG_DIR}/${count}" 1004 export MANAGED_CLUSTER_NAME="managed${count-1}" 1005 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1006 export MANAGED_CLUSTER_ENV="mgd${count-1}" 1007 cd ${GO_REPO_PATH}/verrazzano 1008 ./tests/e2e/config/scripts/register_managed_cluster.sh 1009 """ 1010 } 1011 // ADMIN_VZ_VERSION_AT_REGISTRATION is used by verify register test 1012 env.ADMIN_VZ_VERSION_AT_REGISTRATION = sh(returnStdout: true, 1013 script:"KUBECONFIG=${ADMIN_KUBECONFIG} kubectl get verrazzano -o jsonpath='{.items[0].status.version}'").trim() 1014 print "Admin VZ version at registration is ${env.ADMIN_VZ_VERSION_AT_REGISTRATION}" 1015 } 1016 } 1017 } 1018 1019 // Verify the register of the managed clusters 1020 def verifyRegisterManagedClusters(skipLogging) { 1021 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1022 script { 1023 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1024 for(int count=2; count<=clusterCount; count++) { 1025 sh """ 1026 export MANAGED_CLUSTER_NAME="managed${count-1}" 1027 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1028 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1029 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" multicluster/verify-register/... -- --skipLogging=${skipLogging} 1030 """ 1031 } 1032 } 1033 } 1034 } 1035 1036 // Utility method to create a map that will be used later to perform parallel operations against multiple clusters 1037 Map createClusterExecutionsMap() { 1038 script { 1039 // Create a dictionary of steps to be executed in parallel 1040 // - the first one will always be the Admin cluster 1041 // - clusters 2-max are managed clusters 1042 def clusterExecutionsMap = [:] 1043 return clusterExecutionsMap 1044 } 1045 } 1046 1047 // Verify the deregister of the managed clusters 1048 def verifyDeregisterManagedClusters() { 1049 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1050 script { 1051 def verrazzanoDeregisterManagedClusterStages = createClusterExecutionsMap() 1052 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1053 for(int count=2; count<=clusterCount; count++) { 1054 verrazzanoDeregisterManagedClusterStages["${count} - Verify Deregister Managed Cluster"] = verifyDeregisterManagedCluster(count) 1055 } 1056 parallel verrazzanoDeregisterManagedClusterStages 1057 } 1058 } 1059 } 1060 1061 def verifyDeregisterManagedCluster(count) { 1062 // For parallel execution, wrap this in a Groovy enclosure {} 1063 return { 1064 script { 1065 sh """ 1066 export MANAGED_CLUSTER_NAME="managed${count-1}" 1067 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1068 export ADMIN_KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1069 ./tests/e2e/config/scripts/deregister_managed_cluster.sh 1070 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1071 ginkgo build multicluster/verify-deregister/ 1072 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" multicluster/verify-deregister/*.test 1073 """ 1074 } 1075 } 1076 } 1077 1078 // Verify the managed cluster permissions 1079 def verifyManagedClusterPermissions() { 1080 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1081 script { 1082 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1083 for(int count=2; count<=clusterCount; count++) { 1084 sh """ 1085 export MANAGED_CLUSTER_NAME="managed${count-1}" 1086 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1087 export MANAGED_ACCESS_KUBECONFIG="${KUBECONFIG_DIR}/${count}/managed_kube_config" 1088 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1089 ginkgo -v -stream --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" multicluster/verify-permissions/... 1090 """ 1091 } 1092 } 1093 } 1094 } 1095 1096 // Run ginkgo test suites 1097 def runGinkgo(testSuitePath, clusterDumpDirectory) { 1098 script { 1099 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1100 sh """ 1101 export KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1102 export MANAGED_CLUSTER_NAME="managed1" 1103 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/2/kube_config" 1104 export CLUSTER_COUNT=$clusterCount 1105 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1106 export DUMP_KUBECONFIG="${KUBECONFIG_DIR}/2/kube_config" 1107 export DUMP_DIRECTORY="${clusterDumpDirectory}" 1108 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 1109 """ 1110 } 1111 } 1112 1113 // Run a test suite against all clusters 1114 def runGinkgoRandomize(testSuitePath) { 1115 catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { 1116 script { 1117 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1118 def clusterName = "" 1119 for(int count=1; count<=clusterCount; count++) { 1120 // The first cluster created by this script is named as admin, and the subsequent clusters are named as 1121 // managed1, managed2, etc. 1122 if (count == 1) { 1123 clusterName="admin" 1124 } else { 1125 clusterName="managed${count-1}" 1126 } 1127 sh """ 1128 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1129 export CLUSTER_NAME="${clusterName}" 1130 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1131 ginkgo -p --randomize-all -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 1132 """ 1133 } 1134 } 1135 } 1136 } 1137 1138 // Run a test suite against just the admin cluster 1139 def runGinkgoRandomizeAdmin(testSuitePath) { 1140 sh """ 1141 export KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1142 export CLUSTER_NAME="admin" 1143 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1144 ginkgo -p --randomize-all -v --keep-going --no-color -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 1145 """ 1146 } 1147 1148 // Run a test suite against just the admin cluster 1149 def runGinkgoAdmin(testSuitePath) { 1150 sh """ 1151 export KUBECONFIG="${KUBECONFIG_DIR}/1/kube_config" 1152 export CLUSTER_NAME="admin" 1153 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1154 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" ${testSuitePath}/... 1155 """ 1156 } 1157 1158 // Configure the Admin and Managed cluster installer custom resources 1159 def configureVerrazzanoInstallers(installResourceTemplate, configProcessorScript, String... extraArgs) { 1160 script { 1161 // Concatenate the variable args into a single string 1162 String allArgs = "" 1163 extraArgs.each { allArgs += it + " " } 1164 1165 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1166 for(int count=1; count<=clusterCount; count++) { 1167 def destinationPath = "${env.KUBECONFIG_DIR}/${count}/${installerFileName}" 1168 def installProfile = env.MANAGED_CLUSTER_PROFILE 1169 // Installs using OCI DNS require a unique domain per cluster 1170 // - also, the env name must be <= 10 chars for some reason. 1171 def envName = "mgd${count-1}" 1172 if (count == 1) { 1173 // Cluster "1" is always the admin cluster, use the chosen profile for the VZ install 1174 // with the env name "admin" 1175 installProfile = env.ADMIN_CLUSTER_PROFILE 1176 envName = "admin" 1177 } 1178 sh """ 1179 mkdir -p "${KUBECONFIG_DIR}/${count}" 1180 export PATH=${HOME}/go/bin:${PATH} 1181 cd ${GO_REPO_PATH}/verrazzano 1182 # Copy the template config over for the mgd cluster profile configuration 1183 cp $installResourceTemplate $destinationPath 1184 VZ_ENVIRONMENT_NAME="${envName}" INSTALL_PROFILE=$installProfile $configProcessorScript $destinationPath $allArgs 1185 """ 1186 } 1187 } 1188 } 1189 1190 def configureManagedInstallers(installResourceTemplate, configProcessorScript, filename, String... extraArgs) { 1191 script { 1192 // Concatenate the variable args into a single string 1193 String allArgs = "" 1194 extraArgs.each { allArgs += it + " " } 1195 1196 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1197 1198 // start at 2 to skip the admin cluster 1199 for(int count=2; count<=clusterCount; count++) { 1200 def destinationPath = "${env.KUBECONFIG_DIR}/${count}/${filename}" 1201 def installProfile = env.MANAGED_CLUSTER_PROFILE 1202 def envName = "mgd${count-1}" 1203 sh """ 1204 cd ${GO_REPO_PATH}/verrazzano 1205 # Copy the template config over for the mgd cluster profile configuration 1206 cp $installResourceTemplate $destinationPath 1207 VZ_ENVIRONMENT_NAME="${envName}" INSTALL_PROFILE=$installProfile $configProcessorScript $destinationPath $allArgs 1208 """ 1209 } 1210 } 1211 } 1212 1213 // Create the required KinD clusters 1214 def createKindClusters(start = 1, cleanupKindContainers = true) { 1215 script { 1216 sh """ 1217 echo "tests will execute" > ${TESTS_EXECUTED_FILE} 1218 """ 1219 // NOTE: Eventually we should be able to parallelize the cluster creation, however 1220 // we seem to be getting some kind of timing issue on cluster create; the 2nd 1221 // cluster always seems to get a connect/timeout issue, so for now we are keeping 1222 // the KinD cluster creation serial 1223 1224 // Can these for now, but eventually we should be able to build this based on 1225 // inspecting the Docker bridge network CIDR and splitting up a range based on 1226 // the cluster count. 1227 1228 def addressRanges = [ "172.18.0.231-172.18.0.238", "172.18.0.239-172.18.0.246", "172.18.0.247-172.18.0.254"] 1229 def clusterInstallStages = [:] 1230 boolean connectJenkinsRunnerToNetwork = true 1231 if(start > 1) { 1232 connectJenkinsRunnerToNetwork = false 1233 } 1234 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1235 for(int count=start; count<=clusterCount; count++) { 1236 string metallbAddressRange = addressRanges.get(count-1) 1237 def deployStep = "cluster-${count}" 1238 // Create dictionary of steps for parallel execution 1239 //clusterInstallStages[deployStep] = installKindCluster(count, metallbAddressRange, cleanupKindContainers, connectJenkinsRunnerToNetwork) 1240 // For sequential execution of steps 1241 installKindCluster(count, metallbAddressRange, cleanupKindContainers, connectJenkinsRunnerToNetwork) 1242 cleanupKindContainers = false 1243 connectJenkinsRunnerToNetwork = false 1244 } 1245 // Execute steps in parallel 1246 //parallel clusterInstallStages 1247 } 1248 } 1249 1250 // Invoke the OKE cluster creation script for the desired number of clusters 1251 def createOKEClusters(clusterPrefix) { 1252 script { 1253 sh """ 1254 echo "tests will execute" > ${TESTS_EXECUTED_FILE} 1255 """ 1256 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1257 sh """ 1258 mkdir -p ${KUBECONFIG_DIR} 1259 echo "Create OKE cluster" 1260 cd ${TEST_SCRIPTS_DIR} 1261 TF_VAR_label_prefix=${clusterPrefix} TF_VAR_state_name=multicluster-${env.BUILD_NUMBER}-${env.BRANCH_NAME} ./create_oke_multi_cluster.sh "$clusterCount" "${KUBECONFIG_DIR}" ${params.CREATE_CLUSTER_USE_CALICO} 1262 """ 1263 } 1264 } 1265 1266 def dumpK8sCluster(dumpDirectory) { 1267 script { 1268 if ( fileExists(env.TESTS_EXECUTED_FILE) ) { 1269 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1270 for (int count = 1; count <= clusterCount; count++) { 1271 sh """ 1272 export KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1273 ${GO_REPO_PATH}/verrazzano/ci/scripts/capture_cluster_snapshot.sh ${dumpDirectory}/cluster-snapshot-${count} 1274 """ 1275 } 1276 } 1277 } 1278 } 1279 1280 def dumpVerrazzanoSystemPods() { 1281 script { 1282 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1283 for(int count=1; count<=clusterCount; count++) { 1284 LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1285 sh """ 1286 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1287 mkdir -p ${LOG_DIR} 1288 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-system-pods.log" 1289 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 1290 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-system-certs.log" 1291 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE} 1292 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-system-osd.log" 1293 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE} 1294 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-system-es-master.log" 1295 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-es-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE} 1296 """ 1297 } 1298 } 1299 } 1300 1301 def dumpCattleSystemPods() { 1302 script { 1303 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1304 for(int count=1; count<=clusterCount; count++) { 1305 LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1306 sh """ 1307 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1308 mkdir -p ${LOG_DIR} 1309 export DIAGNOSTIC_LOG="${LOG_DIR}/cattle-system-pods.log" 1310 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE} 1311 export DIAGNOSTIC_LOG="${LOG_DIR}/rancher.log" 1312 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 1313 """ 1314 } 1315 } 1316 } 1317 1318 def dumpNginxIngressControllerLogs() { 1319 script { 1320 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1321 for(int count=1; count<=clusterCount; count++) { 1322 LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1323 sh """ 1324 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1325 mkdir -p ${LOG_DIR} 1326 export DIAGNOSTIC_LOG="${LOG_DIR}/nginx-ingress-controller.log" 1327 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 1328 """ 1329 } 1330 } 1331 } 1332 1333 def dumpVerrazzanoPlatformOperatorLogs() { 1334 script { 1335 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1336 for(int count=1; count<=clusterCount; count++) { 1337 LOG_DIR="${WORKSPACE}/verrazzano-platform-operator/logs/cluster-$count" 1338 sh """ 1339 ## dump out verrazzano-platform-operator logs 1340 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1341 mkdir -p ${LOG_DIR} 1342 kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${LOG_DIR}/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 1343 kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${LOG_DIR}/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 1344 echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log" 1345 echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out" 1346 echo "------------------------------------------" 1347 """ 1348 } 1349 } 1350 } 1351 1352 def dumpVerrazzanoApplicationOperatorLogs() { 1353 script { 1354 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1355 for(int count=1; count<=clusterCount; count++) { 1356 LOG_DIR="${WORKSPACE}/verrazzano-application-operator/logs/cluster-$count" 1357 sh """ 1358 ## dump out verrazzano-application-operator logs 1359 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1360 mkdir -p ${LOG_DIR} 1361 kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${LOG_DIR}/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 1362 kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${LOG_DIR}/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 1363 echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log" 1364 echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out" 1365 echo "------------------------------------------" 1366 """ 1367 } 1368 } 1369 } 1370 1371 def dumpOamKubernetesRuntimeLogs() { 1372 script { 1373 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1374 for(int count=1; count<=clusterCount; count++) { 1375 LOG_DIR="${WORKSPACE}/oam-kubernetes-runtime/logs/cluster-$count" 1376 sh """ 1377 ## dump out oam-kubernetes-runtime logs 1378 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1379 mkdir -p ${LOG_DIR} 1380 kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${LOG_DIR}/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE} 1381 kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${LOG_DIR}/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE} 1382 echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log" 1383 echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out" 1384 echo "------------------------------------------" 1385 """ 1386 } 1387 } 1388 } 1389 1390 def dumpVerrazzanoApiLogs() { 1391 script { 1392 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1393 for(int count=1; count<=clusterCount; count++) { 1394 LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1395 sh """ 1396 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1397 mkdir -p ${LOG_DIR} 1398 export DIAGNOSTIC_LOG="${LOG_DIR}/verrazzano-authproxy.log" 1399 ${GO_REPO_PATH}/verrazzano/platform-operator/scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE} 1400 """ 1401 } 1402 } 1403 } 1404 1405 def dumpInstallLogs() { 1406 script { 1407 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1408 for(int count=1; count<=clusterCount; count++) { 1409 LOG_DIR="${VERRAZZANO_INSTALL_LOGS_DIR}/cluster-$count" 1410 1411 // This function may run on older versions of Verrazzano that have the logs stored in the default namespace 1412 def namespace = "verrazzano-install" 1413 sh """ 1414 ## dump Verrazzano install logs 1415 export KUBECONFIG=${KUBECONFIG_DIR}/$count/kube_config 1416 mkdir -p ${LOG_DIR} 1417 kubectl -n ${namespace} logs --selector=job-name=verrazzano-install-my-verrazzano > ${LOG_DIR}/${VERRAZZANO_INSTALL_LOG} --tail -1 1418 kubectl -n ${namespace} describe pod --selector=job-name=verrazzano-install-my-verrazzano > ${LOG_DIR}/verrazzano-install-job-pod.out 1419 echo "------------------------------------------" 1420 """ 1421 } 1422 } 1423 } 1424 1425 def getEffectiveDumpOnSuccess() { 1426 def effectiveValue = params.DUMP_K8S_CLUSTER_ON_SUCCESS 1427 if (FORCE_DUMP_K8S_CLUSTER_ON_SUCCESS.equals("true") && (env.BRANCH_NAME.equals("master"))) { 1428 effectiveValue = true 1429 echo "Forcing dump on success based on global override setting" 1430 } 1431 return effectiveValue 1432 } 1433 1434 def setDisplayName() { 1435 echo "Start setDisplayName" 1436 def causes = currentBuild.getBuildCauses() 1437 echo "causes: " + causes.toString() 1438 for (cause in causes) { 1439 def causeString = cause.toString() 1440 echo "current cause: " + causeString 1441 if (causeString.contains("UpstreamCause") && causeString.contains("Started by upstream project")) { 1442 echo "This job was caused by " + causeString 1443 if (causeString.contains("verrazzano-periodic-triggered-tests")) { 1444 currentBuild.displayName = env.BUILD_NUMBER + " : PERIODIC" 1445 } else if (causeString.contains("verrazzano-flaky-tests")) { 1446 currentBuild.displayName = env.BUILD_NUMBER + " : FLAKY" 1447 } 1448 } 1449 } 1450 echo "End setDisplayName" 1451 } 1452 1453 def modifyHelloHelidonApp(newNamespace, newProjName) { 1454 sh """ 1455 # create modified versions of the hello helidon MC example 1456 export MC_HH_DEST_DIR=${GO_REPO_PATH}/verrazzano/examples/multicluster/${newNamespace} 1457 export MC_HH_SOURCE_DIR=${GO_REPO_PATH}/verrazzano/examples/multicluster/hello-helidon 1458 export MC_APP_NAMESPACE="${newNamespace}" 1459 export MC_PROJ_NAME="${newProjName}" 1460 ${GO_REPO_PATH}/verrazzano/ci/scripts/generate_mc_hello_deployment_files.sh 1461 """ 1462 } 1463 1464 def deploySampleApp() { 1465 modifyHelloHelidonApp("${SAMPLE_APP_NAMESPACE}", "${SAMPLE_APP_PROJECT}") 1466 sh """ 1467 export KUBECONFIG=$ADMIN_KUBECONFIG 1468 kubectl apply -f ${GO_REPO_PATH}/verrazzano/examples/multicluster/hello-helidon-sample/verrazzano-project.yaml 1469 kubectl apply -f ${GO_REPO_PATH}/verrazzano/examples/multicluster/hello-helidon-sample/hello-helidon-comp.yaml 1470 kubectl apply -f ${GO_REPO_PATH}/verrazzano/examples/multicluster/hello-helidon-sample/mc-hello-helidon-app.yaml 1471 """ 1472 } 1473 1474 def undeploySampleApp() { 1475 sh """ 1476 export KUBECONFIG=$ADMIN_KUBECONFIG 1477 kubectl delete -f ${GO_REPO_PATH}/verrazzano/examples/multicluster/hello-helidon-sample/mc-hello-helidon-app.yaml 1478 kubectl delete -f ${GO_REPO_PATH}/verrazzano/examples/multicluster/hello-helidon-sample/hello-helidon-comp.yaml 1479 kubectl delete -f ${GO_REPO_PATH}/verrazzano/examples/multicluster/hello-helidon-sample/verrazzano-project.yaml 1480 """ 1481 } 1482 1483 def runHelidonNsOpsTest() { 1484 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1485 modifyHelloHelidonApp("hello-helidon-ns", "hello-helidon-ns") 1486 sh """ 1487 export MANAGED_CLUSTER_NAME="managed1" 1488 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/2/kube_config" 1489 export CLUSTER_COUNT=$clusterCount 1490 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1491 export DUMP_KUBECONFIG="${KUBECONFIG_DIR}/2/kube_config" 1492 export DUMP_DIRECTORY="${TEST_DUMP_ROOT}/examples-helidon-ns-ops" 1493 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" multicluster/examples/helidon-ns-ops/... 1494 """ 1495 } 1496 1497 def runMulticlusterVerifyApi() { 1498 int clusterCount = params.TOTAL_CLUSTERS.toInteger() 1499 for(int count=2; count<=clusterCount; count++) { 1500 sh """ 1501 export MANAGED_CLUSTER_NAME="managed${count-1}" 1502 export MANAGED_KUBECONFIG="${KUBECONFIG_DIR}/${count}/kube_config" 1503 cd ${GO_REPO_PATH}/verrazzano/tests/e2e 1504 ginkgo -v --keep-going --no-color ${GINKGO_REPORT_ARGS} -tags="${params.TAGGED_TESTS}" --focus-file="${params.INCLUDED_TESTS}" --skip-file="${params.EXCLUDED_TESTS}" multicluster/verify-api/... 1505 """ 1506 } 1507 } 1508 1509 def runConsoleTests() { 1510 // Set app information used by the application page UI tests to assert for app info 1511 // Console runs on admin cluster and the KUBECONFIG is pointed at it (which is cluster 1) 1512 // Make sure that application page tests are also run by setting RUN_APP_TESTS=true since we deployed 1513 // a sample app for that purpose 1514 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 1515 sh """ 1516 export DUMP_DIRECTORY="${TEST_DUMP_ROOT}/console" 1517 export CONSOLE_REPO_BRANCH="${params.CONSOLE_REPO_BRANCH}" 1518 export CONSOLE_APP_NAME="${SAMPLE_APP_NAME}" 1519 export CONSOLE_APP_NAMESPACE="${SAMPLE_APP_NAMESPACE}" 1520 export CONSOLE_APP_CLUSTER="managed1" 1521 export CONSOLE_APP_COMP="${SAMPLE_APP_COMPONENT}" 1522 KUBECONFIG=${ADMIN_KUBECONFIG} RUN_APP_TESTS=true ${GO_REPO_PATH}/verrazzano/ci/scripts/run_console_tests.sh 1523 """ 1524 } 1525 } 1526 1527 def verifyUpgrade() { 1528 return [ 1529 "Verify Register": { 1530 verifyRegisterManagedClusters() 1531 }, 1532 "verify-permissions": { 1533 verifyManagedClusterPermissions() 1534 }, 1535 "system component metrics": { 1536 runGinkgoRandomize('metrics/syscomponents') 1537 }, 1538 ] 1539 } 1540 1541 def verifyDNSUpdate() { 1542 return [ 1543 "verify-admin-cluster-dns-update": { 1544 runGinkgoAdmin('update/dnsac') 1545 }, 1546 "verify-managed-cluster-dns-update": { 1547 runGinkgoAdmin('update/dnsmc') 1548 }, 1549 ] 1550 } 1551 1552 // NOTE: The stages are executed in parallel, however the tests themselves are executed against 1553 // each cluster in sequence. If possible, we should parallelize that as well. 1554 def verifyInfra() { 1555 return [ 1556 "verify-scripts": { 1557 runGinkgoRandomize('scripts') 1558 }, 1559 "verify-infra restapi": { 1560 runGinkgoRandomize('verify-infra/restapi') 1561 }, 1562 "verify-infra oam": { 1563 runGinkgoRandomize('verify-infra/oam') 1564 }, 1565 "verify-infra vmi": { 1566 runGinkgoRandomize('verify-infra/vmi') 1567 }, 1568 "mc verify-jaeger-install": { 1569 runGinkgoRandomize('multicluster/verify-jaeger/install') 1570 }, 1571 "mc verify-jaeger-system": { 1572 runGinkgoRandomize('multicluster/verify-jaeger/system') 1573 }, 1574 ] 1575 } 1576 1577 def verifyExamples() { 1578 return [ 1579 "mc examples helidon": { 1580 runGinkgo("multicluster/examples/helidon", "${TEST_DUMP_ROOT}/helidon-workload") 1581 }, 1582 "mc examples helidon deprecated": { 1583 runGinkgo("multicluster/examples/helidon-deprecated", "${TEST_DUMP_ROOT}/helidon-deprecated") 1584 }, 1585 "Delete Deployed App NS": { 1586 runHelidonNsOpsTest() 1587 }, 1588 "mc weblogic workload": { 1589 runGinkgo("multicluster/workloads/mcweblogic", "${TEST_DUMP_ROOT}/weblogic-workload") 1590 }, 1591 "mc coherence workload": { 1592 runGinkgo("multicluster/workloads/mccoherence", "${TEST_DUMP_ROOT}/coherence-workload") 1593 }, 1594 "jaeger helidon": { 1595 runGinkgo("multicluster/verify-jaeger/helidon", "${TEST_DUMP_ROOT}/jaeger-helidon") 1596 }, 1597 ] 1598 } 1599 1600 def setVZCRDVersionForInstallation(){ 1601 if(params.CRD_API_VERSION == "v1alpha1"){ 1602 INSTALL_CONFIG_FILE_KIND = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1alpha1/install-vz-prod-kind-multicluster.yaml" 1603 INSTALL_CONFIG_FILE_OCIDNS = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1alpha1/install-verrazzano-ocidns.yaml" 1604 INSTALL_CONFIG_FILE_NIPIO = "${GO_REPO_PATH}/verrazzano/tests/e2e/config/scripts/v1alpha1/install-verrazzano-nipio.yaml" 1605 } 1606 }